jpmschweitzer/settled-reach
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b0bfbfc7ddc7c3293c8dddaa77303871e4a2ddee
settled-reach/tooling/planet-gen/gemma_naming.py
T
jpmschweitzerandClaude Opus 4.6 29f9945291 feat(db): body_radius_km scatter + gas giant/moon scale classes 
Deterministic ±scatter on body radii seeded by body_id hash — no two
bodies share the same radius. Gas giants 40k-60k km, moons 200-2600 km,
rocky planets ±15% from class base. Oort/asteroid skip radius (NULL).
Sol system gets real planetary radii. body_radius_km exported to
star_map_data.json for client orbital diagram sizing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-05-03 20:11:31 +02:00

2538 lines
98 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
gemma_naming.py — Batch-name every empty name field in the reach's
markers.json files using the Gemma 4 E2B tooling pipeline (#833, D-191 §4).
Pipeline per body:
1. Load markers.json; identify feature records whose `name` is empty
or null. Hand-authored names are never overwritten.
2. Build a short corridor-aware prompt per feature.
3. Stream the prompts into `sr-voice serve --stdio` (long-lived
subprocess, restarted every --refresh requests to prevent KV-cache
context bleed).
4. Post-process each response: strip quotes, trim whitespace, reject
blocklisted Earth majors, retry with a bumped seed on collision or
on blocklist hit (up to 3 attempts), fall back to a deterministic
palette-driven name on persistent failure.
5. Dedup within (cultural_corridor, feature_type) so two bodies in the
same corridor never ship the same river name; cross-corridor
collisions are allowed (two "Aldren"s on opposite arcs is fine).
6. Write markers.json back (only if any field changed).
7. Sync every touched body's atlas_* rows in systems.db so
`atlas_cities.name`, `atlas_rivers.name`, etc. pick up the new
strings without needing a follow-up generate_atlas.py pass.
Usage:
tooling/planet-gen/gemma_naming.py # full batch, real model
tooling/planet-gen/gemma_naming.py --body GJ380c # single body
tooling/planet-gen/gemma_naming.py --limit 5 --verbose # smoke test
tooling/planet-gen/gemma_naming.py --mock # mock-stdio.sh (no model)
tooling/planet-gen/gemma_naming.py \\
--sr-voice ~/Projects/settled-reach/binaries/sr-voice-tooling \\
--model ~/Projects/settled-reach/models/gemma-4.gguf
Exit codes:
0 pipeline completed (possibly with skipped bodies)
1 fatal error (subprocess crash, missing binary, missing schema)
"""
import argparse
import datetime
import hashlib
import json
import os
import re
import signal
import subprocess
import sys
import time
from pathlib import Path
TOOLING_DIR = Path(__file__).resolve().parent
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
# Reuse the atlas DB sync logic from generate_atlas.py so there is one
# authoritative path for atlas_* row updates.
sys.path.insert(0, str(TOOLING_DIR))
from generate_atlas import ( # noqa: E402
GRID_H,
GRID_W,
ensure_atlas_schema,
sync_markers_to_db,
)
import sqlite3 # noqa: E402
from naming_core import ( # noqa: E402
name_features_batch,
mood_for_body,
)
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"
BLOCKLIST_PATH = TOOLING_DIR / "earth_blocklist.txt"
# NOTE: The following are vestigial from the Gemma 2 single-name pipeline.
# The live path uses _batch_fill() → name_features_batch() from naming_core.
# TODO(#833): remove in a cleanup pass. Full dead-code island (~750 lines):
# - _CAPTURE_FILE, --dump-prompts argparse (here + main())
# - _build_prompt() and its few-shot example pools (~lines 500-929)
# - post_process(), is_placeholder(), _PLACEHOLDER_TOKENS, _LABEL_PREFIX,
# _MD_BOLD, _MD_UNDER (~lines 1040-1100)
# - is_blocked(), load_blocklist() (~lines 1105-1140)
# - fallback_name(), _FALLBACK_STEMS, _FALLBACK_SUFFIXES (~lines 1145-1205)
# - name_feature() with its retry loop and _is_duplicate (~lines 1530-1655)
_CAPTURE_FILE = None # vestigial — see note above
# Default binary + model paths. The sr-voice binary is platform-specific
# (GPU backend baked in per-build) and lives OUTSIDE any git worktree so
# persistent across worktree cleanup:
#
# ~/Projects/settled-reach/binaries/sr-voice-rocm (AMD / ROCm)
# ~/Projects/settled-reach/binaries/sr-voice-cuda (NVIDIA, future)
# ~/Projects/settled-reach/binaries/sr-voice-vulkan (cross-vendor, future)
# ~/Projects/settled-reach/binaries/sr-voice-cpu (fallback)
#
# See #850 for the multi-backend release-binary ticket. The container
# used to build these is created via the distrobox recipe documented in
# the sr-voice README.
#
# The Gemma 2 model weights live under main/server/models and are shared
# across worktrees (too large to duplicate).
HOME_PROJECTS = Path.home() / "Projects" / "settled-reach"
BINARIES_DIR = HOME_PROJECTS / "binaries"
MODELS_DIR = HOME_PROJECTS / "models"
MAIN_WORKDIR = Path("/var/mnt/data/projects/settled-reach/main")
def _find_sr_voice() -> Path:
"""Resolve the default sr-voice binary path.
Preference order:
1. $HOME/Projects/settled-reach/binaries/sr-voice-tooling — Gemma 4
tooling binary, preferred for content generation.
2. $HOME/Projects/settled-reach/binaries/sr-voice-rocm — Gemma 2
ROCm binary, fallback.
3. main workdir's target/release/sr-voice — legacy.
"""
tooling_bin = BINARIES_DIR / "sr-voice-tooling"
if tooling_bin.exists():
return tooling_bin
rocm_bin = BINARIES_DIR / "sr-voice-rocm"
if rocm_bin.exists():
return rocm_bin
return MAIN_WORKDIR / "server" / "sr-voice" / "target" / "release" / "sr-voice"
def _find_default_model() -> Path:
"""Resolve the default model path. Prefers Gemma 4 over Gemma 2."""
gemma4 = MODELS_DIR / "gemma-4.gguf"
if gemma4.exists():
return gemma4
return MAIN_WORKDIR / "server" / "models" / "gemma2.gguf"
DEFAULT_SR_VOICE = _find_sr_voice()
DEFAULT_MODEL = _find_default_model()
MOCK_STDIO = REPO_ROOT / "server" / "sr-voice" / "mock-stdio.sh"
# ---------------------------------------------------------------------------
# Tee logger — stdout + log file in one call
# ---------------------------------------------------------------------------
class Logger:
"""Write lines to stdout AND an optional log file.
Every message gets a prefix of the form `[HH:MM:SS +00h03m]`:
- HH:MM:SS is wall-clock local time,
- +NNhMMm is the elapsed time since the Logger was constructed.
The elapsed offset tells the user at a glance how long the run has
been going without scrolling back to the banner line. Flushes after
every line so a kill -9 loses at most one entry.
"""
def __init__(self, log_path: Path | None):
self.log_path = log_path
self.started_at = time.monotonic()
self.fh = None
if log_path is not None:
log_path.parent.mkdir(parents=True, exist_ok=True)
# Truncate on open so each run starts fresh — the user can
# rename an old log before kicking off the next run.
self.fh = log_path.open("w", buffering=1) # line buffered
def _elapsed(self) -> str:
secs = int(time.monotonic() - self.started_at)
return f"+{secs // 3600:02d}h{(secs % 3600) // 60:02d}m"
def _prefix(self) -> str:
clock = datetime.datetime.now().strftime("%H:%M:%S")
return f"[{clock} {self._elapsed()}]"
def __call__(self, msg: str = "") -> None:
line = f"{self._prefix()} {msg}" if msg else ""
print(line, flush=True)
if self.fh is not None:
self.fh.write(line + "\n")
self.fh.flush()
def raw(self, msg: str = "") -> None:
"""Print without the timestamp prefix (for banner lines)."""
print(msg, flush=True)
if self.fh is not None:
self.fh.write(msg + "\n")
self.fh.flush()
def close(self) -> None:
if self.fh is not None:
self.fh.close()
self.fh = None
# ---------------------------------------------------------------------------
# Corridor palettes (D-191 §4, glossary.md §Corridors, decisions/economics.md D-175)
# ---------------------------------------------------------------------------
# Each palette is the cultural inflection the prompt asks Gemma to
# produce names in. Palette keys match the values of
# `star_systems.geographic_sector` directly — that column is the real
# source of corridor identity in systems.db (cultural_corridor is a
# legacy field that was never populated beyond sol-gateway-axis).
CORRIDOR_SUBSTYLES: dict[str, list[dict[str, str]]] = {
# Each corridor has a list of sub-style inflections. The pipeline
# picks one per body via hash(body_id) so neighbouring bodies on the
# same planet get different registers, and Gemma's narrow per-register
# vocabulary (~15 stems) stays fresh across hundreds of bodies.
#
# IMPORTANT: the inflection is a DOMINANT bias, not a hard lock.
# A British surveyor on an east_reach moon still names a river after
# their aunt in Dorset. Each sub-style explicitly names its register
# AND invites diaspora variety.
"core": [
{"inflection": "English countryside, rural, agricultural settlers",
"examples": "Thornbury, Bramblewood, Millbrook, Ashford, Weston"},
{"inflection": "British colonial settlement era",
"examples": "New Bristol, Port Augusta, Kingstown, Admiralty, Georgetown"},
{"inflection": "American frontier, practical, geographic",
"examples": "Dusty Creek, Twin Oaks, Cedar Flat, Hawk's Hollow, Red Bluff"},
{"inflection": "American municipal, administrative, cosmopolitan",
"examples": "Prospect Heights, Liberty, Union, Meridian, Commonwealth"},
{"inflection": "Classical references, institutional, civic",
"examples": "Concordia, Aurelius, Prefecture, Senate Landing, Forum"},
{"inflection": "Australian and New Zealand settler",
"examples": "Redfern, Glenelg, Wollongong, Kaikoura, Hawke's Bay"},
],
"north_reach": [
{"inflection": "English rural, village and parish names",
"examples": "Wolcott, Mildern, Ashbourne, Briarfell, Tarndale"},
{"inflection": "Scottish Highland and Lowland place-names",
"examples": "Glenmoray, Dunfermline, Kinross, Brae, Dalwhinnie"},
{"inflection": "Australian outback, station and property names",
"examples": "Redfern, Birdsville, Tennant, Woomera, Coober"},
{"inflection": "Irish rural and coastal settlement",
"examples": "Ballymore, Kilrush, Dunmore, Tralee, Skellig"},
{"inflection": "South African English settler",
"examples": "Grahamstown, Oudtshoorn, Stellenbosch, Graaff, Beaufort"},
],
"south_reach": [
{"inflection": "Portuguese colonial era, Iberian",
"examples": "Monteforte, Serra, Tavira, Oliveira, Porto Novo"},
{"inflection": "Brazilian interior, frontier settlement",
"examples": "Ribeirão, Campo Largo, Várzea, Ilhabela, Pinheiro"},
{"inflection": "East African Swahili coastal",
"examples": "Inhambane, Kilimi, Ngola, Manhica, Quelimane"},
{"inflection": "Cape Verdean and West African",
"examples": "Cabo, Moçambo, Ribeira, Mindelo, Tarrafal"},
{"inflection": "Angolan and Mozambican settlement",
"examples": "Huambo, Lobito, Nampula, Lichinga, Benguela"},
],
"east_reach": [
{"inflection": "Korean place-name tradition",
"examples": "Hanyang, Seorak, Baektu, Saeyeon, Taegong"},
{"inflection": "Japanese rural and coastal settlement",
"examples": "Takamine, Ginoza, Tsukuri, Aomori, Fukagawa"},
{"inflection": "Taiwanese and Hakka settler",
"examples": "Jiufen, Beigang, Hsinchu, Meinong, Tainan"},
{"inflection": "Filipino settler community",
"examples": "Batangas, Legazpi, Tuguegarao, Zambales, Tarlac"},
{"inflection": "Mixed East Asian diaspora, cosmopolitan",
"examples": "Naruhan, Morimine, Kōzan, Midori, Kawasaki"},
],
"west_reach": [
{"inflection": "German settlement, orderly and compound names",
"examples": "Altdorf, Drachenberg, Feldberg, Krakenberg, Lüneborg"},
{"inflection": "Dutch colonial, low-country",
"examples": "Kloosterdam, Hoogland, Oudewater, Nieuwpoort, Voorhout"},
{"inflection": "Nordic and Scandinavian",
"examples": "Sørholm, Torsfell, Bergfjord, Lindeborg, Nordhölm"},
{"inflection": "Polish and Czech settler",
"examples": "Krakowice, Bystrica, Wieliczka, Tarnów, Ostrava"},
{"inflection": "Baltic and Finnish settler",
"examples": "Järvenpää, Tallinna, Pärnu, Turku, Rakvere"},
],
"deep_frontier": [
{"inflection": "frontier founder-name era, surname-first, any Earth culture",
"examples": "Okafor Reach, Stenner Cross, Weller Hold, Pruitt Basin"},
{"inflection": "frontier descriptive, geographic features named by surveyors",
"examples": "Red Mesa, Dry Fork, Iron Flat, Long Ridge, Dust Basin"},
{"inflection": "frontier outpost, functional and military",
"examples": "Forward Post, Relay Station, Survey Camp, Waypoint, Anchor"},
],
}
# Legacy aliases
CORRIDOR_SUBSTYLES["sol-gateway-axis"] = CORRIDOR_SUBSTYLES["core"]
CORRIDOR_SUBSTYLES["inner_corridor"] = CORRIDOR_SUBSTYLES["core"]
CORRIDOR_SUBSTYLES["inner_orbit"] = CORRIDOR_SUBSTYLES["core"]
CORRIDOR_SUBSTYLES["frontier"] = CORRIDOR_SUBSTYLES["deep_frontier"]
DEFAULT_SUBSTYLES = CORRIDOR_SUBSTYLES["core"]
# Processing order for the main run — core first so those bodies win
# the dedup race and the outer sectors fall into the palette fallback
# path when names collide.
SECTOR_PRIORITY: dict[str, int] = {
"core": 0,
"north_reach": 1,
"south_reach": 2,
"east_reach": 3,
"west_reach": 4,
"deep_frontier": 5,
}
def palette_for(corridor: str | None, system_id: str = "") -> dict[str, str]:
"""Pick a sub-style for this system within its corridor.
All bodies in the same system get the same sub-style (consistent
cultural register per star system). Different systems rotate through
the sub-style list via hash(system_id).
This is the FALLBACK path — the preferred path is select_register()
which asks Gemma to pick the register based on wiki/GTTR content.
"""
substyles = CORRIDOR_SUBSTYLES.get(corridor or "core", DEFAULT_SUBSTYLES)
idx = int(hashlib.sha256(system_id.encode()).hexdigest()[:8], 16) % len(substyles)
return substyles[idx]
def _system_slug(system_id: str) -> str:
"""Convert system_id ('GJ 411') to wiki directory slug ('GJ-411')."""
if system_id.startswith("GJ "):
return "GJ-" + system_id[3:]
return system_id
def load_wiki_context(system_id: str) -> tuple[str | None, str | None]:
"""Read index.md and gttr.md for a system from wiki/star-systems/.
Returns (index_text, gttr_text). Either or both may be None if the
file doesn't exist.
"""
slug = _system_slug(system_id)
sys_dir = WIKI_SYSTEMS / slug
index_path = sys_dir / "index.md"
gttr_path = sys_dir / "gttr.md"
index_text = index_path.read_text() if index_path.exists() else None
gttr_text = gttr_path.read_text() if gttr_path.exists() else None
return index_text, gttr_text
def _extract_cultural_lines(wiki_text: str, max_lines: int = 8) -> str:
"""Pull the most culturally relevant lines from a wiki index.md.
Scans for lines mentioning heritage, founding identity, language,
cultural texture, or corridor affiliation. Falls back to the first
prose paragraphs if no keyword hits. Keeps the excerpt short enough
for Gemma 2 2B's 1024-token context.
"""
keywords = (
"cultural", "heritage", "founding", "settler", "surname",
"language", "tradition", "diaspora", "population carried",
"portuguese", "iberian", "japanese", "korean", "chinese",
"filipino", "german", "dutch", "nordic", "scandinavian",
"polish", "czech", "finnish", "baltic", "swahili", "african",
"angolan", "cape verde", "irish", "scottish", "australian",
"british", "brazilian", "mozambic", "norwegian", "frisian",
"afrikaans", "lusophone", "corridor",
)
hits: list[str] = []
prose: list[str] = []
for line in wiki_text.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or stripped.startswith("|") or stripped.startswith("---") or stripped.startswith("<!--"):
continue
low = stripped.lower()
if any(kw in low for kw in keywords):
hits.append(stripped[:200])
elif len(prose) < max_lines:
prose.append(stripped[:200])
selected = hits[:max_lines] if hits else prose[:max_lines]
return "\n".join(selected)
def select_register(
voice: "VoiceSubprocess",
corridor: str,
system_id: str,
wiki_text: str | None,
gttr_text: str | None,
log: "Logger",
max_attempts: int = 3,
) -> dict[str, str] | None:
"""Ask Gemma to pick the best cultural register for this system.
Presents the corridor's sub-style options numbered 1..N alongside
a compact cultural excerpt (gttr_hook + key wiki lines). Gemma
replies with just the number. Returns the selected sub-style dict,
or None if all attempts fail (caller falls back to hash-based
palette_for).
"""
substyles = CORRIDOR_SUBSTYLES.get(corridor, DEFAULT_SUBSTYLES)
if not wiki_text and not gttr_text:
return None
# Build numbered option list — inflection only, no examples, to
# save tokens. Gemma needs to match cultural identity, not mimic
# example names.
options: list[str] = []
for idx, style in enumerate(substyles, 1):
options.append(f"{idx}. {style['inflection']}")
option_block = "\n".join(options)
# Build a compact context block that fits in ~400 tokens.
# The gttr_hook is a 30-45 word summary; the wiki excerpt adds
# the strongest cultural-identity lines.
context_parts: list[str] = []
if gttr_text:
# Use the first substantive GTTR paragraph — skip the title
# line (# THE DRIFTER'S GUIDE ...) and any blank lines.
for para in gttr_text.strip().split("\n\n"):
cleaned = para.replace("#", "").strip()
# Skip title lines and section headers
if cleaned.startswith("THE DRIFTER") or cleaned.startswith("DRIFTER"):
continue
if not cleaned or len(cleaned) < 20:
continue
context_parts.append(cleaned[:300])
break
if wiki_text:
cultural = _extract_cultural_lines(wiki_text)
if cultural:
context_parts.append(cultural)
if not context_parts:
return None
context = "\n".join(context_parts)
# Few-shot format: Gemma 2 2B is much better at pattern completion
# than instruction following. Show 2 worked examples, then the
# target system. Keep examples short and from different corridors
# than the target so they don't bias the answer.
# Fixed preamble + tail that frame the completion pattern.
preamble = (
"Match the star system to the best cultural naming register.\n\n"
"System: Neustadt — German-heritage industrial town, west corridor, orderly municipal governance.\n"
"1. German settlement 2. Dutch colonial 3. Nordic 4. Polish/Czech 5. Baltic/Finnish\n"
"Best: 1\n\n"
"System: Matsue — Japanese precision manufacturing hub, east corridor.\n"
"1. Korean 2. Japanese 3. Taiwanese/Hakka 4. Filipino 5. Mixed East Asian\n"
"Best: 2\n\n"
"System: "
)
tail = f"\n{option_block}\nBest (number only):"
# Reserve tokens for preamble, tail, and a few output tokens.
# Rough estimate: 1 token ≈ 4 chars for English prose.
max_prompt_chars = (voice.ctx_size - 16) * 4 # 16 tokens headroom for output
budget = max_prompt_chars - len(preamble) - len(tail)
if budget < 100:
budget = 100
if len(context) > budget:
context = context[:budget]
prompt = f"{preamble}{context}{tail}"
for attempt in range(max_attempts):
seed = int(
hashlib.sha256(
f"register|{system_id}|{attempt}".encode()
).hexdigest()[:8],
16,
)
try:
raw = voice.request(prompt, seed)
except RuntimeError:
continue
# Extract the first number from the response. Gemma may reply
# "1", "1.", "Option 1", "Answer: 1", etc.
digits = re.search(r"\d+", raw.strip() or "")
if not digits:
continue
try:
choice = int(digits.group())
except ValueError:
continue
if 1 <= choice <= len(substyles):
return substyles[choice - 1]
return None
# ---------------------------------------------------------------------------
# Feature prompt templates — few-shot format with rotating example pools
# ---------------------------------------------------------------------------
# Gemma 2 2B is small and noisy on free-form instruction prompts — it
# loves to echo the prompt back as "[River Name]" / "NAME: ..." /
# "**River: X**" etc. The fix is few-shot: show concrete
# `Style → Answer` examples so the model completes a pattern instead
# of generating to an open-ended instruction.
#
# Two lessons from earlier iterations:
# 1. The examples are the ONLY thing the model actually learns from.
# If they're all epic/classical (Wolcott Beck, Nakamura Stream),
# the model completes in epic/classical register for every body.
# Grounded outputs (Cooper's Creek, West Ridge, Mill Run) require
# grounded examples.
# 2. A single static example set produces uniform output: same prompt
# + similar seeds → similar completions. Rotating through a pool
# of example sets per call injects variation and nudges the
# sampler into different regions of the output distribution.
#
# Each feature type has a POOL of example sets. `_build_prompt()` picks
# one set deterministically per (body_id, local_id) so the same feature
# always gets the same prompt but neighbouring features get different
# prompts. The pools emphasise grounded / first-person / prosaic names
# with the occasional classical one — matching how real settlers on
# frontier worlds actually named places.
#
# Preamble wording matters too: "Settlers name …" reminds the model that
# these are human-chosen names, not fantasy coinages. The negative
# constraint "Most names are mundane" reinforces the grounded bias.
# Each pool entry is a list of `(style_label, example_name)` pairs. The
# style labels are cross-corridor — they teach Gemma the pattern, not
# a specific corridor's vocabulary. The target corridor's inflection
# gets substituted at the END of the prompt.
_RIVER_POOLS: list[list[tuple[str, str]]] = [
# Pool 0 — possessive, surnames dominant with a first-name mixed in
[
("British/Australian", "Cooper's Creek"),
("Irish", "Maura's Run"), # first name
("Dutch", "Van Dael's Beek"),
("Italian", "Fiume Bruno"),
("Japanese", "Tanaka Stream"),
("Polish", "Kowalski Potok"),
],
# Pool 1 — compass / descriptive, mixed cultures
[
("Australian", "West Brook"),
("Nordic", "Nordälven"),
("French", "Ruisseau du Nord"),
("Japanese", "Kita-gawa"),
("Swahili", "Mto wa Kaskazini"),
("Hungarian", "Északi Patak"),
],
# Pool 2 — colour / feature observation
[
("Irish", "Blackwater"),
("German", "Braunbach"),
("Spanish", "Río Verde"),
("Russian", "Chornaya Rechka"),
("Korean", "Ha-gang"),
("Portuguese", "Ribeira Negra"),
],
# Pool 3 — short single-word / old-world prosaic
[
("British", "Mill Run"),
("Dutch", "Oude Wetering"),
("Nordic", "Stenbäck"),
("Japanese", "Sakura-gawa"),
("Portuguese", "Ribeiro Seco"),
("Czech", "Starý Potok"),
],
# Pool 4 — founder surname + feature
[
("British/Australian", "Garner Creek"),
("Dutch", "Meijer Beek"),
("Nordic", "Sveinsström"),
("Korean", "Choi Stream"),
("Italian", "Fiume Marconi"),
("Greek", "Petrakis Rema"),
],
# Pool 5 — founder FIRST name possessive (Clifford's Bay shape)
# Added so first-name-possessive naming joins the rotation alongside
# the surname pools without replacing any of them.
[
("British", "Clifford's Bay"),
("Irish", "Maura's Run"),
("Japanese", "Yuki's Pool"),
("Italian", "Rio di Marco"),
("Portuguese", "Rio de Ana"),
("French", "Rivière d'Elena"),
],
# Pool 6 — classical / institutional / Latinate (occasional ~17%)
[
("British/Australian", "Aqueduct Run"),
("Italian", "Acqua Vetusta"),
("Spanish", "Río Antiguo"),
("French", "Vieille Rivière"),
("German", "Altwasser"),
],
]
_MOUNTAIN_POOLS: list[list[tuple[str, str]]] = [
# Pool 0 — compass / direct observation (the "Western Ridge" shape)
[
("British/Australian", "Western Ridge"),
("Dutch", "Noordrug"),
("Nordic", "Sørkammen"),
("Japanese", "Minami-yama"),
("Portuguese", "Serra do Sul"),
("Hungarian", "Északi Hát"),
],
# Pool 1 — surname + feature, cosmopolitan
[
("British/Australian", "Drayton Hills"),
("Italian", "Monti Rovere"),
("Dutch", "Van Dijk Heuvels"),
("Korean", "Park Sanmaek"),
("Polish", "Góry Brzeskie"),
("French", "Crête Valmont"),
],
# Pool 2 — colour / shape descriptor
[
("British", "The Long Spine"),
("Japanese", "Shiro-yama"),
("Russian", "Bely Khrebet"),
("Portuguese", "Serra Branca"),
("German", "Blauberg"),
("Spanish", "Sierra Roja"),
],
# Pool 3 — short single-word
[
("British", "Fell Back"),
("Japanese", "Takamine"),
("Dutch", "Klipfjord"),
("Nordic", "Torsfell"),
("Portuguese", "Cabeço"),
("German", "Eichfels"),
],
# Pool 4 — something-the-settlers-said (The-word / definite-article)
[
("British", "The Backbone"),
("Spanish", "El Espinazo"),
("Italian", "La Schiena"),
("Russian", "Khrebet"),
("Portuguese", "O Dorso"),
("French", "L'Épine"),
],
# Pool 5 — classical / institutional / Latinate (occasional ~17%)
[
("British/Australian", "Cassian Range"),
("Italian", "Monti Augusti"),
("Portuguese", "Monte Augusto"),
("Latin", "Mons Cassianus"),
("French", "Massif Aurélien"),
],
# Pool 6 — founder FIRST name + feature
[
("British", "Clifford's Ridge"),
("Irish", "Maeve's Back"),
("Japanese", "Keiko's Peak"),
("Spanish", "Sierra de Elena"),
("French", "Crête de Pierre"),
("Russian", "Anushka Khrebet"),
],
]
_LAKE_POOLS: list[list[tuple[str, str]]] = [
[
("British", "Cold Tarn"),
("Dutch", "Winterplas"),
("Nordic", "Kalltjärn"),
("Japanese", "Shizuko"),
("Portuguese", "Lagoa Funda"),
("Finnish", "Kylmäjärvi"),
],
[
("British/Australian", "Mildern Mere"),
("Italian", "Lago d'Argento"),
("Japanese", "Aoike"),
("Polish", "Jezioro Srebrne"),
("German", "Bergsee"),
("French", "Lac Clair"),
],
[
("British", "Three Oaks Pool"),
("Dutch", "Driehoekplas"),
("Japanese", "Midori-ike"),
("Hungarian", "Három Tölgy Tava"),
("Portuguese", "Poça Grande"),
("Spanish", "Laguna Grande"),
],
]
_OCEAN_POOLS: list[list[tuple[str, str]]] = [
[
("British", "Tarnsea"),
("Japanese", "Aomi"),
("Nordic", "Nordhav"),
("Portuguese", "Mar do Sul"),
("Dutch", "Zuidzee"),
("Italian", "Mare Meridio"),
],
[
("British", "The Long Main"),
("Japanese", "Kuro-umi"),
("Nordic", "Stormsø"),
("Portuguese", "Mar Profundo"),
("Russian", "Bolshoye More"),
("French", "Grand Large"),
],
]
_SEA_POOLS: list[list[tuple[str, str]]] = [
[
("British", "Harven Sea"),
("Japanese", "Minami-kai"),
("Nordic", "Sønderhav"),
("Portuguese", "Mar de Quelim"),
("Italian", "Mare Toscano"),
("Dutch", "Zeebocht"),
],
[
("British", "Cold Gulf"),
("Japanese", "Nagi-kai"),
("Nordic", "Iskullfjord"),
("Portuguese", "Golfo das Ilhas"),
("Polish", "Zatoka Zimna"),
("German", "Tiefbucht"),
],
]
_CITY_CAPITAL_POOLS: list[list[tuple[str, str]]] = [
# Pool 0 — founder / homestead / surname-town
[
("British/Australian", "Holmwood"),
("Dutch", "Van Damhoeve"),
("Japanese", "Yuna"),
("Italian", "Borgo Marconi"),
("Polish", "Kowalowo"),
("Portuguese", "Vila Moreira"),
],
# Pool 1 — compass + old-country place name
[
("British", "Westfield"),
("Nordic", "Sørholm"),
("Japanese", "Kita-sato"),
("French", "Saint-Nord"),
("Hungarian", "Kelethegy"),
("German", "Südkamp"),
],
# Pool 2 — short rooted stem (farm / town / kiln etc)
[
("British", "Kiln"),
("Dutch", "Stenen"),
("Japanese", "Sora"),
("Portuguese", "Paço"),
("Italian", "Forno"),
("Czech", "Starovice"),
],
# Pool 3 — explicitly mundane / functional
[
("British", "Landing"),
("Nordic", "Brygga"),
("Japanese", "Habu"),
("Portuguese", "Cabo"),
("Dutch", "Haven"),
("French", "Débarquement"),
],
# Pool 4 — classical / institutional / Latinate (occasional ~20%)
[
("British/Australian", "Meridian"),
("Italian", "Augusta"),
("Portuguese", "Porto Imperial"),
("Latin", "Solarium"),
("French", "Saint-Aurélien"),
],
# Pool 5 — founder FIRST name settlement
[
("British", "Clifford's Landing"),
("Irish", "Maura's Cross"),
("Japanese", "Yuki-mura"),
("Italian", "Villa di Marco"),
("Portuguese", "Vila Helena"),
("French", "Chez Pierre"),
],
]
_CITY_SECONDARY_POOLS: list[list[tuple[str, str]]] = [
[
("British/Australian", "Carberry"),
("Korean/Japanese", "Yurigawa"),
("Dutch", "Kleindorp"),
("Italian", "Piccola Villa"),
("Polish", "Nowawieś"),
("Portuguese", "Ribeirão"),
],
[
("British/Australian", "Garner's Cross"),
("French", "Sainte-Marie"),
("Japanese", "Tanaka-no-mura"),
("Nordic", "Sveinsby"),
("Hungarian", "Kiskút"),
("Portuguese", "Vila Nova"),
],
[
("British", "Kelstern"),
("Japanese", "Shirakawa"),
("Dutch", "Hoogland"),
("Czech", "Starovice"),
("Spanish", "Alta Vista"),
("German", "Talhöhe"),
],
[
("British", "Mill End"),
("Japanese", "Shimo-machi"),
("Nordic", "Nedreby"),
("French", "Les Moulins"),
("Portuguese", "Marginal"),
("Italian", "Fondobasso"),
],
# Classical / Latinate (occasional)
[
("British", "Prospect"),
("Japanese", "Seishin"),
("Italian", "Porta Aurea"),
("Portuguese", "Pórtico"),
("French", "Consulat"),
],
# Founder FIRST-name settlements
[
("British", "Clifford's Ferry"),
("Irish", "Maeve's Quay"),
("Japanese", "Yuki-no-mura"),
("Italian", "Casa Elena"),
("Portuguese", "Vila de Ana"),
("French", "Saint-Martin"),
],
]
_POI_TRANSIT_POOLS: list[list[tuple[str, str]]] = [
[
("British", "Holmwood Gate Terminal"),
("Japanese", "Yurigawa Transit"),
("Dutch", "Noordpoort Gate Terminal"),
("Portuguese", "Porto Exchange"),
("French", "Gare du Nord Concourse"),
],
[
("British", "West Gate Terminal"),
("Nordic", "Brygga Transit"),
("Italian", "Porta Vecchia"),
("Japanese", "Kita-sato Transit"),
("German", "Steinhof Gate Terminal"),
],
]
_POI_INSTITUTIONAL_POOLS: list[list[tuple[str, str]]] = [
[
("British", "Holmwood Assembly Hall"),
("Japanese", "Seungmun Archive"),
("Italian", "Palazzo Civico"),
("Portuguese", "Câmara Municipal"),
("German", "Altes Rathaus"),
],
[
("British", "Founders' Registry"),
("French", "Registre Général"),
("Japanese", "Kō Records Office"),
("Polish", "Archiwum Miejskie"),
("Dutch", "Burgerhuis"),
],
]
_POI_CULTURAL_POOLS: list[list[tuple[str, str]]] = [
[
("British", "The Commons"),
("Japanese", "Yurigawa Grounds"),
("Italian", "Piazza Nuova"),
("Portuguese", "Praça do Vento"),
("German", "Marktplatz"),
],
[
("British", "The Meeting House"),
("French", "Place des Fondateurs"),
("Japanese", "Sakura Grounds"),
("Polish", "Rynek Stary"),
("Nordic", "Gamle Torget"),
],
]
# Map feature type → pools + preamble + length hint + extra context hook.
_PROMPT_CONFIG: dict[str, dict] = {
"river": {
"pools": _RIVER_POOLS,
"subject": "rivers",
"length_hint": "1-3 words",
},
"ocean": {
"pools": _OCEAN_POOLS,
"subject": "oceans",
"length_hint": "1-3 words",
},
"sea": {
"pools": _SEA_POOLS,
"subject": "seas",
"length_hint": "1-3 words",
},
"lake": {
"pools": _LAKE_POOLS,
"subject": "lakes",
"length_hint": "1-3 words",
},
"mountain_range": {
"pools": _MOUNTAIN_POOLS,
"subject": "mountain ranges",
"length_hint": "1-3 words",
},
"city_capital": {
"pools": _CITY_CAPITAL_POOLS,
"subject": "their capital town",
"length_hint": "1-2 words",
"include_planet": True,
},
"city_secondary": {
"pools": _CITY_SECONDARY_POOLS,
"subject": "their secondary towns",
"length_hint": "1-2 words",
"include_planet": True,
},
"poi_transit": {
"pools": _POI_TRANSIT_POOLS,
"subject": "gate terminals / transit hubs",
"length_hint": "2-3 words ending in 'Gate Terminal', 'Transit', "
"'Exchange', or 'Concourse'",
},
"poi_institutional": {
"pools": _POI_INSTITUTIONAL_POOLS,
"subject": "institutional landmarks",
"length_hint": "2-4 words",
},
"poi_cultural": {
"pools": _POI_CULTURAL_POOLS,
"subject": "cultural landmarks",
"length_hint": "2-4 words",
},
}
def _build_prompt(
feature_type: str,
inflection: str,
planet_class: str,
body_id: str,
local_id: str,
attempt: int,
system_hook: str | None = None,
system_name: str | None = None,
body_name: str | None = None,
) -> str:
"""Assemble a few-shot prompt for the given feature type.
The example pool rotates per call via a deterministic hash of
(body_id, local_id, attempt) — this puts a finger on the sampling
scales so neighbouring features on the same body don't all draw
from an identical prompt and collapse to identical outputs.
`system_hook` is the pre-extracted gttr.md characterisation for
this system (e.g. "RAN is, by all accounts, a very nice place to
live, provided you are the sort of person whose grandparents owned
the land"). When present it gets woven into the preamble so Gemma
has per-system cultural context — this is the single biggest lever
for producing names that feel like they belong to THIS world rather
than any world in this corridor. When absent the prompt degrades to
the corridor inflection alone.
"""
cfg = _PROMPT_CONFIG.get(feature_type)
if cfg is None:
return ""
pools: list[list[tuple[str, str]]] = cfg["pools"]
# Deterministic pool pick: same feature always hits the same pool on
# attempt 0; retries rotate forward so a rejected name gets a
# different example set, not just a different seed.
salt = int(
hashlib.sha256(f"{body_id}|{local_id}|{attempt}".encode()).hexdigest()[:8],
16,
)
pool = pools[salt % len(pools)]
subject = cfg["subject"]
# Use "named X" for collective/plural subjects, "called X" for
# singular possessive ones ("their capital town"). Heuristic: if the
# subject starts with "their", use "called"; otherwise "named".
verb = "called" if subject.startswith("their ") else "named"
# Context block. When system / body proper names are available,
# label them explicitly so Gemma has the canonical identifiers in
# addition to the gttr hook. Redundancy helps on a small model —
# the hook already opens with the system name but putting it in a
# labeled "System:" line makes it harder to gloss over. When the
# gttr hook exists too, it goes on its own line as the cultural
# one-liner.
context_lines: list[str] = []
ident_parts = []
if system_name:
ident_parts.append(f"System: {system_name}")
if body_name:
ident_parts.append(f"Planet: {body_name}")
if ident_parts:
context_lines.append(". ".join(ident_parts) + ".")
if system_hook:
context_lines.append(f"About the system: {system_hook}")
preamble = (
f"Settlers {verb} {subject} after themselves, after what they saw, "
f"or after places back home. Most names are mundane, short, and "
f"direct — a surname, a compass direction, a feature, a practical "
f"description. Classical or epic names are rare. "
f"Avoid the obvious choice. Do not repeat the system or planet name. "
f"Each name should be unique and surprising within its register. "
f"Reply with ONLY the name, {cfg['length_hint']}, no brackets, "
f"no quotes, no markdown, no label."
)
lines = [preamble, ""]
if context_lines:
lines.extend(context_lines)
lines.append("")
for style, example in pool:
lines.append(f"Style: {style}. Answer: {example}")
lines.append("")
tail = f"Style: {inflection}."
if cfg.get("include_planet"):
tail += f" Planet: {planet_class}."
tail += " Answer:"
lines.append(tail)
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Blocklist
# ---------------------------------------------------------------------------
def load_blocklist(path: Path = BLOCKLIST_PATH) -> set[str]:
"""Load earth_blocklist.txt into a lowercase set for exact-match checks."""
if not path.exists():
return set()
blocked: set[str] = set()
for line in path.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
blocked.add(line.lower())
return blocked
_THE_PREFIX = re.compile(r"^the\s+", re.IGNORECASE)
def is_blocked(name: str, blocklist: set[str]) -> bool:
"""True iff the case-insensitive name appears in the blocklist.
Also strips a leading "The " before comparing, so "The Great Divide"
and "Great Divide" both match a `great divide` entry. Prefixed
variants like 'Nouveau Paris', 'Neu Berlin', 'New Tokyo' still do
NOT match because those prefixes are substantive (a new place), while
"The" is just the definite article.
"""
lc = name.strip().lower()
if lc in blocklist:
return True
stripped = _THE_PREFIX.sub("", lc).strip()
return stripped in blocklist
def is_placeholder(name: str) -> bool:
"""True iff the cleaned name looks like Gemma echoed the prompt
or drifted into verbose poetry rather than producing a usable name.
Catches:
- Empty, <3 chars, or pure punctuation after cleaning.
- Any whole-word token from _PLACEHOLDER_TOKENS appearing in the
name ('Name', 'Example', 'Placeholder', 'TBD', …).
- Strings that start with 'River ', 'City ', 'Lake ', etc. with a
single-letter suffix (clear prompt fragments).
- 5+ word outputs — the prompt requests 1-3 words; when Gemma
drifts into 'The Grand Lake of the Astral Sea' territory it's
generating description, not a name. 4 words is the practical
upper bound for clean Latin/Korean/Germanic 3-token names with
a 'The' prefix.
"""
s = name.strip()
if len(s) < 3:
return True
if not re.search(r"[A-Za-z]", s):
return True
lowered = s.lower()
tokens = re.findall(r"[a-z]+", lowered)
if any(tok in _PLACEHOLDER_TOKENS for tok in tokens):
return True
if len(tokens) >= 5:
return True
if re.fullmatch(
r"(river|lake|ocean|sea|city|capital|town|mountain|range|peak|poi|"
r"gate|terminal)\s+[a-z]", lowered
):
return True
return False
# ---------------------------------------------------------------------------
# Post-processing
# ---------------------------------------------------------------------------
_TRAILING_PUNCT = re.compile(r"[\s\.,;:!?\"'`\-]+$")
_LEADING_PUNCT = re.compile(r"^[\s\.,;:!?\"'`\-]+")
# Strip surrounding square brackets the model likes to emit
# (`[River Name]`, `[Lake Foo]`, `[Optional: Bar]`).
_BRACKET_WRAP = re.compile(r"^\[\s*(.*?)\s*\]$")
# Tokens that indicate the model failed the instruction and echoed the
# prompt back. If any of these appear as whole words (case-insensitive)
# in the cleaned output, reject the whole name and retry.
_PLACEHOLDER_TOKENS = {
"name", "names", "placeholder", "example", "example1", "exampleone",
"optional", "tbd", "todo",
}
# Labels Gemma likes to prepend to the answer. Matched case-insensitively
# at the start of the line, with optional whitespace and a `:` or `-`.
_LABEL_PREFIX = re.compile(
r"^(name|answer|output|result|response|city|town|capital|village|river|"
r"stream|ocean|sea|lake|bay|gulf|mountain|range|peak|ridge|spine|poi|"
r"landmark|terminal|gate|optional|alternative|alt|suggestion|"
r"example|note)\s*[:\-]\s*",
re.IGNORECASE,
)
# Strip markdown bold/italic wrappers (`**X**`, `*X*`, `__X__`, `_X_`)
# that Gemma sometimes emits even when told "no quotes or explanation".
_MD_BOLD = re.compile(r"^\*+\s*(.*?)\s*\*+$")
_MD_UNDER = re.compile(r"^_+\s*(.*?)\s*_+$")
def post_process(raw: str) -> str:
"""Extract a clean single-line name from Gemma's raw output.
Handles, in order:
- First non-empty line only (Gemma often continues with an explanation).
- Strip surrounding markdown bold/italic (`**X**`, `*X*`, `__X__`, `_X_`).
- Strip label prefixes the model likes to prepend (`Name:`, `City:`,
`River:`, `Ocean:`, `Mountain:`, etc.) — case-insensitive, with or
without a `:` or `-` separator.
- Strip surrounding quotes / punctuation.
- Collapse internal whitespace.
- Truncate at 40 chars as a hard safety limit.
The result is the canonical form used for both writing to disk AND
dedup comparison, so "River: Aureus" and a later raw "Aureus" normalize
to the same string and collide as intended.
"""
text = (raw or "").strip()
if not text:
return ""
# First non-empty line only.
for line in text.splitlines():
line = line.strip()
if line:
text = line
break
else:
return ""
# Strip surrounding markdown bold/italic, square brackets, label
# prefixes, and stray asterisks/underscores in alternation until the
# text stops shrinking. This handles every combination: `**Foo**`,
# `**City: Foo**`, `City: **Foo**`, `**City:** Foo`, `_Name: Bar_`,
# `[River Foo]`, `[City: Bar]`, etc.
while True:
before = text
m = _MD_BOLD.match(text) or _MD_UNDER.match(text) or _BRACKET_WRAP.match(text)
if m:
text = m.group(1).strip()
text = text.strip("*_ \t-[]")
new = _LABEL_PREFIX.sub("", text)
if new != text:
text = new.strip()
if text == before:
break
# Strip surrounding quotes / punctuation.
text = _LEADING_PUNCT.sub("", text)
text = _TRAILING_PUNCT.sub("", text)
# Collapse internal whitespace.
text = re.sub(r"\s+", " ", text).strip()
# Hard length safety.
if len(text) > 40:
text = text[:40].rstrip()
return text
# ---------------------------------------------------------------------------
# Fallback palette generator (deterministic, no LLM)
# ---------------------------------------------------------------------------
# Fallback stems — 40+ per corridor drawn from grounded, cosmopolitan
# place-name roots. Each corridor now spans multiple cultural registers
# so fallback names inherit the same diversity the few-shot pools teach.
# The old 10-stem Latin-institutional lists produced very same-y
# "Concord Ridge / Axis Spine / Senatus Heights" clusters when dense
# core bodies exhausted the LLM's unique output budget.
#
# Stems are deliberately a mix of real surnames, real place-name
# fragments, and administrative English — because settlers named
# places that way.
_FALLBACK_STEMS: dict[str, list[str]] = {
"north_reach": [
# British / Australian / Irish place-name stems
"Ashford", "Bellfield", "Briarcroft", "Clifford", "Drayton",
"Elmhurst", "Fairfield", "Garner", "Holmwood", "Irvington",
"Kelsworth", "Larkfield", "Mildern", "Netherbrook", "Oakdale",
"Pendleton", "Redfern", "Standen", "Tarndale", "Upwold",
"Waltham", "Wolcott", "Ashbourne", "Briarfell", "Harford",
"Kelston", "Pennhowe", "Rowanmoor", "Threlwood", "Carberry",
"Glendale", "Hollowmere", "Ingledene", "Kirkstone", "Maybourne",
"Newbridge", "Pinegrove", "Ridgemont", "Silverpine", "Tallgrass",
],
"south_reach": [
# Portuguese / Brazilian / Swahili / Cape Verdean stems
"Alves", "Brandão", "Cabral", "Dantas", "Évora", "Figueira",
"Gomes", "Hlanganani", "Inhambane", "Jabali", "Kilimi", "Lourenço",
"Manhica", "Natal", "Oliveira", "Pinheiro", "Quelimane", "Ribeiro",
"Serra", "Tavira", "Vargas", "Monteforte", "Serravale", "Cabo",
"Ilhabela", "Moçambo", "Ngola", "Porto", "Ribeira", "Costa",
"Marinha", "Várzea", "Campo", "Outeiro", "Pedregal", "Telhado",
],
"east_reach": [
# Korean / Japanese / Taiwanese stems
"Aomori", "Baektu", "Chōshi", "Daisen", "Etajima", "Fukagawa",
"Gogen", "Hanyang", "Izumi", "Junan", "Kita", "Mori", "Nagare",
"Okami", "Saeyeon", "Takamine", "Unari", "Yurigawa", "Ginoza",
"Naruhan", "Morimine", "Taegong", "Tsukuri", "Seorak", "Shiroyama",
"Kōzan", "Hirano", "Nakayama", "Midori", "Kawasaki", "Tsukushi",
"Higashi", "Nishimura", "Miyazaki", "Tachibana",
],
"west_reach": [
# German / Dutch / Nordic / Polish / Czech stems
"Altdorf", "Bergfjord", "Drachenberg", "Eikhof", "Feldberg",
"Hellekade", "Hoogland", "Kloosterdam", "Lindeborg", "Nordhölm",
"Oudewater", "Sørholm", "Straend", "Torsfell", "Vanebach",
"Viskrans", "Aalburg", "Blankhof", "Dalhem", "Grunendal",
"Halvorsen", "Järvenpää", "Krakenberg", "Lüneborg", "Meinhof",
"Nieuwpoort", "Osterdal", "Reinwoud", "Svarteberg", "Tjornholm",
"Urlinden", "Voorhout", "Weserhof", "Östby", "Brabanthof",
],
"core": [
# Administrative English / Gateway-era, with a side of classical
# for the occasional institutional feel
"Ashfield", "Bellview", "Cedarbrook", "Concord", "Crestline",
"Easthill", "Fairmont", "Foxgrove", "Gateway", "Glenbrook",
"Haversham", "Holloway", "Ironwood", "Kirkwood", "Lakeside",
"Libertyhill", "Linden", "Marchwood", "Meridian", "Midvale",
"Newbridge", "Northfield", "Oakdale", "Pinehurst", "Riverside",
"Rosemont", "Southgate", "Standen", "Stonewall", "Sunnyvale",
"Tallwood", "Union", "Uplands", "Valewood", "Westbrook",
"Willowgate", "Windmere", "Claremont", "Fernwood", "Hartley",
"Cardinal", "Prefecture", "Senate", "Assembly", "Federal",
],
# Aliases for legacy geographic_sector names
"inner_corridor": [
"Ashfield", "Bellview", "Cedarbrook", "Concord", "Crestline",
"Easthill", "Fairmont", "Foxgrove", "Gateway", "Glenbrook",
"Meridian", "Midvale", "Northfield", "Oakdale", "Pinehurst",
"Riverside", "Rosemont", "Southgate", "Uplands", "Westbrook",
],
"inner_orbit": [
"Ashfield", "Bellview", "Cedarbrook", "Concord", "Crestline",
"Easthill", "Fairmont", "Foxgrove", "Gateway", "Glenbrook",
"Meridian", "Midvale", "Northfield", "Oakdale", "Pinehurst",
"Riverside", "Rosemont", "Southgate", "Uplands", "Westbrook",
],
"sol-gateway-axis": [
"Ashfield", "Bellview", "Cedarbrook", "Concord", "Crestline",
"Easthill", "Fairmont", "Foxgrove", "Gateway", "Glenbrook",
"Meridian", "Midvale", "Northfield", "Oakdale", "Pinehurst",
"Riverside", "Rosemont", "Southgate", "Uplands", "Westbrook",
],
"deep_frontier": [
"Okafor", "Stenner", "Weller", "Pruitt", "Hale", "Kettle",
"Bowman", "Alder", "Risher", "Vickery", "Morgan", "Zhou",
"Novak", "Lindström", "Barbosa", "Farrow", "Tillman", "Kellogg",
"Grayson", "Whitby", "Pickering", "Cortland", "Deacon", "Huckle",
"Stonebrook", "Clayridge", "Ironvale", "Redrock", "Blackwater",
"Dustgate",
],
"frontier": [
"Okafor", "Stenner", "Weller", "Pruitt", "Hale", "Kettle",
"Bowman", "Alder", "Risher", "Vickery",
],
}
_FALLBACK_SUFFIXES: dict[str, list[str]] = {
"river": ["Run", "Water", "Beck", "Rill", "Course", "Brook",
"Stream", "Flow", "Creek"],
"ocean": ["Sea", "Expanse", "Deep", "Reach", "Main", "Vast"],
"sea": ["Sea", "Gulf", "Basin", "Bay", "Firth"],
"lake": ["Lake", "Mere", "Tarn", "Pool", "Loch", "Pond"],
"mountain_range": ["Range", "Ridge", "Spine", "Heights", "Scarp",
"Hills", "Peaks", "Back", "Crest", "Fell"],
"city_capital": ["Hold", "Prime", "Seat", "Court", "Landing",
"Cross", "Hub"],
"city_secondary": ["Cross", "Reach", "Hollow", "Fields", "Stand",
"Ferry", "Mill", "End", "Gate"],
"poi_transit": ["Gate Terminal", "Transit", "Concourse", "Exchange",
"Junction"],
"poi_institutional": ["Archive", "Hall", "Assembly", "Registry",
"Council", "Office"],
"poi_cultural": ["Commons", "Grounds", "Circle", "Square", "Park",
"Plaza"],
}
def fallback_name(corridor: str, feature_type: str, salt: int) -> str:
"""Deterministic palette-driven fallback when the LLM can't produce
a usable name after retries. Picks a stem + suffix using `salt` so
the same (body, feature) always gets the same fallback."""
stems = _FALLBACK_STEMS.get(corridor) or _FALLBACK_STEMS["inner_corridor"]
suffixes = _FALLBACK_SUFFIXES.get(feature_type, ["Place"])
stem = stems[salt % len(stems)]
suffix = suffixes[(salt // max(1, len(stems))) % len(suffixes)]
return f"{stem} {suffix}"
# ---------------------------------------------------------------------------
# Subprocess manager
# ---------------------------------------------------------------------------
class VoiceSubprocess:
"""Thin wrapper around sr-voice stdio mode.
The subprocess holds a KV cache across requests — after N_REFRESH
requests we tear it down and start a fresh one so earlier prompts
don't pollute later ones (context bleed). Real model load takes a
few seconds; mock mode is instantaneous.
"""
def __init__(
self,
sr_voice_bin: Path,
model_path: Path | None,
mock: bool,
refresh_every: int,
verbose: bool,
distrobox: str | None = None,
ctx_size: int = 1024,
):
self.sr_voice_bin = sr_voice_bin
self.model_path = model_path
self.mock = mock
self.refresh_every = max(1, refresh_every)
self.verbose = verbose
self.distrobox = distrobox
self.ctx_size = ctx_size
self.proc: subprocess.Popen | None = None
self.request_count = 0
self._start()
def _build_cmd(self) -> list[str]:
if self.mock:
return [str(MOCK_STDIO), "serve", "--stdio"]
# sr-voice-tooling (Gemma 4) takes --model directly;
# sr-voice (Gemma 2) needs `serve --stdio` subcommand.
bin_name = self.sr_voice_bin.name if hasattr(self.sr_voice_bin, 'name') else str(self.sr_voice_bin).rsplit("/", 1)[-1]
if "tooling" in bin_name:
inner_cmd = [str(self.sr_voice_bin),
"--ctx-size", str(self.ctx_size)]
else:
inner_cmd = [str(self.sr_voice_bin), "serve", "--stdio",
"--ctx-size", str(self.ctx_size)]
if self.model_path is not None:
inner_cmd += ["--model", str(self.model_path)]
# If a distrobox container was requested, invoke the binary
# inside the container. Needed when the built binary depends on
# libs (e.g. libhipblas.so.2) that only exist in the container,
# not on the host. `distrobox enter <name> -- <command>`
# forwards stdin/stdout through the container's podman exec
# pipe, which is exactly what VoiceSubprocess needs — the
# stdio JSONL protocol flows through unchanged.
if self.distrobox:
return ["distrobox", "enter", self.distrobox, "--"] + inner_cmd
return inner_cmd
def _start(self) -> None:
cmd = self._build_cmd()
if self.verbose:
print(f" [subprocess] starting: {' '.join(cmd)}", flush=True)
self.proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL if not self.verbose else None,
text=True,
bufsize=1, # line-buffered
start_new_session=True, # own process group so _stop can kill the whole chain
)
self.request_count = 0
# Mock prints a stderr banner synchronously; real sr-voice prints
# stderr while loading the model. Either way we drive it request-
# reply so no readiness probe is needed — the first request just
# blocks until the model is ready.
def request(self, prompt: str, seed: int) -> str:
"""Send one JSONL request, read one JSONL response, return raw text.
On subprocess death, restart once and retry. On restart
threshold, tear down and restart cleanly before the request.
"""
if self.request_count >= self.refresh_every:
if self.verbose:
print(
f" [subprocess] refresh after {self.request_count} requests",
flush=True,
)
self._stop()
self._start()
req = json.dumps({"prompt": prompt, "seed": seed})
assert self.proc is not None and self.proc.stdin is not None and self.proc.stdout is not None
try:
self.proc.stdin.write(req + "\n")
self.proc.stdin.flush()
line = self.proc.stdout.readline()
except (BrokenPipeError, OSError) as e:
print(f" [subprocess] pipe broken ({e}) — restarting", flush=True)
self._stop()
self._start()
self.proc.stdin.write(req + "\n") # type: ignore[union-attr]
self.proc.stdin.flush() # type: ignore[union-attr]
line = self.proc.stdout.readline() # type: ignore[union-attr]
self.request_count += 1
if not line:
raise RuntimeError("sr-voice returned empty response (subprocess died?)")
try:
resp = json.loads(line.strip())
except json.JSONDecodeError as e:
raise RuntimeError(f"sr-voice returned non-JSON: {line!r} ({e})") from e
if "error" in resp:
raise RuntimeError(f"sr-voice error: {resp['error']}")
return resp.get("text", "") or ""
def _stop(self) -> None:
if self.proc is None:
return
try:
if self.proc.stdin:
self.proc.stdin.close()
except Exception:
pass
# Kill the entire process group (distrobox → podman → sr-voice)
# rather than just the top-level shell. Without this, the inner
# sr-voice binary survives terminate() and holds the GPU.
pgid = None
try:
pgid = os.getpgid(self.proc.pid)
except (ProcessLookupError, OSError):
pass
try:
if pgid:
os.killpg(pgid, signal.SIGTERM)
else:
self.proc.terminate()
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
if pgid:
os.killpg(pgid, signal.SIGKILL)
else:
self.proc.kill()
except (ProcessLookupError, OSError):
pass
try:
self.proc.wait(timeout=3)
except Exception:
pass
except Exception:
pass
self.proc = None
def close(self) -> None:
self._stop()
def __enter__(self) -> "VoiceSubprocess":
return self
def __exit__(self, *exc) -> None:
self.close()
# ---------------------------------------------------------------------------
# Naming (per-feature request + retry)
# ---------------------------------------------------------------------------
def _seed_for(world_seed: int, body_id: str, local_id: str, attempt: int) -> int:
"""Deterministic per-feature seed. Same (world, body, feature) always
starts from the same seed; retries bump `attempt` to get a different
sample path without losing determinism."""
h = hashlib.sha256(
f"{world_seed}|{body_id}|{local_id}|{attempt}".encode()
).hexdigest()
return int(h[:16], 16)
def body_population_band(population: int) -> str:
if population <= 0:
return "uninhabited"
if population < 100_000:
return "outpost (<100k)"
if population < 10_000_000:
return "small (<10M)"
if population < 100_000_000:
return "medium (<100M)"
if population < 1_000_000_000:
return "large (<1B)"
return "megaworld (1B+)"
def name_feature(
voice: VoiceSubprocess,
feature_type: str,
ctx: dict,
blocklist: set[str],
corpus: dict[tuple, set[str]],
body_used: set[str],
system_hook: str | None,
system_id: str,
world_seed: int,
body_id: str,
local_id: str,
hop: int,
log: "Logger",
verbose: bool,
max_attempts: int = 5,
palette_override: dict[str, str] | None = None,
) -> str:
"""Request a name from Gemma, enforce blocklist + per-system dedup +
per-body cross-type dedup, skip on persistent failure.
Dedup scopes:
- `corpus[(system_id, feature_type)]` — no two features of the
same type in the same system share a name.
- `body_used` — per-body set across ALL feature types. Prevents
the same name from appearing as a river AND an ocean AND a
mountain range on the same world.
When `palette_override` is set, it replaces the hash-based palette
selection — used when select_register() picked a wiki-grounded
cultural register for this system.
"""
corridor = ctx.get("cultural_corridor") or "core"
if feature_type not in _PROMPT_CONFIG:
return fallback_name(corridor, feature_type, _seed_for(world_seed, body_id, local_id, 0))
palette = palette_override or palette_for(corridor, system_id)
planet_class = ctx.get("planet_class") or "habitable"
dedup_key = (system_id, feature_type)
used = corpus.setdefault(dedup_key, set())
# Capture mode: build the attempt-0 prompt, log it, return a unique
# placeholder. No LLM call, no validation gauntlet — we want the raw
# prompt set as Gemma would first see it, before retries kick in.
if _CAPTURE_FILE is not None:
prompt = _build_prompt(
feature_type,
inflection=palette["inflection"],
planet_class=planet_class,
body_id=body_id,
local_id=local_id,
attempt=0,
system_hook=system_hook,
system_name=ctx.get("system_proper_name"),
body_name=ctx.get("body_proper_name"),
)
seed = _seed_for(world_seed, body_id, local_id, 0)
_CAPTURE_FILE.write(json.dumps({
"body_id": body_id,
"local_id": local_id,
"feature_type": feature_type,
"corridor": corridor,
"seed": seed,
"prompt": prompt,
}) + "\n")
_CAPTURE_FILE.flush()
# Unique deterministic placeholder. Embeds the seed so every
# call returns a distinct string — passes dedup. Two letters +
# hex keep it short, no _PLACEHOLDER_TOKENS, no length issues.
placeholder = f"Zq{seed:08x}"
used.add(placeholder)
body_used.add(placeholder)
return placeholder
# Pre-build lowercase shadow sets for O(1) dedup checks
used_lower = {n.lower() for n in used}
body_used_lower = {n.lower() for n in body_used}
def _is_duplicate(candidate: str) -> bool:
lc = candidate.lower()
return lc in used_lower or lc in body_used_lower
def _commit_name(final: str) -> None:
used.add(final)
used_lower.add(final.lower())
body_used.add(final)
body_used_lower.add(final.lower())
rejections: dict[str, int] = {"empty": 0, "placeholder": 0, "blocklist": 0,
"dedup": 0, "error": 0}
for attempt in range(max_attempts):
seed = _seed_for(world_seed, body_id, local_id, attempt)
prompt = _build_prompt(
feature_type,
inflection=palette["inflection"],
planet_class=planet_class,
body_id=body_id,
local_id=local_id,
attempt=attempt,
system_hook=system_hook,
system_name=ctx.get("system_proper_name"),
body_name=ctx.get("body_proper_name"),
)
try:
raw = voice.request(prompt, seed)
except RuntimeError as e:
rejections["error"] += 1
log(f" error {body_id}/{local_id} attempt {attempt}: {e}")
continue
cleaned = post_process(raw)
if not cleaned:
rejections["empty"] += 1
continue
if is_placeholder(cleaned):
rejections["placeholder"] += 1
continue
if is_blocked(cleaned, blocklist):
rejections["blocklist"] += 1
continue
if _is_duplicate(cleaned):
rejections["dedup"] += 1
continue
_commit_name(cleaned)
return cleaned
# All attempts exhausted. Leave the name as None so the preserved
# path skips it on a fill round — a second pass with a fresh corpus
# will pick it up and try again with different dedup pressure.
reasons = " ".join(f"{k}={v}" for k, v in rejections.items() if v > 0)
log(f" skipped: {body_id}/{local_id} [{reasons}]")
return None
# ---------------------------------------------------------------------------
# Body walker
# ---------------------------------------------------------------------------
def load_body_context(body_id: str, system_id: str, conn: sqlite3.Connection) -> dict:
row = conn.execute(
"""
SELECT b.planet_class, b.settlement_pattern,
COALESCE(b.cultural_corridor, s.cultural_corridor, s.geographic_sector),
b.population, b.economic_role,
b.proper_name, s.proper_name
FROM bodies b
JOIN star_systems s ON b.system_id = s.system_id
WHERE b.body_id = ?
""",
(body_id,),
).fetchone()
if row is None:
return {
"planet_class": None, "settlement_pattern": None,
"cultural_corridor": None, "population": 0, "economic_role": None,
"pop_band": "unknown",
"body_proper_name": None, "system_proper_name": None,
}
(planet_class, settlement_pattern, corridor, population, economic_role,
body_proper_name, system_proper_name) = row
return {
"planet_class": planet_class,
"settlement_pattern": settlement_pattern,
"cultural_corridor": corridor,
"population": population or 0,
"economic_role": economic_role,
"pop_band": body_population_band(population or 0),
"body_proper_name": body_proper_name,
"system_proper_name": system_proper_name,
}
def load_system_gttr_hooks(conn: sqlite3.Connection) -> dict[str, str]:
"""Return `{system_id: gttr_hook}` for every system with a populated
hook. The gttr_hook column is filled by
`tooling/db/populate_gttr_hook.py` from wiki/star-systems/<slug>/gttr.md
and captures the system's canonical characterisation in 30-45 words
("where the rules live", "forty years old and still in the draft",
…). The naming pipeline injects this into the prompt preamble so
Gemma has per-system cultural context without having to parse
thousands of lines of wiki markdown at runtime.
Systems whose gttr_hook is NULL return nothing from this map — the
caller falls back to the corridor palette alone.
"""
rows = conn.execute(
"SELECT system_id, gttr_hook FROM star_systems "
"WHERE gttr_hook IS NOT NULL AND gttr_hook != ''"
).fetchall()
return {row[0]: row[1] for row in rows}
def load_body_hop_order(conn: sqlite3.Connection) -> dict[str, tuple[int, str, int, int, str]]:
"""Return a `{body_id: (hop, system_id, uninhabited, neg_pop, body_id)}`
sort-key map. The pipeline walks the reach from core outward, keeps
all bodies in the same system together, and within each system
processes inhabited bodies first (sorted by population descending).
This gives the most important worlds — the ones players will
actually visit — first pick of the cultural register's vocabulary.
Barren moons get whatever's left or the adjacent-register refill,
which is fine for star-map dressing.
Sort key components:
- hop: system distance from Gateway (0 = core, higher = frontier)
- system_id: groups all bodies in same system together
- uninhabited: 0 for inhabited, 1 for uninhabited (inhabited first)
- neg_pop: negative population (higher pop sorts first)
- body_id: tiebreaker for determinism
"""
rows = conn.execute(
"""
SELECT b.body_id,
b.system_id,
COALESCE(sg.hop_distance_from_gateway, 99) AS hop,
b.inhabited,
COALESCE(b.population, 0) AS pop
FROM bodies b
LEFT JOIN system_gates sg ON b.system_id = sg.system_id
"""
).fetchall()
return {
body_id: (hop, system_id, 0 if inhabited else 1, -(pop or 0), body_id)
for body_id, system_id, hop, inhabited, pop in rows
}
def _is_blank(value) -> bool:
return value is None or (isinstance(value, str) and value.strip() == "")
def _feature_type_for_city(city: dict) -> str:
return "city_capital" if city.get("kind") == "capital" else "city_secondary"
def _feature_type_for_poi(poi: dict) -> str:
kind = (poi.get("kind") or "").lower()
if kind in ("transit", "gate_terminal"):
return "poi_transit"
if kind in ("institutional", "corporate", "government"):
return "poi_institutional"
if kind in ("cultural", "commercial", "heritage"):
return "poi_cultural"
return "poi_institutional"
def _feature_type_for_ocean(water: dict) -> str:
kind = (water.get("kind") or "ocean").lower()
if kind == "lake":
return "lake"
if kind == "sea":
return "sea"
return "ocean"
def process_body(
body_id: str,
system_id: str,
markers_path: Path,
voice: VoiceSubprocess,
conn: sqlite3.Connection,
blocklist: set[str],
corpus: dict[tuple, set[str]],
system_hook: str | None,
world_seed: int,
hop: int,
log: "Logger",
verbose: bool,
palette_override: dict[str, str] | None = None,
) -> dict:
"""Fill every empty name field in this body's markers.json. Returns
a summary counts dict plus a `generated` dict mapping section name
to the list of new names produced (for the main loop to log)."""
try:
markers = json.loads(markers_path.read_text())
except json.JSONDecodeError as e:
return {"error": f"invalid JSON: {e}"}
grid = markers.get("grid") or {}
if grid.get("w") != GRID_W or grid.get("h") != GRID_H:
return {
"error": (
f"grid mismatch {grid} != {{'w': {GRID_W}, 'h': {GRID_H}}}"
)
}
ctx = load_body_context(body_id, system_id, conn)
corridor = ctx.get("cultural_corridor") or "core"
counts = {
"cities": 0, "rivers": 0, "oceans": 0, "mountain_ranges": 0, "pois": 0,
"preserved": 0,
}
generated: dict[str, list[str]] = {
"cities": [], "rivers": [], "oceans": [], "mountain_ranges": [], "pois": [],
}
changed = False
# Per-body dedup set — no name may appear twice on the same body,
# even across feature types. Seeded with every hand-authored name
# already present so templates (Lendel, Edict, Estrade, …) keep their
body_used: set[str] = set()
for key in ("cities", "rivers", "oceans", "mountain_ranges", "pois"):
for feat in markers.get(key) or []:
name = feat.get("name")
if name and isinstance(name, str) and name.strip():
body_used.add(name.strip())
# Batch naming: group blank features by type, request N names at
# once, rank by Levenshtein distinctiveness, assign.
inflection = (palette_override or palette_for(corridor, system_id))["inflection"]
substyles = CORRIDOR_SUBSTYLES.get(corridor, DEFAULT_SUBSTYLES)
mood = mood_for_body(body_id, world_seed)
# Build cultural-history context for the prompt (#886 §6):
# Collect the inflection descriptions of all *secondary* registers in
# this corridor so the model sees the full settlement layering — e.g.
# "Scottish Highland" as primary, but also the Irish and Australian
# substyles that represent earlier or interleaved waves of settlers.
# Limited to 3 secondary styles to keep the prompt concise.
_secondary_inflections = [
s["inflection"] for s in substyles if s["inflection"] != inflection
][:3]
cultural_history: str | None = (
"; ".join(_secondary_inflections) if _secondary_inflections else None
)
# Helper: batch-name blank features in a marker section
def _batch_fill(
section_key: str,
feature_type_fn, # callable(feat) -> str
count_key: str,
):
nonlocal changed
features = markers.get(section_key) or []
# Separate preserved vs blank
blank = []
for feat in features:
if not _is_blank(feat.get("name")):
ft = feature_type_fn(feat)
# Corpus key: (corridor, feature_type) for cross-system dedup
# within the same cultural corridor (#853 §1, §2).
corpus.setdefault((corridor, ft), set()).add(feat["name"])
counts["preserved"] += 1
else:
blank.append(feat)
if not blank:
return
# Group blanks by feature_type (cities have capital/secondary,
# pois have transit/institutional/cultural, oceans have ocean/sea/lake)
by_type: dict[str, list[dict]] = {}
for feat in blank:
ft = feature_type_fn(feat)
by_type.setdefault(ft, []).append(feat)
for ft, feats in by_type.items():
need = len(feats)
# Build taken list from corpus (cross-body corridor-scoped dedup).
# Two bodies in the same corridor never get the same city/mountain name.
taken = list(corpus.get((corridor, ft), set()))
# Also include body_used to avoid cross-type collisions on same body
taken_full = taken + list(body_used)
names = name_features_batch(
voice=voice,
feature_type=ft,
count=need,
inflection=inflection,
corridor=corridor,
corridor_substyles=substyles,
taken=taken_full,
prompt_config=_PROMPT_CONFIG,
system_name=ctx.get("system_proper_name"),
body_name=ctx.get("body_proper_name"),
system_hook=system_hook,
mood=mood,
body_id=body_id,
world_seed=world_seed,
cultural_history=cultural_history,
ctx_size=voice.ctx_size,
)
# Assign names to features in order
body_proper = ctx.get("body_proper_name") or body_id
for i, feat in enumerate(feats):
if i < len(names):
feat["name"] = names[i]
counts[count_key] += 1
generated[count_key].append(names[i])
corpus.setdefault((corridor, ft), set()).add(names[i])
body_used.add(names[i])
changed = True
elif section_key == "mountain_ranges":
# Empty-name fallback for mountains (#853 §2):
# If Gemma returned fewer names than needed, use a
# deterministic fallback rather than leave the field blank.
fallback = f"{body_proper} Range {i + 1}"
feat["name"] = fallback
counts[count_key] += 1
generated[count_key].append(fallback)
corpus.setdefault((corridor, ft), set()).add(fallback)
body_used.add(fallback)
changed = True
_batch_fill("cities", _feature_type_for_city, "cities")
_batch_fill("rivers", lambda f: "river", "rivers")
_batch_fill("oceans", _feature_type_for_ocean, "oceans")
_batch_fill("mountain_ranges", lambda f: "mountain_range", "mountain_ranges")
_batch_fill("pois", _feature_type_for_poi, "pois")
# Mountain suffix monotony check + auto-fix (#853 §3, #886 §3):
# If >40% of mountain names on a single body share a trailing word,
# re-query with the offending names added to `taken` so the model is
# forced to diversify. One retry per body; if the retry still clusters
# (rare), record a warning for post-run inspection.
mountain_names = [
f.get("name", "") for f in (markers.get("mountain_ranges") or [])
if f.get("name")
]
if len(mountain_names) >= 3:
suffix_counts: dict[str, int] = {}
for mn in mountain_names:
words = mn.split()
if words:
suffix_counts[words[-1].lower()] = suffix_counts.get(words[-1].lower(), 0) + 1
dominant = max(suffix_counts, key=lambda k: suffix_counts[k])
dominant_frac = suffix_counts[dominant] / len(mountain_names)
if dominant_frac > 0.40:
# Targeted retry: identify features with the dominant suffix,
# re-request names for them with the monotonous names as `taken`.
offending_features = [
f for f in (markers.get("mountain_ranges") or [])
if f.get("name") and f["name"].split()[-1].lower() == dominant
]
retry_taken = (
list(body_used)
+ list(corpus.get((corridor, "mountain_range"), set()))
)
retry_names = name_features_batch(
voice=voice,
feature_type="mountain_range",
count=len(offending_features),
inflection=inflection,
corridor=corridor,
corridor_substyles=substyles,
taken=retry_taken,
prompt_config=_PROMPT_CONFIG,
system_name=ctx.get("system_proper_name"),
body_name=ctx.get("body_proper_name"),
system_hook=system_hook,
mood=mood,
body_id=body_id,
world_seed=world_seed + 1, # bump seed to force different output
cultural_history=cultural_history,
ctx_size=voice.ctx_size,
)
for i, feat in enumerate(offending_features):
if i < len(retry_names):
old_name = feat["name"]
feat["name"] = retry_names[i]
body_used.discard(old_name)
body_used.add(retry_names[i])
corpus.setdefault((corridor, "mountain_range"), set()).discard(old_name)
corpus.setdefault((corridor, "mountain_range"), set()).add(retry_names[i])
changed = True
# Re-check after retry; record warning if still clustered
mountain_names_after = [
f.get("name", "") for f in (markers.get("mountain_ranges") or [])
if f.get("name")
]
suffix_counts_after: dict[str, int] = {}
for mn in mountain_names_after:
words = mn.split()
if words:
suffix_counts_after[words[-1].lower()] = (
suffix_counts_after.get(words[-1].lower(), 0) + 1
)
if suffix_counts_after:
dominant_after = max(suffix_counts_after, key=lambda k: suffix_counts_after[k])
dominant_frac_after = suffix_counts_after[dominant_after] / len(mountain_names_after)
if dominant_frac_after > 0.40:
counts["suffix_monotony_warning"] = (
f"mountain suffix '{dominant_after}' still on "
f"{suffix_counts_after[dominant_after]}/{len(mountain_names_after)} "
f"features ({dominant_frac_after:.0%}) after retry"
)
# Infrastructure naming (#853 §7):
# Assign deterministic city-pair names to unnamed roads and railroads.
# Convention: "{CityA}–{CityB} {corridor_suffix}"
# (e.g. "Aldren–Forgehaven Corridor" for core,
# "Matamba–Dakar Estrada" for south_reach)
# This is a deterministic post-pass — no LLM needed.
_ROAD_SUFFIX: dict[str, str] = {
"core": "Corridor", "sol-gateway-axis": "Corridor",
"inner_corridor": "Corridor", "inner_orbit": "Corridor",
"north_reach": "Road", "south_reach": "Estrada",
"east_reach": "Road", "west_reach": "Strasse",
"deep_frontier": "Track", "frontier": "Track",
}
_RAIL_SUFFIX: dict[str, str] = {
"core": "Express", "sol-gateway-axis": "Express",
"inner_corridor": "Express", "inner_orbit": "Express",
"north_reach": "Line", "south_reach": "Linha",
"east_reach": "Line", "west_reach": "Bahn",
"deep_frontier": "Run", "frontier": "Run",
}
road_sfx = _ROAD_SUFFIX.get(corridor, "Road")
rail_sfx = _RAIL_SUFFIX.get(corridor, "Line")
cities_list = markers.get("cities") or []
def _nearest_city_name(path: list, cities: list[dict]) -> str:
"""Return the proper_name of the city nearest to a path endpoint."""
if not cities or not path:
return ""
endpoint = path[0] # first path point
if not isinstance(endpoint, (list, tuple)) or len(endpoint) < 2:
return ""
er, ec = endpoint[0], endpoint[1]
best_name = ""
best_dist = float("inf")
for city in cities:
center = city.get("center")
if not center or len(center) < 2:
continue
cr, cc = center[0], center[1]
dist = abs(er - cr) + abs(ec - cc)
if dist < best_dist and city.get("name"):
best_dist = dist
best_name = city["name"]
return best_name
for section_key, suffix in (("roads", road_sfx), ("railroads", rail_sfx)):
infra_list = markers.get(section_key) or []
for infra in infra_list:
if infra.get("name"):
continue # already named
path = infra.get("path") or []
if len(path) < 2:
continue
city_a = _nearest_city_name(path[:1], cities_list)
city_b = _nearest_city_name(path[-1:], cities_list)
if city_a and city_b and city_a != city_b:
infra["name"] = f"{city_a}–{city_b} {suffix}"
elif city_a:
infra["name"] = f"{city_a} {suffix}"
else:
continue
changed = True
if changed:
markers_path.write_text(json.dumps(markers, indent=2) + "\n")
# Refresh atlas_* rows for this body so DB queries pick up the
# new names without a separate generate_atlas.py pass. Commit
# immediately so a mid-run crash / kill loses at most one body
# of DB state — the markers.json files are already persisted
# above, atomically, via Path.write_text.
# Skip DB sync for orphan bodies (markers.json exists but no
# row in the bodies table — FK constraint would fail).
try:
sync_markers_to_db(conn, body_id, markers)
conn.commit()
except Exception as e:
conn.rollback()
log(f" DB sync skipped for {body_id}: {e}")
counts["generated"] = generated
return counts
# ---------------------------------------------------------------------------
# Body discovery
# ---------------------------------------------------------------------------
def _body_id_from_path(markers_path: Path) -> tuple[str, str]:
"""From wiki/star-systems/GJ-244A/bodies/GJ244Ad/markers.json
return ('GJ244Ad', 'GJ 244A')."""
body_id = markers_path.parent.name
system_slug = markers_path.parent.parent.parent.name
system_id = system_slug.replace("-", " ", 1) if system_slug.startswith("GJ-") else system_slug
return body_id, system_id
def discover_bodies(
body_filter: str | None,
limit: int | None,
hop_order: dict[str, tuple[int, str]],
) -> list[Path]:
"""Discover every markers.json under wiki/star-systems/ and return the
list sorted by hop distance from Gateway ascending (core first, deep
frontier last). Files whose body_id is not in the hop_order map —
e.g. bodies deleted from the DB but still carrying a markers.json —
sort to the end with a sentinel hop of 99 so they don't pollute
early dedup decisions.
"""
all_markers = list(WIKI_SYSTEMS.glob("*/bodies/*/markers.json"))
if body_filter:
all_markers = [
p for p in all_markers if _body_id_from_path(p)[0] == body_filter
]
def sort_key(path: Path) -> tuple[int, str, int, int, str]:
body_id, system_id = _body_id_from_path(path)
return hop_order.get(body_id, (99, system_id, 1, 0, body_id))
all_markers.sort(key=sort_key)
if limit is not None:
all_markers = all_markers[:limit]
return all_markers
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Batch-name atlas features via Gemma 4 E2B tooling pipeline (#833)"
)
parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db")
parser.add_argument("--body", help="Process only this body_id")
parser.add_argument(
"--limit",
type=int,
help="Process at most N bodies (in sorted order). Smoke testing.",
)
parser.add_argument(
"--mock",
action="store_true",
help="Use mock-stdio.sh instead of the real sr-voice binary",
)
parser.add_argument(
"--sr-voice",
default=str(DEFAULT_SR_VOICE),
help="Path to the sr-voice release binary",
)
parser.add_argument(
"--model",
default=str(DEFAULT_MODEL),
help="Path to the GGUF model file (ignored in --mock mode)",
)
parser.add_argument(
"--distrobox",
default=None,
help="Name of a distrobox container to invoke sr-voice inside. "
"Use this when the built binary depends on libraries (e.g. "
"libhipblas.so.2) that are only installed in the container, "
"not on the host. Example: --distrobox reach-build. "
"Default: run sr-voice directly on the host.",
)
parser.add_argument(
"--refresh",
type=int,
default=1000,
help="Restart the voice subprocess every N requests (default: 1000) "
"to prevent KV-cache context bleed",
)
parser.add_argument(
"--seed",
type=int,
default=42,
help="World seed for deterministic naming (default: 42)",
)
parser.add_argument(
"--log",
default=str(REPO_ROOT / ".tmp" / "gemma_naming.log"),
help="Path to a log file. Every status line is written to both "
"stdout and the log. Default: .tmp/gemma_naming.log. "
"Pass '-' to disable file logging.",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Print per-feature retry detail (noisy) and subprocess "
"lifecycle events",
)
parser.add_argument(
"--dump-prompts",
default=None,
help="Capture mode: write the attempt-0 prompt for every feature "
"to this JSONL path and return unique placeholders, without "
"calling any LLM. Use to feed the same prompt set to a "
"different backend (e.g. a Claude agent) for A/B comparison.",
)
args = parser.parse_args()
global _CAPTURE_FILE
if args.dump_prompts:
Path(args.dump_prompts).parent.mkdir(parents=True, exist_ok=True)
_CAPTURE_FILE = open(args.dump_prompts, "w")
# Capture mode short-circuits inside name_feature and never
# actually calls the voice subprocess. Force --mock so we boot
# the cheap mock-stdio shell instead of loading a real model.
args.mock = True
log_path = None if args.log == "-" else Path(args.log)
db_path = Path(args.db)
if not db_path.exists():
print(f"error: {db_path} not found", file=sys.stderr)
sys.exit(1)
sr_voice_bin = Path(args.sr_voice)
model_path = Path(args.model) if args.model else None
if not args.mock:
if not sr_voice_bin.exists():
print(
f"error: sr-voice binary not found at {sr_voice_bin}\n"
"Either build it (`make build-sr-voice` in the main workdir) "
"or run with --mock for a dry-fire test.",
file=sys.stderr,
)
sys.exit(1)
if model_path and not model_path.exists():
print(
f"error: model not found at {model_path}\n"
"Either download the model GGUF or run with --mock.",
file=sys.stderr,
)
sys.exit(1)
else:
if not MOCK_STDIO.exists():
print(f"error: mock stdio script not found at {MOCK_STDIO}", file=sys.stderr)
sys.exit(1)
log = Logger(log_path)
blocklist = load_blocklist()
conn = sqlite3.connect(str(db_path), timeout=30.0)
# WAL mode + busy_timeout so two concurrent shards serialize writes
# without locking errors. WAL is a pragma-level switch, safe to
# re-apply on every connect.
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=15000")
conn.execute("PRAGMA foreign_keys=ON")
ensure_atlas_schema(conn)
hop_order = load_body_hop_order(conn)
system_gttr_hooks = load_system_gttr_hooks(conn)
# Preload corridor for each system so select_register can scope to
# the right sub-style list without re-querying per body.
system_corridors: dict[str, str] = {
row[0]: row[1] or "core"
for row in conn.execute(
"SELECT system_id, COALESCE(cultural_corridor, geographic_sector, 'core') "
"FROM star_systems"
).fetchall()
}
markers_paths = discover_bodies(args.body, args.limit, hop_order)
if not markers_paths:
log(f"error: no markers.json found (body={args.body})")
conn.close()
sys.exit(1)
# Count distinct systems so the progress lines can report
# `systems X/Y done` alongside `bodies X/Y done`.
total_systems = len({_body_id_from_path(p)[1] for p in markers_paths})
seen_systems: set[str] = set()
first_hop = hop_order.get(_body_id_from_path(markers_paths[0])[0], (99, ""))[0]
last_hop = hop_order.get(_body_id_from_path(markers_paths[-1])[0], (99, ""))[0]
log.raw("")
log.raw(f" Gemma 4 Batch Naming Pipeline (#833)")
log.raw(f" DB: {db_path}")
log.raw(f" Mode: {'MOCK' if args.mock else 'REAL'}")
log.raw(f" sr-voice: {MOCK_STDIO if args.mock else sr_voice_bin}")
if not args.mock:
log.raw(f" model: {model_path}")
log.raw(f" seed: {args.seed} refresh: every {args.refresh} requests")
log.raw(f" {len(markers_paths)} markers.json files to process")
log.raw(f" hop {first_hop} → hop {last_hop}, core-first ordering")
log.raw(f" {total_systems} distinct systems")
log.raw(f" blocklist: {len(blocklist)} Earth-major entries")
if log.log_path is not None:
log.raw(f" log: {log.log_path}")
log.raw("")
log.raw(
" Resume: re-run this command any time. Bodies whose markers.json "
"already has non-empty name fields will be skipped (preserved path)."
)
log.raw("")
corpus: dict[tuple, set[str]] = {}
# Seed corpus from existing atlas_* names so re-runs don't collide with
# names that were already committed to the DB on a previous pass (#853 §1).
# Key: (cultural_corridor, feature_type) — corridor-scoped dedup.
_ATLAS_SEED_QUERIES: list[tuple[str, str]] = [
("atlas_cities", "city"),
("atlas_rivers", "river"),
("atlas_mountain_ranges", "mountain_range"),
("atlas_oceans", "ocean"),
("atlas_pois", "poi_transit"),
]
try:
for table, ft in _ATLAS_SEED_QUERIES:
rows = conn.execute(
f"SELECT c.name, b.cultural_corridor "
f"FROM {table} c "
f"JOIN bodies b ON c.body_id = b.body_id "
f"WHERE c.name IS NOT NULL AND c.name != ''"
).fetchall()
for name, corridor_val in rows:
key = (corridor_val or "core", ft)
corpus.setdefault(key, set()).add(name)
except Exception as e:
log(f" warning: corpus seeding from DB failed ({e}) — cross-run dedup disabled")
# Cache of system_id → proper_name so we can emit a header line the
# first time we hit each system without re-querying per body.
system_name_cache: dict[str, str] = {
row[0]: row[1] or ""
for row in conn.execute(
"SELECT system_id, proper_name FROM star_systems"
).fetchall()
}
body_name_cache: dict[str, str] = {
row[0]: row[1] or ""
for row in conn.execute(
"SELECT body_id, proper_name FROM bodies"
).fetchall()
}
last_system_id: str | None = None
current_palette: dict[str, str] | None = None
totals = {
"cities": 0, "rivers": 0, "oceans": 0, "mountain_ranges": 0, "pois": 0,
"preserved": 0, "errors": 0,
}
bodies_touched = 0
t_total = time.time()
try:
with VoiceSubprocess(
sr_voice_bin=sr_voice_bin,
model_path=None if args.mock else model_path,
mock=args.mock,
refresh_every=args.refresh,
verbose=args.verbose,
distrobox=args.distrobox,
) as voice:
for i, markers_path in enumerate(markers_paths):
body_id, system_id = _body_id_from_path(markers_path)
# System header: print when we enter a new system, so the
# user can see which part of the reach we're in. Includes
# the proper_name if the system has one (e.g. "Tau Ceti",
# "p Eridani", "Groombridge").
if system_id != last_system_id:
last_system_id = system_id
sys_proper = system_name_cache.get(system_id, "")
sys_hop = hop_order.get(body_id, (99, "", 1, 0, ""))[0]
label = f"{system_id}"
if sys_proper:
label = f"{system_id} — {sys_proper}"
log(f" ── SYSTEM {len(seen_systems)+1}/{total_systems} "
f"{label} (hop {sys_hop})")
# Ask Gemma to pick the cultural register based on
# wiki content instead of the hash-based randomizer.
corridor = system_corridors.get(system_id, "core")
wiki_index, wiki_gttr = load_wiki_context(system_id)
current_palette = select_register(
voice, corridor, system_id,
wiki_index, wiki_gttr, log,
)
if current_palette:
log(f" register: {current_palette['inflection']}")
else:
current_palette = palette_for(corridor, system_id)
log(f" register: {current_palette['inflection']} (hash fallback)")
seen_systems.add(system_id)
t0 = time.time()
hop = hop_order.get(body_id, (99, "", 1, 0, ""))[0]
counts = process_body(
body_id=body_id,
system_id=system_id,
markers_path=markers_path,
voice=voice,
conn=conn,
blocklist=blocklist,
corpus=corpus,
system_hook=system_gttr_hooks.get(system_id),
world_seed=args.seed,
hop=hop,
log=log,
verbose=args.verbose,
palette_override=current_palette,
)
elapsed = time.time() - t0
body_progress = f"body {i+1}/{len(markers_paths)}"
sys_progress = f"sys {len(seen_systems)}/{total_systems}"
hop = hop_order.get(body_id, (99, "", 1, 0, ""))[0]
progress = f"{body_progress} {sys_progress} hop={hop}"
# Append the body's proper name if it has one ("Threshold",
# "Arden", "Earth") so the log reads like a tour through
# the reach rather than a wall of body_id slugs.
body_proper = body_name_cache.get(body_id, "")
body_label = body_id if not body_proper else f"{body_id:14s} ({body_proper})"
body_label = body_label if body_proper else f"{body_id:14s}"
if "error" in counts:
totals["errors"] += 1
log(f" [{progress}] {body_label} ERROR: {counts['error']}")
continue
generated: dict[str, list[str]] = counts.pop("generated", {}) or {
"cities": [], "rivers": [], "oceans": [],
"mountain_ranges": [], "pois": [],
}
touched = (
counts["cities"] + counts["rivers"] + counts["oceans"]
+ counts["mountain_ranges"] + counts["pois"]
)
if touched:
bodies_touched += 1
for k in ("cities", "rivers", "oceans", "mountain_ranges", "pois", "preserved"):
totals[k] += counts[k]
log(
f" [{progress}] {body_label} +{touched} names "
f"({elapsed:.1f}s) — "
f"cities={counts['cities']} rivers={counts['rivers']} "
f"oceans={counts['oceans']} mtns={counts['mountain_ranges']} "
f"pois={counts['pois']}"
)
# Print the new names so the user can eyeball quality
# as the run progresses.
for section_label, key in (
("cities", "cities"),
("rivers", "rivers"),
("waters", "oceans"),
("mtns", "mountain_ranges"),
("pois", "pois"),
):
names = generated.get(key) or []
if names:
preview = ", ".join(names[:10])
if len(names) > 10:
preview += f", … (+{len(names) - 10} more)"
log(f" {section_label:7s} {preview}")
else:
totals["preserved"] += counts["preserved"]
log(
f" [{progress}] {body_label} "
f"(skip — {counts['preserved']} names already set)"
)
# Periodic cumulative snapshot so the log has regular
# checkpoint lines the user can scroll to.
if (i + 1) % 25 == 0 or (i + 1) == len(markers_paths):
cum = (
totals["cities"] + totals["rivers"] + totals["oceans"]
+ totals["mountain_ranges"] + totals["pois"]
)
rate = cum / max(time.time() - t_total, 1e-6)
remaining = len(markers_paths) - (i + 1)
if remaining > 0 and (i + 1) > 0:
per_body = (time.time() - t_total) / (i + 1)
eta_s = int(per_body * remaining)
eta = f"{eta_s // 3600}h{(eta_s % 3600) // 60:02d}m"
else:
eta = "--"
log(
f" >> CHECKPOINT bodies {i+1}/{len(markers_paths)} "
f"systems {len(seen_systems)}/{total_systems} "
f"names {cum} {rate:.1f}/s eta {eta}"
)
# Final explicit commit for anything we accumulated since
# the last per-body commit (should be no-op since we commit
# per body, but defensive).
conn.commit()
finally:
conn.close()
elapsed_total = time.time() - t_total
log.raw("")
log.raw(f" Done: {elapsed_total:.0f}s ({elapsed_total/60:.1f} min)")
log.raw(f" bodies processed: {len(markers_paths)}")
log.raw(f" bodies touched: {bodies_touched}")
log.raw(f" systems seen: {len(seen_systems)}/{total_systems}")
log.raw(f" cities named: {totals['cities']}")
log.raw(f" rivers named: {totals['rivers']}")
log.raw(f" oceans named: {totals['oceans']}")
log.raw(f" mountains named: {totals['mountain_ranges']}")
log.raw(f" pois named: {totals['pois']}")
log.raw(f" preserved: {totals['preserved']}")
log.raw(f" errors: {totals['errors']}")
log.raw("")
log.close()
if totals["errors"] > 0:
sys.exit(1)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink