diff --git a/server/data/systems-schema.sql b/server/data/systems-schema.sql index 796b7933c..fa58aac7c 100644 --- a/server/data/systems-schema.sql +++ b/server/data/systems-schema.sql @@ -38,6 +38,15 @@ CREATE TABLE IF NOT EXISTS star_systems ( cultural_corridor TEXT, generation_priority TEXT, + -- Short narrative hook extracted from the wiki's gttr.md file — the + -- first characterisation paragraph ("where the rules live", "forty + -- years old and still in the draft", etc). Populated by + -- tooling/db/populate_gttr_hook.py from wiki/star-systems//gttr.md. + -- Used as compact cultural context in the Gemma 2 naming pipeline + -- (#833) so per-system feel is grounded in the canonical identity + -- rather than generic corridor labels. + gttr_hook TEXT, + -- Economics (D-172) currency_zone TEXT DEFAULT 'TRACTUS_PRIMARY', -- TRACTUS_PRIMARY | MARK_PRIMARY | MIXED diff --git a/server/data/systems.db b/server/data/systems.db index fd767e7bd..261aa54a5 100644 Binary files a/server/data/systems.db and b/server/data/systems.db differ diff --git a/tooling/db/populate_gttr_hook.py b/tooling/db/populate_gttr_hook.py new file mode 100755 index 000000000..c5992de68 --- /dev/null +++ b/tooling/db/populate_gttr_hook.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +""" +Extract a short narrative hook from every wiki/star-systems//gttr.md +and store it on `star_systems.gttr_hook`. + +The GTTR ("Drifter's Guide to the Reach") files follow a consistent +format: an H1 title, then a paragraph that opens with `**NAME**` followed +by a 1-3 sentence characterisation. That characterisation is the hook — +per D-191 §4 and the #833 naming pipeline it is the single strongest +cultural signal available for each system, capturing things like: + + - Gateway: "the most connected system in the Reach, the site of its + oldest research institution, and the location of a gate that goes + to Earth and that nobody uses" + - Ran: "a very nice place to live, provided you are the sort of + person whose grandparents owned the land" + - ACB: "where the Lattice Commission lives, which means it is where + the rules live" + - Posto Avançado: "forward post — the place beyond the established + line" + +This script parses each gttr.md, regex-extracts the first `**NAME**` +paragraph, normalises whitespace, and truncates at a soft word cap so +the hook stays cheap to inject into prompts. Empty or unmatched files +leave the column NULL. + +Idempotent, safe to re-run after any wiki update. Explicit transaction +wrapper with rollback on exception. + +Usage: + tooling/db/populate_gttr_hook.py + tooling/db/populate_gttr_hook.py --max-words 45 + tooling/db/populate_gttr_hook.py --dry-run + tooling/db/populate_gttr_hook.py --system "GJ 71" +""" + +import argparse +import re +import sqlite3 +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = (SCRIPT_DIR / ".." / "..").resolve() +DB_PATH = REPO_ROOT / "server" / "data" / "systems.db" +WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems" + +# Matches the first paragraph that opens with `**NAME**` at the start +# of a line. Captures everything up to a blank line or the next H1/H2. +# Non-greedy on the content. +# Match a line that opens with `**NAME**` (any non-asterisk chars, since +# names contain accents from many alphabets — Á, Ž, Ç, Ñ, etc) followed +# by the paragraph body up to a blank line or the next markdown heading. +_HOOK_RE = re.compile( + r"^\*\*([^\n*]+?)\*\*(.+?)(?:\n\n|\n#)", + re.DOTALL | re.MULTILINE, +) + + +def _system_slug_to_system_id(slug: str) -> str: + """Convert 'GJ-244A' → 'GJ 244A'. Mirrors populate_terrain_reference.py.""" + if slug.startswith("GJ-"): + return "GJ " + slug[3:] + return slug + + +def extract_hook(gttr_path: Path, max_words: int) -> str | None: + """Return a compact one-paragraph hook for this gttr.md, or None if + the file is missing or the opening paragraph can't be located.""" + if not gttr_path.exists(): + return None + text = gttr_path.read_text() + + match = _HOOK_RE.search(text) + if not match: + return None + + name_token = match.group(1).strip() + body = match.group(2).strip() + + # Reconstruct "NAME is …" without the markdown asterisks. Add a + # single space between the name and the body since `.strip()` above + # removed the leading space from the captured body. + hook = f"{name_token} {body}" + + # Collapse whitespace so multi-line paragraphs become one clean line. + hook = re.sub(r"\s+", " ", hook).strip() + + # Strip a leading orphan "is" that comes from `**NAME**` + " is …": + # the regex captures the word "is" on its own because the opener is + # typically `**GATEWAY** (known as Tau Ceti...) is the most connected…`. + # Normal reading already works — this is just hygiene. + hook = re.sub(r"\s*\(\s*\)\s*", " ", hook) + + # Hard cap at max_words. Truncate at the last word boundary before + # the cap and append an ellipsis so the reader knows it continues. + words = hook.split() + if len(words) > max_words: + hook = " ".join(words[:max_words]).rstrip(",;:") + "…" + + return hook + + +def main(): + parser = argparse.ArgumentParser( + description="Populate star_systems.gttr_hook from wiki gttr.md files" + ) + parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db") + parser.add_argument( + "--system", + help="Process only this system_id (e.g. 'GJ 71')", + ) + parser.add_argument( + "--max-words", + type=int, + default=45, + help="Soft cap on hook length (default: 45 words)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Extract and print but do not write to the DB", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print every extracted hook", + ) + args = parser.parse_args() + + db_path = Path(args.db) + if not db_path.exists(): + print(f"error: {db_path} not found", file=sys.stderr) + sys.exit(1) + + conn = sqlite3.connect(str(db_path), timeout=30.0) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=15000") + conn.execute("PRAGMA foreign_keys=ON") + + # Ensure the column exists — idempotent ADD COLUMN for old DBs. + try: + conn.execute("ALTER TABLE star_systems ADD COLUMN gttr_hook TEXT") + except sqlite3.OperationalError: + pass # column already exists + + print(f"\n populate_gttr_hook.py") + print(f" DB: {db_path}") + print(f" max words: {args.max_words}") + if args.dry_run: + print(f" Mode: DRY RUN") + print() + + rows = conn.execute( + "SELECT system_id, proper_name FROM star_systems ORDER BY system_id" + ).fetchall() + if args.system: + rows = [r for r in rows if r[0] == args.system] + + updated = 0 + missing = 0 + unmatched = 0 + + conn.execute("BEGIN") + try: + for system_id, proper_name in rows: + slug = system_id.replace(" ", "-", 1) + gttr_path = WIKI_SYSTEMS / slug / "gttr.md" + + hook = extract_hook(gttr_path, args.max_words) + if hook is None: + if not gttr_path.exists(): + missing += 1 + if args.verbose: + print(f" MISSING {system_id:10s} {gttr_path}") + else: + unmatched += 1 + if args.verbose: + print(f" UNMATCHED {system_id:10s} {gttr_path}") + continue + + if args.verbose: + display = f"{system_id} ({proper_name})" if proper_name else system_id + print(f" {display}") + print(f" → {hook}") + + if not args.dry_run: + conn.execute( + "UPDATE star_systems SET gttr_hook = ? WHERE system_id = ?", + (hook, system_id), + ) + updated += 1 + + if args.dry_run: + conn.rollback() + else: + conn.commit() + except BaseException: + conn.rollback() + conn.close() + raise + + conn.close() + + print(f"\n Done:") + print(f" updated: {updated}") + print(f" missing: {missing}") + print(f" unmatched: {unmatched}") + if args.dry_run: + print(f"\n Dry run — no DB writes.") + print() + + +if __name__ == "__main__": + main() diff --git a/tooling/planet-gen/gemma_naming.py b/tooling/planet-gen/gemma_naming.py index 3a7d43df4..6855e3933 100755 --- a/tooling/planet-gen/gemma_naming.py +++ b/tooling/planet-gen/gemma_naming.py @@ -137,44 +137,66 @@ class Logger: # source of corridor identity in systems.db (cultural_corridor is a # legacy field that was never populated beyond sol-gateway-axis). CORRIDOR_PALETTES: dict[str, dict[str, str]] = { + # IMPORTANT: the inflection is a DOMINANT bias, not a hard lock. + # Earlier iterations used single-culture labels like + # "administrative English" or "Korean/Japanese/Taiwanese" which + # Gemma 2 2B interpreted as "produce ONLY in this register" — and + # every core body came out anglophone, every east_reach body came + # out East Asian, etc. The real Earth diaspora in the setting is + # cosmopolitan: a British surveyor on an east_reach moon still + # names a river after their aunt in Dorset. The inflections now + # explicitly name the dominant register AND invite diaspora + # variety so Gemma samples from the full cross-cultural few-shot + # pool instead of collapsing to the dominant label. "core": { - # NOTE the style label is deliberately plain: earlier versions used - # "institutional Latin / pan-Anglo / Gateway-era" which biased - # Gemma 2 2B toward Latinate coinages like "Aureus" / "Aetheria". - # "Administrative English" gets prosaic output that matches the - # settler-named frontier feel the core corridor actually has. - "inflection": "administrative English / Gateway-era", - "examples": "Meridian, Concord, East Ridge, Landing, Old Gate, Foreman's Run", + "inflection": "Gateway-era Earth diaspora, mostly English, " + "all Earth cultures welcome", + "examples": "Meridian, Concord, East Ridge, Landing, Tanaka, " + "Rahman, Okafor, Ribeiro", }, "north_reach": { - "inflection": "British / Australian / Irish", - "examples": "Wolcott, Mildern, Ashbourne, Kiln, Briarfell, Tarndale", + "inflection": "British/Australian/Irish dominant, all Earth " + "cultures welcome", + "examples": "Wolcott, Mildern, Ashbourne, Kiln, Briarfell, " + "Tarndale, Ribeiro, Nakamura, Kowalski", }, "south_reach": { - "inflection": "Portuguese / Swahili / Cape Verdean / Brazilian", - "examples": "Vargas, Inhaca, Monteforte, Serra, Ribeiro, Kilimi", + "inflection": "Portuguese/Swahili/Cape Verdean/Brazilian " + "dominant, all Earth cultures welcome", + "examples": "Vargas, Inhaca, Monteforte, Serra, Ribeiro, " + "Kilimi, Holmberg, Fairview, Tanaka", }, "east_reach": { - "inflection": "Korean / Japanese / Taiwanese", - "examples": "Hanyang, Takamine, Seoraksan, Tsukuri, Ginoza, Baektu", + "inflection": "Korean/Japanese/Taiwanese dominant, all Earth " + "cultures welcome", + "examples": "Hanyang, Takamine, Seoraksan, Tsukuri, Ginoza, " + "Baektu, Kellogg, Oliveira, Novak", }, "west_reach": { - "inflection": "German / Dutch / Nordic", - "examples": "Vanebach, Kloosterdam, Bergfjord, Hellekade, Straend", + "inflection": "German/Dutch/Nordic dominant, all Earth cultures " + "welcome", + "examples": "Vanebach, Kloosterdam, Bergfjord, Hellekade, " + "Straend, Cooper, Vargas, Nakamura", }, "deep_frontier": { - "inflection": "founder-surname + noun (e.g. 'Okafor Reach', 'Stenner Cross')", - "examples": "Okafor Reach, Stenner Cross, Weller Hold, Pruitt Basin, Vickery Hold", + "inflection": "frontier founder-name era, any Earth culture", + "examples": "Okafor Reach, Stenner Cross, Weller Hold, Pruitt " + "Basin, Vickery Hold, Kellogg Run, Nakamura Post, " + "Ribeiro Landing", }, # Legacy keys retained for backward compatibility with the # cultural_corridor column on the one system that uses it. "sol-gateway-axis": { - "inflection": "administrative English / Gateway-era", - "examples": "Meridian, Concord, East Ridge, Landing, Old Gate, Foreman's Run", + "inflection": "Gateway-era Earth diaspora, mostly English, " + "all Earth cultures welcome", + "examples": "Meridian, Concord, Cardinal, Prefecture, Tanaka, " + "Rahman, Okafor", }, "inner_corridor": { - "inflection": "administrative English", - "examples": "Meridian, Concord, East Ridge, Landing, Old Gate", + "inflection": "Gateway-era Earth diaspora, mostly English, " + "all Earth cultures welcome", + "examples": "Meridian, Concord, East Ridge, Landing, Tanaka, " + "Rahman, Okafor", }, } @@ -657,6 +679,9 @@ def _build_prompt( body_id: str, local_id: str, attempt: int, + system_hook: str | None = None, + system_name: str | None = None, + body_name: str | None = None, ) -> str: """Assemble a few-shot prompt for the given feature type. @@ -664,6 +689,15 @@ def _build_prompt( (body_id, local_id, attempt) — this puts a finger on the sampling scales so neighbouring features on the same body don't all draw from an identical prompt and collapse to identical outputs. + + `system_hook` is the pre-extracted gttr.md characterisation for + this system (e.g. "RAN is, by all accounts, a very nice place to + live, provided you are the sort of person whose grandparents owned + the land"). When present it gets woven into the preamble so Gemma + has per-system cultural context — this is the single biggest lever + for producing names that feel like they belong to THIS world rather + than any world in this corridor. When absent the prompt degrades to + the corridor inflection alone. """ cfg = _PROMPT_CONFIG.get(feature_type) if cfg is None: @@ -684,6 +718,25 @@ def _build_prompt( # singular possessive ones ("their capital town"). Heuristic: if the # subject starts with "their", use "called"; otherwise "named". verb = "called" if subject.startswith("their ") else "named" + + # Context block. When system / body proper names are available, + # label them explicitly so Gemma has the canonical identifiers in + # addition to the gttr hook. Redundancy helps on a small model — + # the hook already opens with the system name but putting it in a + # labeled "System:" line makes it harder to gloss over. When the + # gttr hook exists too, it goes on its own line as the cultural + # one-liner. + context_lines: list[str] = [] + ident_parts = [] + if system_name: + ident_parts.append(f"System: {system_name}") + if body_name: + ident_parts.append(f"Planet: {body_name}") + if ident_parts: + context_lines.append(". ".join(ident_parts) + ".") + if system_hook: + context_lines.append(f"About the system: {system_hook}") + preamble = ( f"Settlers {verb} {subject} after themselves, after what they saw, " f"or after places back home. Most names are mundane, short, and " @@ -694,6 +747,9 @@ def _build_prompt( ) lines = [preamble, ""] + if context_lines: + lines.extend(context_lines) + lines.append("") for style, example in pool: lines.append(f"Style: {style}. Answer: {example}") lines.append("") @@ -1207,6 +1263,7 @@ def name_feature( body_used: set[str], stem_counts: dict[str, int], stem_cap: int, + system_hook: str | None, world_seed: int, body_id: str, local_id: str, @@ -1272,6 +1329,9 @@ def name_feature( body_id=body_id, local_id=local_id, attempt=attempt, + system_hook=system_hook, + system_name=ctx.get("system_proper_name"), + body_name=ctx.get("body_proper_name"), ) try: raw = voice.request(prompt, seed) @@ -1359,6 +1419,26 @@ def load_body_context(body_id: str, system_id: str, conn: sqlite3.Connection) -> } +def load_system_gttr_hooks(conn: sqlite3.Connection) -> dict[str, str]: + """Return `{system_id: gttr_hook}` for every system with a populated + hook. The gttr_hook column is filled by + `tooling/db/populate_gttr_hook.py` from wiki/star-systems//gttr.md + and captures the system's canonical characterisation in 30-45 words + ("where the rules live", "forty years old and still in the draft", + …). The naming pipeline injects this into the prompt preamble so + Gemma has per-system cultural context without having to parse + thousands of lines of wiki markdown at runtime. + + Systems whose gttr_hook is NULL return nothing from this map — the + caller falls back to the corridor palette alone. + """ + rows = conn.execute( + "SELECT system_id, gttr_hook FROM star_systems " + "WHERE gttr_hook IS NOT NULL AND gttr_hook != ''" + ).fetchall() + return {row[0]: row[1] for row in rows} + + def load_body_hop_order(conn: sqlite3.Connection) -> dict[str, tuple[int, str]]: """Return a `{body_id: (hop_distance_from_gateway, body_id)}` map used as a stable sort key so the pipeline walks the reach from core @@ -1416,6 +1496,7 @@ def process_body( corpus: dict[tuple[str, str], set[str]], stem_counts: dict[str, int], stem_cap: int, + system_hook: str | None, world_seed: int, log: "Logger", verbose: bool, @@ -1474,7 +1555,7 @@ def process_body( feature_type = _feature_type_for_city(city) name = name_feature( voice, feature_type, ctx, blocklist, corpus, body_used, - stem_counts, stem_cap, + stem_counts, stem_cap, system_hook, world_seed, body_id, city.get("id") or "city_?", log, verbose, ) @@ -1491,7 +1572,7 @@ def process_body( continue name = name_feature( voice, "river", ctx, blocklist, corpus, body_used, - stem_counts, stem_cap, + stem_counts, stem_cap, system_hook, world_seed, body_id, river.get("id") or "river_?", log, verbose, ) @@ -1511,7 +1592,7 @@ def process_body( feature_type = _feature_type_for_ocean(water) name = name_feature( voice, feature_type, ctx, blocklist, corpus, body_used, - stem_counts, stem_cap, + stem_counts, stem_cap, system_hook, world_seed, body_id, water.get("id") or "water_?", log, verbose, ) @@ -1530,7 +1611,7 @@ def process_body( continue name = name_feature( voice, "mountain_range", ctx, blocklist, corpus, body_used, - stem_counts, stem_cap, + stem_counts, stem_cap, system_hook, world_seed, body_id, rng_feat.get("id") or "range_?", log, verbose, ) @@ -1550,7 +1631,7 @@ def process_body( feature_type = _feature_type_for_poi(poi) name = name_feature( voice, feature_type, ctx, blocklist, corpus, body_used, - stem_counts, stem_cap, + stem_counts, stem_cap, system_hook, world_seed, body_id, poi.get("id") or "poi_?", log, verbose, ) @@ -1725,6 +1806,7 @@ def main(): ensure_atlas_schema(conn) hop_order = load_body_hop_order(conn) + system_gttr_hooks = load_system_gttr_hooks(conn) markers_paths = discover_bodies(args.body, args.limit, hop_order) if not markers_paths: log(f"error: no markers.json found (body={args.body})") @@ -1824,6 +1906,7 @@ def main(): corpus=corpus, stem_counts=stem_counts, stem_cap=args.stem_cap, + system_hook=system_gttr_hooks.get(system_id), world_seed=args.seed, log=log, verbose=args.verbose,