jpmschweitzer/settled-reach
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b0bfbfc7ddc7c3293c8dddaa77303871e4a2ddee
settled-reach/tooling/planet-gen/qa_naming.py
T
jpmschweitzerandClaude Opus 4.6 9ad9b88d7c feat(tooling): Gemma 4 batch naming pipeline with wiki-grounded register selection (#833) 
Replace the one-at-a-time Gemma 2 naming pipeline with a batch-oriented
Gemma 4 E2B pipeline. Key changes:

- naming_core.py: shared library with Levenshtein distinctiveness ranking,
  batch prompt building, mood injection pool, name validation, and
  adjacent-register refill logic
- Wiki-grounded register selection: per-system LLM call picks the cultural
  register based on wiki/GTTR content instead of hash randomizer
- Batch naming: requests N*2 names per call, ranks by word-average
  Levenshtein distance, fills quota from most-distinct candidates
- Mood pool: 13 emotional seeds randomized per-body for vocabulary
  divergence (ambition, fear, isolation, defiance, etc.)
- Adjacent-register refill: when primary register exhausts, automatically
  switches to next corridor substyle
- Inhabited-first body ordering: habitable worlds get first pick of
  register vocabulary, barren moons get leftovers
- Process group cleanup: SIGTERM/SIGKILL the full distrobox chain on
  subprocess refresh to prevent GPU zombie processes
- qa_naming.py: QA report, fix_fewshot_bleed.py: post-hoc fix script
- test_batch_naming.py, test_register_selection.py: test harnesses

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-17 16:09:23 +02:00

524 lines
19 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""QA report on atlas naming quality and distribution.
Runs checks against systems.db and markers.json files:
- Exact duplicates within systems
- Stem repetition (shared root words)
- Prompt fragment leaks
- Register bleed (wrong cultural register for corridor)
- Feature-type mismatches (street names as mountains, etc.)
- Short/long name outliers
- Body-name echo (planet name used as feature stem)
- Coverage gaps
- Distribution by corridor and register
Usage:
python3 tooling/planet-gen/qa_naming.py
python3 tooling/planet-gen/qa_naming.py --verbose
"""
import re
import sqlite3
import sys
from collections import Counter, defaultdict
from pathlib import Path
TOOLING_DIR = Path(__file__).resolve().parent
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"
sys.path.insert(0, str(TOOLING_DIR))
from naming_core import _words
# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------
def load_all_names(conn):
"""Load all named features grouped by system and body."""
results = []
for table, ftype in [
("atlas_cities", "city"),
("atlas_rivers", "river"),
("atlas_mountain_ranges", "mountain"),
("atlas_oceans", "ocean"),
("atlas_pois", "poi"),
]:
rows = conn.execute(f"""
SELECT a.body_id, a.name, b.system_id,
COALESCE(s.geographic_sector, 'unknown') as corridor,
COALESCE(s.proper_name, s.system_id) as system_name,
COALESCE(b.proper_name, '') as body_name,
b.inhabited
FROM {table} a
JOIN bodies b ON a.body_id = b.body_id
JOIN star_systems s ON b.system_id = s.system_id
WHERE a.name IS NOT NULL AND a.name != ''
""").fetchall()
for body_id, name, system_id, corridor, sys_name, body_name, inhabited in rows:
results.append({
"body_id": body_id,
"name": name,
"system_id": system_id,
"corridor": corridor,
"system_name": sys_name,
"body_name": body_name,
"feature_type": ftype,
"inhabited": bool(inhabited),
})
return results
# ---------------------------------------------------------------------------
# Checks
# ---------------------------------------------------------------------------
def check_prompt_fragments(names):
"""Find names that look like prompt leaks."""
fragments = [
"style:", "answer:", "insert your", "example", "placeholder",
"number only", "names:", "generate", "already used", "do not",
"must be", "distinct", "comma-separated", "best:", "option",
]
hits = []
for n in names:
low = n["name"].lower()
for frag in fragments:
if frag in low:
hits.append((n["name"], n["body_id"], n["system_name"], frag))
break
return hits
def check_exact_dupes_within_system(names):
"""Find exact duplicate names within the same system + feature type."""
by_sys_type = defaultdict(list)
for n in names:
key = (n["system_id"], n["feature_type"])
by_sys_type[key].append(n)
dupes = []
for key, group in by_sys_type.items():
seen = {}
for n in group:
low = n["name"].lower()
if low in seen:
dupes.append((n["name"], n["body_id"], seen[low], n["system_name"], n["feature_type"]))
else:
seen[low] = n["body_id"]
return dupes
def check_exact_dupes_within_body(names):
"""Find exact duplicate names within the same body across all types."""
by_body = defaultdict(list)
for n in names:
by_body[n["body_id"]].append(n)
dupes = []
for body_id, group in by_body.items():
seen = {}
for n in group:
low = n["name"].lower()
if low in seen:
dupes.append((n["name"], body_id, n["feature_type"], seen[low], n["system_name"]))
else:
seen[low] = n["feature_type"]
return dupes
def check_body_name_echo(names):
"""Find names where the body/system proper name dominates."""
hits = []
for n in names:
if not n["body_name"]:
continue
body_stem = n["body_name"].lower()
name_low = n["name"].lower()
# Check if body name appears as a word in the feature name
name_words = set(name_low.split())
body_words = set(body_stem.split())
if body_words & name_words:
hits.append((n["name"], n["body_name"], n["body_id"], n["system_name"]))
return hits
def check_register_bleed(names):
"""Find names that look like they're from the wrong cultural register.
Uses keyword heuristics — not perfect but catches obvious mismatches.
"""
# Register keywords that should NOT appear in certain corridors
bleed_patterns = {
"core": {
"wrong": ["kimchi", "samurai", "fjord", "veld", "kopje", "baobab"],
},
"east_reach": {
"wrong": ["bramble", "meadow", "thatch", "croft", "basilica", "forum", "senate"],
},
"west_reach": {
"wrong": ["sakura", "bamboo", "lotus", "jade", "dragon", "phoenix"],
},
"south_reach": {
"wrong": ["fjord", "viking", "norse", "highland", "glen"],
},
"north_reach": {
"wrong": ["sakura", "bamboo", "jade", "polder", "graben"],
},
}
# NZ/Australian names in non-core/north corridors
nz_keywords = ["pōhutukawa", "waitara", "wairarapa", "fiordland", "aotearoa",
"rangitoto", "wellington", "canterbury", "auckland", "otago",
"kauri", "pukekohe", "taranaki", "moana"]
hits = []
for n in names:
low = n["name"].lower()
corridor = n["corridor"]
# Check NZ bleed into non-Australian registers
if corridor not in ("core", "north_reach"):
for kw in nz_keywords:
if kw in low:
hits.append((n["name"], n["body_id"], corridor, n["system_name"],
f"NZ/AU keyword '{kw}' in {corridor}"))
break
# Check corridor-specific wrong keywords
if corridor in bleed_patterns:
for kw in bleed_patterns[corridor]["wrong"]:
if kw in low:
hits.append((n["name"], n["body_id"], corridor, n["system_name"],
f"keyword '{kw}' wrong for {corridor}"))
break
return hits
def check_feature_type_mismatch(names):
"""Find names that sound wrong for their feature type."""
# Street/road names shouldn't be mountains
street_words = {"street", "avenue", "boulevard", "drive", "road", "lane",
"way", "highway", "route", "thoroughfare"}
# Building names shouldn't be rivers/oceans
building_words = {"hall", "house", "building", "tower", "plaza", "square",
"station", "terminal", "center", "centre"}
hits = []
for n in names:
words = set(n["name"].lower().split())
if n["feature_type"] == "mountain" and words & street_words:
hits.append((n["name"], n["body_id"], n["feature_type"], n["system_name"],
f"street name as mountain"))
if n["feature_type"] in ("river", "ocean") and words & building_words:
# Allow "Hall" for classical register
if n["corridor"] != "core":
hits.append((n["name"], n["body_id"], n["feature_type"], n["system_name"],
f"building name as {n['feature_type']}"))
return hits
def check_stem_repetition(names):
"""Find bodies where too many features share the same first word."""
by_body = defaultdict(list)
for n in names:
by_body[n["body_id"]].append(n)
hits = []
for body_id, group in by_body.items():
# Count first significant word per feature type
by_type = defaultdict(list)
for n in group:
by_type[n["feature_type"]].append(n["name"])
for ftype, fnames in by_type.items():
if len(fnames) < 4:
continue
first_words = [_words(name)[0] if _words(name) else "" for name in fnames]
counts = Counter(first_words)
for word, count in counts.most_common(3):
if count >= 4 and word:
hits.append((body_id, ftype, word, count, len(fnames),
group[0]["system_name"]))
return hits
def check_short_long_names(names):
"""Find very short (1 word, ≤3 chars) or very long names."""
short = [(n["name"], n["body_id"], n["system_name"])
for n in names if len(n["name"]) <= 3]
long_ = [(n["name"], n["body_id"], n["system_name"])
for n in names if len(n["name"]) > 40]
return short, long_
def check_numbers_in_names(names):
"""Find names containing digits."""
return [(n["name"], n["body_id"], n["system_name"])
for n in names if re.search(r"\d", n["name"])]
def corridor_distribution(names):
"""Count names per corridor."""
counts = Counter(n["corridor"] for n in names)
return counts
def feature_type_distribution(names):
"""Count names per feature type."""
counts = Counter(n["feature_type"] for n in names)
return counts
def coverage_gaps(conn):
"""Find bodies with unnamed features."""
gaps = []
for table, ftype in [
("atlas_mountain_ranges", "mountain"),
("atlas_oceans", "ocean"),
("atlas_pois", "poi"),
]:
rows = conn.execute(f"""
SELECT a.body_id, COUNT(*) as total,
SUM(CASE WHEN a.name IS NULL OR a.name = '' THEN 1 ELSE 0 END) as blank,
COALESCE(s.proper_name, b.system_id) as sys_name,
b.inhabited
FROM {table} a
JOIN bodies b ON a.body_id = b.body_id
JOIN star_systems s ON b.system_id = s.system_id
GROUP BY a.body_id
HAVING blank > 0
ORDER BY b.inhabited DESC, blank DESC
""").fetchall()
for body_id, total, blank, sys_name, inhabited in rows:
gaps.append((body_id, ftype, blank, total, sys_name, bool(inhabited)))
return gaps
def most_common_names(names, top_n=20):
"""Find the most frequently used names across all systems."""
counts = Counter(n["name"].lower() for n in names)
return counts.most_common(top_n)
# ---------------------------------------------------------------------------
# Report
# ---------------------------------------------------------------------------
def main():
verbose = "--verbose" in sys.argv
conn = sqlite3.connect(str(DB_PATH), timeout=30.0)
print("Loading named features from DB...")
names = load_all_names(conn)
print(f" {len(names):,} named features loaded\n")
# === Distribution ===
print("=" * 70)
print("DISTRIBUTION")
print("=" * 70)
print("\nBy corridor:")
for corridor, count in sorted(corridor_distribution(names).items(), key=lambda x: -x[1]):
print(f" {corridor:20s} {count:>6,}")
print("\nBy feature type:")
for ftype, count in sorted(feature_type_distribution(names).items(), key=lambda x: -x[1]):
print(f" {ftype:20s} {count:>6,}")
# === Most common names ===
print(f"\n{'=' * 70}")
print("MOST COMMON NAMES (potential over-generation)")
print("=" * 70)
for name, count in most_common_names(names, 30):
if count >= 3:
print(f" {count:>4}x {name}")
# === Prompt fragments ===
print(f"\n{'=' * 70}")
print("PROMPT FRAGMENT LEAKS")
print("=" * 70)
fragments = check_prompt_fragments(names)
if fragments:
for name, body, sys_name, frag in fragments[:20]:
print(f" [{sys_name}/{body}] \"{name}\" (matched: {frag})")
if len(fragments) > 20:
print(f" ... and {len(fragments) - 20} more")
else:
print(" None found ✓")
print(f" Total: {len(fragments)}")
# === Exact dupes within system ===
print(f"\n{'=' * 70}")
print("EXACT DUPLICATES WITHIN SYSTEM (same name, same feature type)")
print("=" * 70)
sys_dupes = check_exact_dupes_within_system(names)
if sys_dupes:
for name, body1, body2, sys_name, ftype in sys_dupes[:20]:
print(f" [{sys_name}] \"{name}\" ({ftype}) on {body1} and {body2}")
if len(sys_dupes) > 20:
print(f" ... and {len(sys_dupes) - 20} more")
else:
print(" None found ✓")
print(f" Total: {len(sys_dupes)}")
# === Exact dupes within body ===
print(f"\n{'=' * 70}")
print("EXACT DUPLICATES WITHIN BODY (same name, different feature types)")
print("=" * 70)
body_dupes = check_exact_dupes_within_body(names)
if body_dupes:
for name, body, ftype1, ftype2, sys_name in body_dupes[:20]:
print(f" [{sys_name}/{body}] \"{name}\" as {ftype1} and {ftype2}")
if len(body_dupes) > 20:
print(f" ... and {len(body_dupes) - 20} more")
else:
print(" None found ✓")
print(f" Total: {len(body_dupes)}")
# === Stem repetition ===
print(f"\n{'=' * 70}")
print("STEM REPETITION (4+ features sharing first word on same body)")
print("=" * 70)
stems = check_stem_repetition(names)
if stems:
for body, ftype, word, count, total, sys_name in stems[:20]:
print(f" [{sys_name}/{body}] \"{word}\" appears {count}/{total} times in {ftype}s")
if len(stems) > 20:
print(f" ... and {len(stems) - 20} more")
else:
print(" None found ✓")
print(f" Total: {len(stems)}")
# === Body name echo ===
print(f"\n{'=' * 70}")
print("BODY NAME ECHO (planet name appears in feature name)")
print("=" * 70)
echoes = check_body_name_echo(names)
if echoes:
# Group by body
by_body = defaultdict(list)
for name, body_name, body_id, sys_name in echoes:
by_body[(body_id, body_name, sys_name)].append(name)
for (body_id, body_name, sys_name), echo_names in sorted(
by_body.items(), key=lambda x: -len(x[1])
)[:15]:
print(f" [{sys_name}/{body_id}] body=\"{body_name}\": {', '.join(echo_names[:5])}"
f"{'...' if len(echo_names) > 5 else ''} ({len(echo_names)} total)")
else:
print(" None found ✓")
print(f" Total: {len(echoes)} names across {len(set(e[2] for e in echoes))} bodies")
# === Register bleed ===
print(f"\n{'=' * 70}")
print("REGISTER BLEED (cultural mismatch for corridor)")
print("=" * 70)
bleeds = check_register_bleed(names)
if bleeds:
for name, body, corridor, sys_name, reason in bleeds[:30]:
print(f" [{sys_name}/{body}] \"{name}\" — {reason}")
if len(bleeds) > 30:
print(f" ... and {len(bleeds) - 30} more")
else:
print(" None found ✓")
print(f" Total: {len(bleeds)}")
# === Feature type mismatch ===
print(f"\n{'=' * 70}")
print("FEATURE TYPE MISMATCH (street names as mountains, etc.)")
print("=" * 70)
mismatches = check_feature_type_mismatch(names)
if mismatches:
for name, body, ftype, sys_name, reason in mismatches[:20]:
print(f" [{sys_name}/{body}] \"{name}\" — {reason}")
if len(mismatches) > 20:
print(f" ... and {len(mismatches) - 20} more")
else:
print(" None found ✓")
print(f" Total: {len(mismatches)}")
# === Short/long names ===
print(f"\n{'=' * 70}")
print("SHORT NAMES (≤3 chars)")
print("=" * 70)
short, long_ = check_short_long_names(names)
if short:
for name, body, sys_name in short[:15]:
print(f" [{sys_name}/{body}] \"{name}\"")
if len(short) > 15:
print(f" ... and {len(short) - 15} more")
else:
print(" None found ✓")
print(f" Total: {len(short)}")
print(f"\n{'=' * 70}")
print("LONG NAMES (>40 chars)")
print("=" * 70)
if long_:
for name, body, sys_name in long_[:15]:
print(f" [{sys_name}/{body}] \"{name}\"")
else:
print(" None found ✓")
print(f" Total: {len(long_)}")
# === Numbers in names ===
print(f"\n{'=' * 70}")
print("NUMBERS IN NAMES")
print("=" * 70)
numbered = check_numbers_in_names(names)
if numbered:
for name, body, sys_name in numbered[:15]:
print(f" [{sys_name}/{body}] \"{name}\"")
else:
print(" None found ✓")
print(f" Total: {len(numbered)}")
# === Coverage gaps ===
print(f"\n{'=' * 70}")
print("COVERAGE GAPS (bodies with unnamed features)")
print("=" * 70)
gaps = coverage_gaps(conn)
inhabited_gaps = [g for g in gaps if g[5]]
uninhabited_gaps = [g for g in gaps if not g[5]]
if inhabited_gaps:
print(f"\n INHABITED bodies with gaps ({len(inhabited_gaps)}):")
for body, ftype, blank, total, sys_name, _ in inhabited_gaps[:10]:
print(f" [{sys_name}/{body}] {blank}/{total} {ftype}s unnamed")
if uninhabited_gaps:
print(f"\n Uninhabited bodies with gaps ({len(uninhabited_gaps)}):")
gap_by_type = Counter(g[1] for g in uninhabited_gaps)
for ftype, count in gap_by_type.most_common():
total_blank = sum(g[2] for g in uninhabited_gaps if g[1] == ftype)
print(f" {ftype}: {count} bodies, {total_blank} unnamed features")
# === Summary ===
print(f"\n{'=' * 70}")
print("SUMMARY")
print("=" * 70)
total = len(names)
issues = (len(fragments) + len(sys_dupes) + len(body_dupes) +
len(stems) + len(bleeds) + len(mismatches) +
len(short) + len(long_) + len(numbered))
print(f" Total named features: {total:>8,}")
print(f" Total QA issues found: {issues:>8,}")
print(f" Issue rate: {issues/total*100:>7.2f}%")
print(f" Prompt fragment leaks: {len(fragments):>8,}")
print(f" Exact dupes (system): {len(sys_dupes):>8,}")
print(f" Exact dupes (body): {len(body_dupes):>8,}")
print(f" Stem repetition: {len(stems):>8,}")
print(f" Body name echo: {len(echoes):>8,}")
print(f" Register bleed: {len(bleeds):>8,}")
print(f" Feature type mismatch: {len(mismatches):>8,}")
print(f" Short names: {len(short):>8,}")
print(f" Long names: {len(long_):>8,}")
print(f" Numbers in names: {len(numbered):>8,}")
print(f" Coverage gaps (inhabited):{len(inhabited_gaps):>8,}")
print(f" Coverage gaps (uninh.): {len(uninhabited_gaps):>8,}")
conn.close()
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink