jpmschweitzerandClaude Opus 4.6
8ba3a1d0a7
Hoshe: - Mark --dump-prompts / name_feature() as vestigial with TODO note - Fix --refresh help string: 200 → 1000 (matches actual default) - Fix _RIVER_POOLS comment numbering: Pool 6 before Pool 5 → correct order - Remove dead first-pass code in fix_fewshot_bleed.py - _CAPTURE_FILE leak noted in vestigial TODO Tyre: - Fix stale "Gemma 2" strings in banner, argparse description, model help - Note dead code for cleanup pass (name_feature ~700 lines) Hoshe (prune): - prune_atlas_features.py: named features sort before unnamed, preventing silent discard of hand-authored names during pruning naming_core: - v0.2: few-shot blocklist, stricter is_valid_name (min 3 chars, no digits, no brackets), prompt fragment rejection expanded Miri clarification: the 261 "empty-string" files contain only roads (37) and railroads (37) — infrastructure features never in naming scope. All cities/rivers/oceans/mountains/POIs are clean. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
207 lines
8.5 KiB
Python
207 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Replace few-shot example names that bled into the output.
|
|
|
|
The batch naming prompt uses Scottish Highland and Dutch colonial
|
|
examples. The Scottish ones (Glen Moray, Dunvegan Ridge, Torridon,
|
|
Cairn Brae, The Kelpie's Spine) leaked into 270 features. This script
|
|
replaces them with unique names from a combined Scottish/Welsh/Irish
|
|
pool, ensuring no collisions with the existing corpus.
|
|
"""
|
|
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
TOOLING_DIR = Path(__file__).resolve().parent
|
|
REPO_ROOT = (TOOLING_DIR / ".." / "..").resolve()
|
|
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
|
|
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"
|
|
|
|
sys.path.insert(0, str(TOOLING_DIR))
|
|
from generate_atlas import sync_markers_to_db
|
|
|
|
# The few-shot names to replace
|
|
FEWSHOT_NAMES = {
|
|
"glen moray", "dunvegan ridge", "torridon", "cairn brae", "the kelpie's spine",
|
|
"kloosterbeek", "nieuw rijn", "hoogland run", "van diemen's creek",
|
|
}
|
|
|
|
# Scottish / Welsh / Irish replacement pool — 300+ names to cover 270 replacements
|
|
# with room for Levenshtein filtering. Mix of geographic feature styles.
|
|
REPLACEMENT_POOL = [
|
|
# Scottish
|
|
"Glenfinnan", "Dalwhinnie Pass", "Cairngorm", "Loch Maree",
|
|
"Kinlochleven", "Strathspey", "Brae Morar", "Skye Reach",
|
|
"Ardnamurchan", "Kintail", "Glen Affric", "Lochaber",
|
|
"Killiecrankie", "Rannoch Moor", "Glen Coe", "Strathnaver",
|
|
"Applecross", "Torrisdale", "Durness", "Assynt",
|
|
"Coigach", "Inverpolly", "Sandwood", "Cape Wrath",
|
|
"Sutherland", "Helmsdale", "Brora", "Golspie",
|
|
"Cromarty", "Dornoch", "Nairn", "Forres",
|
|
"Culbin", "Findhorn", "Spey Bay", "Buckie",
|
|
"Banff", "Fraserburgh", "Peterhead", "Cruden Bay",
|
|
"Slains", "Ythan", "Bennachie", "Morven",
|
|
"Lochnagar", "Braemar", "Balmoral", "Crathie",
|
|
"Ballater", "Dinnet", "Tarland", "Lumphanan",
|
|
"Corgarff", "Tomintoul", "Glenlivet", "Dufftown",
|
|
"Craigellachie", "Aberlour", "Knockando", "Archiestown",
|
|
"Rothes", "Elgin", "Lossiemouth", "Burghead",
|
|
"Kinloss", "Alves", "Pluscarden", "Dallas",
|
|
# Welsh
|
|
"Cwm Idwal", "Beddgelert", "Crib Goch", "Tryfan",
|
|
"Ogwen", "Llyn Padarn", "Dolgellau", "Harlech",
|
|
"Rhinog", "Cader Idris", "Barmouth", "Aberdovey",
|
|
"Tywyn", "Machynlleth", "Pumlumon", "Hafren",
|
|
"Elan Valley", "Claerwen", "Llandrindod", "Brecon",
|
|
"Pen y Fan", "Corn Du", "Crickhowell", "Llangorse",
|
|
"Talgarth", "Hay Bluff", "Mynydd Troed", "Mynydd Llangorse",
|
|
"Skirrid", "Blorenge", "Llanfoist", "Govilon",
|
|
"Gilwern", "Llangattock", "Crug Hywel", "Cwm Clydach",
|
|
"Pontneddfechan", "Ystradfellte", "Sgwd yr Eira", "Henrhyd",
|
|
"Carreg Cennen", "Dinefwr", "Llandeilo", "Dryslwyn",
|
|
"Tywi Valley", "Carmarthen", "Kidwelly", "Pembrey",
|
|
"Gower", "Rhossili", "Oxwich", "Port Eynon",
|
|
"Pennard", "Langland", "Caswell", "Mumbles",
|
|
"Merthyr Mawr", "Ogmore", "Dunraven", "Llantwit",
|
|
"Monknash", "Nash Point", "Aberthaw", "Fonmon",
|
|
# Irish
|
|
"Glendalough", "Lugnaquilla", "Glen Imaal", "Wicklow Gap",
|
|
"Sally Gap", "Kippure", "Djuce", "Maulin",
|
|
"Djouce", "Great Sugar Loaf", "Bray Head", "Killiney",
|
|
"Dalkey", "Howth", "Lambay", "Ireland's Eye",
|
|
"Malahide", "Portmarnock", "Donabate", "Skerries",
|
|
"Balbriggan", "Gormanston", "Bettystown", "Laytown",
|
|
"Slane", "Newgrange", "Dowth", "Knowth",
|
|
"Tara", "Trim", "Navan", "Kells",
|
|
"Loughcrew", "Oldcastle", "Castlepollard", "Fore",
|
|
"Delvin", "Mullingar", "Kilbeggan", "Tullamore",
|
|
"Clara", "Ferbane", "Banagher", "Shannonbridge",
|
|
"Clonmacnoise", "Ballinasloe", "Aughrim", "Loughrea",
|
|
"Portumna", "Mountshannon", "Killaloe", "Ballina",
|
|
"Nenagh", "Roscrea", "Templemore", "Thurles",
|
|
"Cashel", "Cahir", "Clonmel", "Carrick-on-Suir",
|
|
"Piltown", "Mooncoin", "Waterford", "Tramore",
|
|
"Bunmahon", "Ardmore", "Youghal", "Midleton",
|
|
"Cobh", "Crosshaven", "Kinsale", "Clonakilty",
|
|
"Skibbereen", "Bantry", "Glengarriff", "Kenmare",
|
|
"Sneem", "Caherdaniel", "Waterville", "Cahersiveen",
|
|
"Valentia", "Portmagee", "Skellig", "Dingle",
|
|
"Brandon", "Castlegregory", "Fenit", "Tralee",
|
|
"Listowel", "Ballybunion", "Tarbert", "Glin",
|
|
"Foynes", "Askeaton", "Adare", "Patrickswell",
|
|
# More Scottish/Gaelic to fill
|
|
"Stornoway", "Tarbert", "Scalpay", "Eriskay",
|
|
"Barra", "Vatersay", "Minguilay", "Pabbay",
|
|
"Berneray", "Monach Isles", "Balranald", "Lochmaddy",
|
|
"Benbecula", "Grimsay", "Ronay", "Wiay",
|
|
"Canna", "Rum", "Eigg", "Muck",
|
|
"Ardnish", "Arisaig", "Morar", "Mallaig",
|
|
"Knoydart", "Barrisdale", "Arnisdale", "Glenelg",
|
|
"Sandaig", "Brochs of Borve", "Callanish", "Garenin",
|
|
"Carloway", "Arnol", "Barvas", "Tolsta",
|
|
"Ness", "Europie", "Swainbost", "Skigersta",
|
|
# Additional Welsh/Irish
|
|
"Aberystwyth", "Llanberis", "Betws-y-Coed", "Conwy",
|
|
"Caernarfon", "Pwllheli", "Abersoch", "Nefyn",
|
|
"Llanbedrog", "Criccieth", "Porthmadog", "Portmeirion",
|
|
"Trawsfynydd", "Ffestiniog", "Blaenau", "Llyn Tegid",
|
|
"Corwen", "Llangollen", "Chirk", "Oswestry",
|
|
]
|
|
|
|
|
|
def load_global_names(conn):
|
|
"""Load all existing names globally for uniqueness checking."""
|
|
names = set()
|
|
for table in ['atlas_cities', 'atlas_rivers', 'atlas_mountain_ranges',
|
|
'atlas_oceans', 'atlas_pois']:
|
|
rows = conn.execute(
|
|
f"SELECT lower(name) FROM {table} WHERE name IS NOT NULL AND name != ''"
|
|
).fetchall()
|
|
names.update(r[0] for r in rows)
|
|
return names
|
|
|
|
|
|
def main():
|
|
conn = sqlite3.connect(str(DB_PATH), timeout=30.0)
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA busy_timeout=15000")
|
|
|
|
global_names = load_global_names(conn)
|
|
print(f"Loaded {len(global_names)} existing names")
|
|
|
|
# Build available replacements (not already in corpus)
|
|
available = [n for n in REPLACEMENT_POOL if n.lower() not in global_names]
|
|
print(f"Available replacements: {len(available)} (from pool of {len(REPLACEMENT_POOL)})")
|
|
|
|
# Find all features that need replacement
|
|
replacements_needed = []
|
|
for markers_path in sorted(WIKI_SYSTEMS.glob("*/bodies/*/markers.json")):
|
|
body_id = markers_path.parent.name
|
|
m = json.loads(markers_path.read_text())
|
|
for section in ("cities", "rivers", "oceans", "mountain_ranges", "pois"):
|
|
for feat in m.get(section, []):
|
|
name = feat.get("name", "")
|
|
if name and name.lower() in FEWSHOT_NAMES:
|
|
replacements_needed.append((markers_path, body_id, section, feat))
|
|
|
|
print(f"Features to replace: {len(replacements_needed)}")
|
|
|
|
if len(available) < len(replacements_needed):
|
|
print(f"WARNING: only {len(available)} replacements for {len(replacements_needed)} features")
|
|
print(" some features will keep their few-shot names")
|
|
|
|
# Group by file, reload, replace, write
|
|
replacement_idx = 0
|
|
used_per_body = defaultdict(set)
|
|
changed_bodies = []
|
|
|
|
# Group by file
|
|
by_file = defaultdict(list)
|
|
for markers_path, body_id, section, feat in replacements_needed:
|
|
by_file[markers_path].append((body_id, section, feat["id"] if "id" in feat else None))
|
|
|
|
for markers_path, entries in by_file.items():
|
|
body_id = markers_path.parent.name
|
|
m = json.loads(markers_path.read_text())
|
|
changed = False
|
|
|
|
for _, section, feat_id in entries:
|
|
for feat in m.get(section, []):
|
|
name = feat.get("name") or ""
|
|
if not name or name.lower() not in FEWSHOT_NAMES:
|
|
continue
|
|
|
|
assigned = None
|
|
for attempt in range(len(available)):
|
|
candidate = available[(replacement_idx + attempt) % len(available)]
|
|
if candidate.lower() not in used_per_body[body_id]:
|
|
assigned = candidate
|
|
replacement_idx = (replacement_idx + attempt + 1) % len(available)
|
|
break
|
|
|
|
if assigned:
|
|
feat["name"] = assigned
|
|
used_per_body[body_id].add(assigned.lower())
|
|
changed = True
|
|
|
|
if changed:
|
|
markers_path.write_text(json.dumps(m, indent=2) + "\n")
|
|
changed_bodies.append(body_id)
|
|
# Sync to DB if body exists
|
|
try:
|
|
sync_markers_to_db(conn, body_id, m)
|
|
except Exception:
|
|
pass # orphan body
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
print(f"\nReplaced few-shot names on {len(changed_bodies)} bodies")
|
|
print("Done.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|