settled-reach/tooling/db/populate_gttr_hook.py

#!/usr/bin/env python3
"""
Extract a short narrative hook from every wiki/star-systems/<slug>/gttr.md
and store it on `star_systems.gttr_hook`.

The GTTR ("Drifter's Guide to the Reach") files follow a consistent
format: an H1 title, then a paragraph that opens with `**NAME**` followed
by a 1-3 sentence characterisation. That characterisation is the hook —
per D-191 §4 and the #833 naming pipeline it is the single strongest
cultural signal available for each system, capturing things like:

  - Gateway: "the most connected system in the Reach, the site of its
    oldest research institution, and the location of a gate that goes
    to Earth and that nobody uses"
  - Ran: "a very nice place to live, provided you are the sort of
    person whose grandparents owned the land"
  - ACB: "where the Lattice Commission lives, which means it is where
    the rules live"
  - Posto Avançado: "forward post — the place beyond the established
    line"

This script parses each gttr.md, regex-extracts the first `**NAME**`
paragraph, normalises whitespace, and truncates at a soft word cap so
the hook stays cheap to inject into prompts. Empty or unmatched files
leave the column NULL.

Idempotent, safe to re-run after any wiki update. Explicit transaction
wrapper with rollback on exception.

Usage:
    tooling/db/populate_gttr_hook.py
    tooling/db/populate_gttr_hook.py --max-words 45
    tooling/db/populate_gttr_hook.py --dry-run
    tooling/db/populate_gttr_hook.py --system "GJ 71"
"""

import argparse
import re
import sqlite3
import sys
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = (SCRIPT_DIR / ".." / "..").resolve()
DB_PATH = REPO_ROOT / "server" / "data" / "systems.db"
WIKI_SYSTEMS = REPO_ROOT / "wiki" / "star-systems"

# Matches the first paragraph that opens with `**NAME**` at the start
# of a line. Captures everything up to a blank line or the next H1/H2.
# Non-greedy on the content.
# Match a line that opens with `**NAME**` (any non-asterisk chars, since
# names contain accents from many alphabets — Á, Ž, Ç, Ñ, etc) followed
# by the paragraph body up to a blank line or the next markdown heading.
_HOOK_RE = re.compile(
    r"^\*\*([^\n*]+?)\*\*(.+?)(?:\n\n|\n#)",
    re.DOTALL | re.MULTILINE,
)


def _system_slug_to_system_id(slug: str) -> str:
    """Convert 'GJ-244A' → 'GJ 244A'. Mirrors populate_terrain_reference.py."""
    if slug.startswith("GJ-"):
        return "GJ " + slug[3:]
    return slug


def extract_hook(gttr_path: Path, max_words: int) -> str | None:
    """Return a compact one-paragraph hook for this gttr.md, or None if
    the file is missing or the opening paragraph can't be located."""
    if not gttr_path.exists():
        return None
    text = gttr_path.read_text()

    match = _HOOK_RE.search(text)
    if not match:
        return None

    name_token = match.group(1).strip()
    body = match.group(2).strip()

    # Reconstruct "NAME is …" without the markdown asterisks. Add a
    # single space between the name and the body since `.strip()` above
    # removed the leading space from the captured body.
    hook = f"{name_token} {body}"

    # Collapse whitespace so multi-line paragraphs become one clean line.
    hook = re.sub(r"\s+", " ", hook).strip()

    # Strip a leading orphan "is" that comes from `**NAME**` + " is …":
    # the regex captures the word "is" on its own because the opener is
    # typically `**GATEWAY** (known as Tau Ceti...) is the most connected…`.
    # Normal reading already works — this is just hygiene.
    hook = re.sub(r"\s*\(\s*\)\s*", " ", hook)

    # Hard cap at max_words. Truncate at the last word boundary before
    # the cap and append an ellipsis so the reader knows it continues.
    words = hook.split()
    if len(words) > max_words:
        hook = " ".join(words[:max_words]).rstrip(",;:") + "…"

    return hook


def main():
    parser = argparse.ArgumentParser(
        description="Populate star_systems.gttr_hook from wiki gttr.md files"
    )
    parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db")
    parser.add_argument(
        "--system",
        help="Process only this system_id (e.g. 'GJ 71')",
    )
    parser.add_argument(
        "--max-words",
        type=int,
        default=45,
        help="Soft cap on hook length (default: 45 words)",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Extract and print but do not write to the DB",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Print every extracted hook",
    )
    args = parser.parse_args()

    db_path = Path(args.db)
    if not db_path.exists():
        print(f"error: {db_path} not found", file=sys.stderr)
        sys.exit(1)

    conn = sqlite3.connect(str(db_path), timeout=30.0)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=15000")
    conn.execute("PRAGMA foreign_keys=ON")

    # Ensure the column exists — idempotent ADD COLUMN for old DBs.
    try:
        conn.execute("ALTER TABLE star_systems ADD COLUMN gttr_hook TEXT")
    except sqlite3.OperationalError:
        pass  # column already exists

    print("\n  populate_gttr_hook.py")
    print(f"  DB:        {db_path}")
    print(f"  max words: {args.max_words}")
    if args.dry_run:
        print("  Mode:      DRY RUN")
    print()

    rows = conn.execute(
        "SELECT system_id, proper_name FROM star_systems ORDER BY system_id"
    ).fetchall()
    if args.system:
        rows = [r for r in rows if r[0] == args.system]

    updated = 0
    missing = 0
    unmatched = 0

    conn.execute("BEGIN")
    try:
        for system_id, proper_name in rows:
            slug = system_id.replace(" ", "-", 1)
            gttr_path = WIKI_SYSTEMS / slug / "gttr.md"

            hook = extract_hook(gttr_path, args.max_words)
            if hook is None:
                if not gttr_path.exists():
                    missing += 1
                    if args.verbose:
                        print(f"  MISSING {system_id:10s}  {gttr_path}")
                else:
                    unmatched += 1
                    if args.verbose:
                        print(f"  UNMATCHED {system_id:10s}  {gttr_path}")
                continue

            if args.verbose:
                display = f"{system_id} ({proper_name})" if proper_name else system_id
                print(f"  {display}")
                print(f"    → {hook}")

            if not args.dry_run:
                conn.execute(
                    "UPDATE star_systems SET gttr_hook = ? WHERE system_id = ?",
                    (hook, system_id),
                )
            updated += 1

        if args.dry_run:
            conn.rollback()
        else:
            conn.commit()
    except BaseException:
        conn.rollback()
        conn.close()
        raise

    conn.close()

    print("\n  Done:")
    print(f"    updated:   {updated}")
    print(f"    missing:   {missing}")
    print(f"    unmatched: {unmatched}")
    if args.dry_run:
        print("\n  Dry run — no DB writes.")
    print()


if __name__ == "__main__":
    main()