settled-reach/tooling/wiki/extract-csv.py

#!/usr/bin/env python3
"""
extract-csv.py
Extracts all wiki system data into a single CSV file.

Reads: docs/wiki/systems/*/index.md (YAML frontmatter + markdown body sections)
Writes: docs/design/star-systems.csv

Body sections (## headers) are mapped to columns:
  Supply Dependency -> supply_dependency
  Faction Notes -> faction_notes
  Silence Topic -> silence_topic
  Narrative Hook -> narrative_hook
  Calibration Note -> calibration_note
"""

import csv
import re
from pathlib import Path

WIKI_DIR = Path(__file__).parent.parent.parent / "docs/wiki/systems"
OUTPUT_CSV = Path(__file__).parent.parent.parent / "docs/design/star-systems.csv"

# CSV column order — matches systems-framework.md section order
COLUMNS = [
    # I. Identity and Location
    "system_id", "astronomical_id", "proper_name", "system_name", "star_type",
    "geographic_sector", "geographic_band", "political_zone",
    # II. Physical Character
    "habitable_planet_count", "inhabited_planet_count", "asteroid_belt",
    "gas_giant", "habitability_profile",
    # III. Gate Infrastructure
    "horizon_station", "aperture_count", "gate_connections", "gate_topology",
    "span_gate_network",
    # IV. Settlement History
    "settlement_wave", "founding_motivation", "founding_culture_primary",
    "founding_culture_secondary", "cultural_persistence", "historical_event",
    "historical_event_age_years", "religious_status", "religious_generation_count",
    # V. Economic Life
    "economic_tier", "population", "economic_base_primary",
    "economic_base_secondary", "distribution_index", "imprint_access",
    "supply_dependency",
    # VI. Governance
    "governance_type", "dominant_faction", "primary_fault_line",
    "secondary_fault_line",
    # VII. Faction Presence
    "assembly_presence", "commission_presence", "syndic_presence", "syndic_type",
    "separatist_presence", "separatist_type", "institute_presence",
    "guardians_presence", "faction_notes",
    # VIII. Cultural Voice
    "cultural_register", "cultural_register_secondary", "ambient_anxiety",
    "local_pride", "atmospheric_tone", "atmospheric_tone_secondary",
    "active_situation", "silence_threshold", "silence_topic",
    # IX. Political Character
    "earth_alignment", "earth_proximity", "earth_tension",
    # X. Narrative Profile
    "primary_archetype", "secondary_archetype", "narrative_notable",
    "narrative_hook", "generation_priority",
    # XI. Content Notes
    "stability_index", "system_volatility", "cultural_corridor",
    "calibration_note",
]

# Map markdown section headers to column names
BODY_SECTION_MAP = {
    "supply dependency": "supply_dependency",
    "faction notes": "faction_notes",
    "silence topic": "silence_topic",
    "narrative hook": "narrative_hook",
    "calibration note": "calibration_note",
}


def parse_frontmatter(text: str) -> dict[str, str]:
    """Extract YAML frontmatter between --- delimiters, ignoring comments."""
    match = re.match(r"^---\n(.*?\n)---", text, re.DOTALL)
    if not match:
        return {}

    fm_text = match.group(1)
    result = {}

    for line in fm_text.split("\n"):
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        if ":" not in line:
            continue

        key, _, value = line.partition(":")
        key = key.strip()
        value = value.strip()

        # Strip quotes
        if value.startswith('"') and value.endswith('"'):
            value = value[1:-1]
        elif value.startswith("'") and value.endswith("'"):
            value = value[1:-1]

        # Convert YAML null to empty string
        if value == "~" or value.lower() == "null" or value == "":
            value = ""

        result[key] = value

    return result


def parse_body_sections(text: str) -> dict[str, str]:
    """Extract content from ## sections in the markdown body."""
    # Find where frontmatter ends
    match = re.match(r"^---\n.*?\n---\n", text, re.DOTALL)
    if not match:
        body = text
    else:
        body = text[match.end():]

    sections: dict[str, str] = {}
    current_key: str | None = None
    current_lines: list[str] = []

    for line in body.split("\n"):
        if line.startswith("## "):
            # Save previous section
            if current_key is not None:
                sections[current_key] = "\n".join(current_lines).strip()
            header = line[3:].strip().lower()
            current_key = BODY_SECTION_MAP.get(header)
            current_lines = []
        elif current_key is not None:
            # Skip HTML comments and the Derived Fields section
            if line.strip().startswith("<!--") and line.strip().endswith("-->"):
                continue
            current_lines.append(line)

    # Save last section
    if current_key is not None:
        sections[current_key] = "\n".join(current_lines).strip()

    return sections


def main():
    wiki_dirs = sorted(WIKI_DIR.iterdir())
    rows: list[dict[str, str]] = []
    errors: list[str] = []

    for d in wiki_dirs:
        if not d.is_dir():
            continue
        index_file = d / "index.md"
        if not index_file.exists():
            errors.append(f"Missing index.md in {d.name}")
            continue

        text = index_file.read_text()

        # Parse frontmatter
        fm = parse_frontmatter(text)

        # Parse body sections
        body = parse_body_sections(text)

        # Merge into row
        row: dict[str, str] = {}
        for col in COLUMNS:
            if col in fm:
                row[col] = fm[col]
            elif col in body:
                row[col] = body[col]
            else:
                row[col] = ""

        rows.append(row)

    # Sort by system_id
    rows.sort(key=lambda r: r.get("system_id", ""))

    # Write CSV
    OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True)
    with open(OUTPUT_CSV, "w", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=COLUMNS)
        writer.writeheader()
        writer.writerows(rows)

    # Summary
    non_empty_counts = {col: 0 for col in COLUMNS}
    for row in rows:
        for col in COLUMNS:
            if row.get(col, ""):
                non_empty_counts[col] += 1

    print(f"Extracted {len(rows)} systems to {OUTPUT_CSV}")
    if errors:
        print(f"\nErrors ({len(errors)}):")
        for e in errors:
            print(f"  {e}")

    # Completion stats
    total_cells = len(rows) * len(COLUMNS)
    filled_cells = sum(non_empty_counts.values())
    print(f"\nCompletion: {filled_cells}/{total_cells} cells filled ({filled_cells/total_cells*100:.1f}%)")
    print(f"\nColumn fill rates:")
    for col in COLUMNS:
        count = non_empty_counts[col]
        pct = count / len(rows) * 100 if rows else 0
        bar = "█" * int(pct / 5) + "░" * (20 - int(pct / 5))
        status = "✓" if pct == 100 else " "
        print(f"  {status} {col:35s}  {count:3d}/{len(rows)}  {bar}  {pct:.0f}%")


if __name__ == "__main__":
    main()