#!/usr/bin/env python3 """ extract-csv.py Extracts all wiki system data into a single CSV file. Reads: docs/wiki/systems/*/index.md (YAML frontmatter + markdown body sections) Writes: docs/design/star-systems.csv Body sections (## headers) are mapped to columns: Supply Dependency -> supply_dependency Faction Notes -> faction_notes Silence Topic -> silence_topic Narrative Hook -> narrative_hook Calibration Note -> calibration_note """ import csv import re from pathlib import Path WIKI_DIR = Path(__file__).parent.parent.parent / "docs/wiki/systems" OUTPUT_CSV = Path(__file__).parent.parent.parent / "docs/design/star-systems.csv" # CSV column order — matches systems-framework.md section order COLUMNS = [ # I. Identity and Location "system_id", "astronomical_id", "proper_name", "system_name", "star_type", "geographic_sector", "geographic_band", "political_zone", # II. Physical Character "habitable_planet_count", "inhabited_planet_count", "asteroid_belt", "gas_giant", "habitability_profile", # III. Gate Infrastructure "horizon_station", "aperture_count", "gate_connections", "gate_topology", "span_gate_network", # IV. Settlement History "settlement_wave", "founding_motivation", "founding_culture_primary", "founding_culture_secondary", "cultural_persistence", "historical_event", "historical_event_age_years", "religious_status", "religious_generation_count", # V. Economic Life "economic_tier", "population", "economic_base_primary", "economic_base_secondary", "distribution_index", "imprint_access", "supply_dependency", # VI. Governance "governance_type", "dominant_faction", "primary_fault_line", "secondary_fault_line", # VII. Faction Presence "assembly_presence", "commission_presence", "syndic_presence", "syndic_type", "separatist_presence", "separatist_type", "institute_presence", "guardians_presence", "faction_notes", # VIII. Cultural Voice "cultural_register", "cultural_register_secondary", "ambient_anxiety", "local_pride", "atmospheric_tone", "atmospheric_tone_secondary", "active_situation", "silence_threshold", "silence_topic", # IX. Political Character "earth_alignment", "earth_proximity", "earth_tension", # X. Narrative Profile "primary_archetype", "secondary_archetype", "narrative_notable", "narrative_hook", "generation_priority", # XI. Content Notes "stability_index", "system_volatility", "cultural_corridor", "calibration_note", ] # Map markdown section headers to column names BODY_SECTION_MAP = { "supply dependency": "supply_dependency", "faction notes": "faction_notes", "silence topic": "silence_topic", "narrative hook": "narrative_hook", "calibration note": "calibration_note", } def parse_frontmatter(text: str) -> dict[str, str]: """Extract YAML frontmatter between --- delimiters, ignoring comments.""" match = re.match(r"^---\n(.*?\n)---", text, re.DOTALL) if not match: return {} fm_text = match.group(1) result = {} for line in fm_text.split("\n"): line = line.strip() if not line or line.startswith("#"): continue if ":" not in line: continue key, _, value = line.partition(":") key = key.strip() value = value.strip() # Strip quotes if value.startswith('"') and value.endswith('"'): value = value[1:-1] elif value.startswith("'") and value.endswith("'"): value = value[1:-1] # Convert YAML null to empty string if value == "~" or value.lower() == "null" or value == "": value = "" result[key] = value return result def parse_body_sections(text: str) -> dict[str, str]: """Extract content from ## sections in the markdown body.""" # Find where frontmatter ends match = re.match(r"^---\n.*?\n---\n", text, re.DOTALL) if not match: body = text else: body = text[match.end():] sections: dict[str, str] = {} current_key: str | None = None current_lines: list[str] = [] for line in body.split("\n"): if line.startswith("## "): # Save previous section if current_key is not None: sections[current_key] = "\n".join(current_lines).strip() header = line[3:].strip().lower() current_key = BODY_SECTION_MAP.get(header) current_lines = [] elif current_key is not None: # Skip HTML comments and the Derived Fields section if line.strip().startswith(""): continue current_lines.append(line) # Save last section if current_key is not None: sections[current_key] = "\n".join(current_lines).strip() return sections def main(): wiki_dirs = sorted(WIKI_DIR.iterdir()) rows: list[dict[str, str]] = [] errors: list[str] = [] for d in wiki_dirs: if not d.is_dir(): continue index_file = d / "index.md" if not index_file.exists(): errors.append(f"Missing index.md in {d.name}") continue text = index_file.read_text() # Parse frontmatter fm = parse_frontmatter(text) # Parse body sections body = parse_body_sections(text) # Merge into row row: dict[str, str] = {} for col in COLUMNS: if col in fm: row[col] = fm[col] elif col in body: row[col] = body[col] else: row[col] = "" rows.append(row) # Sort by system_id rows.sort(key=lambda r: r.get("system_id", "")) # Write CSV OUTPUT_CSV.parent.mkdir(parents=True, exist_ok=True) with open(OUTPUT_CSV, "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=COLUMNS) writer.writeheader() writer.writerows(rows) # Summary non_empty_counts = {col: 0 for col in COLUMNS} for row in rows: for col in COLUMNS: if row.get(col, ""): non_empty_counts[col] += 1 print(f"Extracted {len(rows)} systems to {OUTPUT_CSV}") if errors: print(f"\nErrors ({len(errors)}):") for e in errors: print(f" {e}") # Completion stats total_cells = len(rows) * len(COLUMNS) filled_cells = sum(non_empty_counts.values()) print(f"\nCompletion: {filled_cells}/{total_cells} cells filled ({filled_cells/total_cells*100:.1f}%)") print(f"\nColumn fill rates:") for col in COLUMNS: count = non_empty_counts[col] pct = count / len(rows) * 100 if rows else 0 bar = "█" * int(pct / 5) + "░" * (20 - int(pct / 5)) status = "✓" if pct == 100 else " " print(f" {status} {col:35s} {count:3d}/{len(rows)} {bar} {pct:.0f}%") if __name__ == "__main__": main()