chore(tooling): update tooling for GJ ID migration and wiki sync
Update wiki_sync.py and process-wiki-system-changes to remove astronomical_id references. Add migrate-s-to-gj.py migration script. Update generate-stubs.py and validate-ron for new schema. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
+111
-113
@@ -1,19 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
process-wiki-system-changes
|
||||
Propagates wiki changes to star-map.json and SQLite.
|
||||
Pipeline for star systems data.
|
||||
|
||||
Wiki pages (wiki/star-systems/*/index.md) are the single source of truth.
|
||||
This script:
|
||||
1. Reads all wiki frontmatter
|
||||
2. Updates star-map.json nodes (preserving edges/topology)
|
||||
3. Syncs all data into SQLite star_systems table
|
||||
4. Reports completion stats
|
||||
Modes:
|
||||
(default) Import wiki frontmatter → systems.db, update star-map.json,
|
||||
then regenerate wiki pages from DB (infobox + topology).
|
||||
--generate Generate wiki pages from existing DB (no import).
|
||||
--rebuild-db Drop and recreate systems.db from wiki (fresh start).
|
||||
--stats Show fill rates.
|
||||
--dry-run Show what star-map.json changes would occur.
|
||||
|
||||
Usage:
|
||||
tooling/process-wiki-system-changes Run full sync
|
||||
tooling/process-wiki-system-changes --dry-run Show what would change
|
||||
tooling/process-wiki-system-changes --stats Just show fill rates
|
||||
Database: server/data/systems.db
|
||||
Schema: server/data/systems-schema.sql
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -24,111 +23,100 @@ from pathlib import Path
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
WORKTREE_ROOT = (SCRIPT_DIR / "..").resolve()
|
||||
STAR_MAP_PATH = WORKTREE_ROOT / "docs" / "design" / "star-map.json"
|
||||
DB_PATH = WORKTREE_ROOT / "server" / "data" / "systems.db"
|
||||
|
||||
# Import wiki_sync for DB operations
|
||||
# Import wiki_sync
|
||||
sys.path.insert(0, str(SCRIPT_DIR / "db"))
|
||||
import wiki_sync
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Star-map.json update
|
||||
# Star-map.json update (topology fields only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Fields that star-map.json nodes carry (topology + join keys)
|
||||
STAR_MAP_NODE_FIELDS = [
|
||||
"system_id", "astronomical_id",
|
||||
"system_id",
|
||||
"gate_topology", "aperture_count", "gate_connections",
|
||||
"hop_distance_from_gateway",
|
||||
]
|
||||
|
||||
|
||||
def update_star_map(wiki_systems: dict) -> dict:
|
||||
"""Update star-map.json node fields from wiki data. Returns summary."""
|
||||
def update_star_map_from_db():
|
||||
"""Update star-map.json node fields from systems.db."""
|
||||
conn = wiki_sync.get_connection()
|
||||
|
||||
with open(STAR_MAP_PATH, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
nodes = data["nodes"]
|
||||
updated = 0
|
||||
added = 0
|
||||
wiki_sids = set(wiki_systems.keys())
|
||||
map_sids = {n["system_id"] for n in nodes}
|
||||
|
||||
# Update existing nodes
|
||||
for node in nodes:
|
||||
sid = node["system_id"]
|
||||
if sid not in wiki_systems:
|
||||
row = conn.execute(
|
||||
"SELECT g.gate_topology, g.aperture_count, g.gate_connections "
|
||||
"FROM star_systems s LEFT JOIN system_gates g ON s.system_id = g.system_id "
|
||||
"WHERE s.system_id = ?", (sid,)
|
||||
).fetchone()
|
||||
|
||||
if not row:
|
||||
continue
|
||||
|
||||
wiki = wiki_systems[sid]
|
||||
changed = False
|
||||
for field in STAR_MAP_NODE_FIELDS:
|
||||
if field in wiki and wiki[field] != "":
|
||||
val = wiki[field]
|
||||
# Coerce integers
|
||||
if field in ("aperture_count", "gate_connections", "hop_distance_from_gateway"):
|
||||
try:
|
||||
val = int(val)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if node.get(field) != val:
|
||||
node[field] = val
|
||||
changed = True
|
||||
field_map = {
|
||||
"gate_topology": row["gate_topology"],
|
||||
"aperture_count": row["aperture_count"],
|
||||
"gate_connections": row["gate_connections"],
|
||||
}
|
||||
|
||||
for field, val in field_map.items():
|
||||
if val is not None and node.get(field) != val:
|
||||
node[field] = val
|
||||
changed = True
|
||||
|
||||
# Preserve is_gateway flag
|
||||
if changed:
|
||||
updated += 1
|
||||
|
||||
# Add nodes for wiki pages that aren't in star-map yet
|
||||
for sid in wiki_sids - map_sids:
|
||||
wiki = wiki_systems[sid]
|
||||
new_node = {"system_id": sid}
|
||||
for field in STAR_MAP_NODE_FIELDS:
|
||||
if field in wiki and wiki[field] != "":
|
||||
val = wiki[field]
|
||||
if field in ("aperture_count", "gate_connections", "hop_distance_from_gateway"):
|
||||
try:
|
||||
val = int(val)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
new_node[field] = val
|
||||
nodes.append(new_node)
|
||||
added += 1
|
||||
conn.close()
|
||||
|
||||
with open(STAR_MAP_PATH, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.write("\n")
|
||||
|
||||
return {
|
||||
"nodes": len(nodes),
|
||||
"edges": len(data["edges"]),
|
||||
"updated": updated,
|
||||
"added": added,
|
||||
}
|
||||
return {"nodes": len(nodes), "edges": len(data["edges"]), "updated": updated}
|
||||
|
||||
|
||||
def dry_run(wiki_systems: dict) -> dict:
|
||||
"""Show what would change in star-map.json without writing."""
|
||||
def dry_run_star_map():
|
||||
"""Show what would change in star-map.json."""
|
||||
conn = wiki_sync.get_connection()
|
||||
|
||||
with open(STAR_MAP_PATH, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
changes = []
|
||||
for node in data["nodes"]:
|
||||
sid = node["system_id"]
|
||||
if sid not in wiki_systems:
|
||||
continue
|
||||
wiki = wiki_systems[sid]
|
||||
for field in STAR_MAP_NODE_FIELDS:
|
||||
if field in wiki and wiki[field] != "":
|
||||
val = wiki[field]
|
||||
if field in ("aperture_count", "gate_connections", "hop_distance_from_gateway"):
|
||||
try:
|
||||
val = int(val)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if node.get(field) != val:
|
||||
changes.append(f" {sid}.{field}: {node.get(field)} -> {val}")
|
||||
row = conn.execute(
|
||||
"SELECT g.gate_topology, g.aperture_count, g.gate_connections "
|
||||
"FROM star_systems s LEFT JOIN system_gates g ON s.system_id = g.system_id "
|
||||
"WHERE s.system_id = ?", (sid,)
|
||||
).fetchone()
|
||||
|
||||
return {"changes": changes}
|
||||
if not row:
|
||||
continue
|
||||
|
||||
field_map = {
|
||||
"gate_topology": row["gate_topology"],
|
||||
"aperture_count": row["aperture_count"],
|
||||
"gate_connections": row["gate_connections"],
|
||||
}
|
||||
|
||||
for field, val in field_map.items():
|
||||
if val is not None and node.get(field) != val:
|
||||
changes.append(f" {sid}.{field}: {node.get(field)} -> {val}")
|
||||
|
||||
conn.close()
|
||||
return changes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -138,63 +126,73 @@ def dry_run(wiki_systems: dict) -> dict:
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Propagate wiki changes to star-map.json and SQLite"
|
||||
description="Star systems data pipeline: wiki ↔ systems.db ↔ star-map.json"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show changes without writing")
|
||||
parser.add_argument("--stats", action="store_true", help="Just show fill rates")
|
||||
parser.add_argument("--generate", action="store_true", help="Generate wiki from DB only")
|
||||
parser.add_argument("--rebuild-db", action="store_true", help="Drop and recreate DB from wiki")
|
||||
parser.add_argument("--stats", action="store_true", help="Show fill rates")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show star-map changes without writing")
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg = wiki_sync.load_config()
|
||||
|
||||
if args.stats:
|
||||
wiki_sync.stats(cfg)
|
||||
wiki_sync.stats()
|
||||
return
|
||||
|
||||
# Load wiki
|
||||
wiki_systems, warnings, body_count = wiki_sync.load_wiki_pages()
|
||||
if not wiki_systems:
|
||||
print("ERROR: No wiki pages found")
|
||||
for w in warnings:
|
||||
print(f" {w}")
|
||||
sys.exit(1)
|
||||
if args.generate:
|
||||
print("--- Generating wiki pages from DB ---")
|
||||
result = wiki_sync.generate_wiki()
|
||||
print(result["summary"])
|
||||
return
|
||||
|
||||
print(f"Loaded {len(wiki_systems)} wiki pages ({body_count} body sections filled)")
|
||||
if warnings:
|
||||
print(f"Warnings: {len(warnings)}")
|
||||
for w in warnings[:5]:
|
||||
print(f" {w}")
|
||||
if len(warnings) > 5:
|
||||
print(f" ... and {len(warnings) - 5} more")
|
||||
if args.rebuild_db:
|
||||
print(f"--- Rebuilding {DB_PATH} from wiki ---")
|
||||
if DB_PATH.exists():
|
||||
DB_PATH.unlink()
|
||||
print(" Deleted existing DB")
|
||||
|
||||
if args.dry_run:
|
||||
result = dry_run(wiki_systems)
|
||||
if result["changes"]:
|
||||
print(f"\nstar-map.json changes ({len(result['changes'])}):")
|
||||
for c in result["changes"]:
|
||||
# Need DB populated first for dry-run to work
|
||||
if not DB_PATH.exists():
|
||||
print("No DB found. Run without --dry-run first.")
|
||||
sys.exit(1)
|
||||
changes = dry_run_star_map()
|
||||
if changes:
|
||||
print(f"star-map.json changes ({len(changes)}):")
|
||||
for c in changes:
|
||||
print(c)
|
||||
else:
|
||||
print("\nstar-map.json: no changes")
|
||||
print("\n(dry run — nothing written)")
|
||||
print("star-map.json: no changes")
|
||||
return
|
||||
|
||||
# Step 1: Update star-map.json
|
||||
print("\n--- Updating star-map.json ---")
|
||||
map_result = update_star_map(wiki_systems)
|
||||
print(f"Nodes: {map_result['nodes']}, Edges: {map_result['edges']}")
|
||||
print(f"Updated: {map_result['updated']}, Added: {map_result['added']}")
|
||||
# Full pipeline: wiki → DB → star-map.json → wiki pages
|
||||
print("=== Star Systems Pipeline ===\n")
|
||||
|
||||
# Step 2: Sync to SQLite
|
||||
print("\n--- Syncing to SQLite ---")
|
||||
db_result = wiki_sync.sync(cfg)
|
||||
if db_result["ok"]:
|
||||
print(db_result["summary"])
|
||||
else:
|
||||
print(f"ERROR: {db_result.get('error')}")
|
||||
# Step 1: Import wiki frontmatter into DB
|
||||
print("--- Step 1: Wiki → DB ---")
|
||||
result = wiki_sync.import_from_wiki()
|
||||
if not result["ok"]:
|
||||
print(f"ERROR: {result.get('error')}")
|
||||
sys.exit(1)
|
||||
print(result["summary"])
|
||||
if result["warnings"]:
|
||||
for w in result["warnings"][:5]:
|
||||
print(f" ⚠ {w}")
|
||||
if len(result["warnings"]) > 5:
|
||||
print(f" ... and {len(result['warnings']) - 5} more")
|
||||
|
||||
# Step 3: Quick stats
|
||||
# Step 2: Update star-map.json from DB
|
||||
print("\n--- Step 2: DB → star-map.json ---")
|
||||
map_result = update_star_map_from_db()
|
||||
print(f"Nodes: {map_result['nodes']}, Edges: {map_result['edges']}, Updated: {map_result['updated']}")
|
||||
|
||||
# Step 3: Generate wiki pages from DB
|
||||
print("\n--- Step 3: DB → Wiki pages ---")
|
||||
gen_result = wiki_sync.generate_wiki()
|
||||
print(gen_result["summary"])
|
||||
|
||||
# Step 4: Stats
|
||||
print("\n--- Completion ---")
|
||||
wiki_sync.stats(cfg)
|
||||
wiki_sync.stats()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user