jpmschweitzerandClaude Fable 5
4f73624eff
import_economics.py 2,620 → 309 lines — a thin orchestrator keeping the exact CLI, single-transaction/rollback contract, and exit codes. The 16 import steps, MIGRATION_SQL, brands shell-out, validators, and stamp write now live in tooling/economy-db/economy_import/ (db, migration, economy, corporations, brands, bodies, atlas, specialization, traits, validators, stamp, paths, errors). Full type hints throughout. tooling/generator_sources.py replaces the triplicated source registry (importer / stamp checker / pr-process watch list — the skill now derives its list via --list). The registry stamps itself, and economy_import/ modules are globbed fail-closed, so a future module is stamped the moment it exists — closing the silently-weakened-stamp failure mode. Rider: connector config helpers centralized in tooling/db/common.py. Byte-identical behavior proven: full-import table dump diff EMPTY over 107,843 lines / 37 tables (volatile timestamp fields excluded); dry-run output parity; generated_brands.toml sha unchanged. make test-tooling PASS; ruff clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
310 lines
14 KiB
Python
Executable File
310 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
Import economics data into systems.db.
|
||
|
||
Reads TOML/JSON source files and populates the economics tables:
|
||
- gate_links from docs/design/star-map.json (335 edges, bidirectional)
|
||
- commodities from wiki/economics/commodities.toml (36 types)
|
||
- production_chains + chain_inputs from wiki/economics/production_chains.toml
|
||
- currency_zone on star_systems (default TRACTUS_PRIMARY)
|
||
- gate_energy_connected on star_systems (D-186: false for MARK_PRIMARY zones)
|
||
- corporations from wiki/corporations/*.md (sync + insert new records)
|
||
- corp_presence from wiki/corporations/*.md (headquarters location data)
|
||
|
||
Validation (hard errors, non-zero exit on any failure):
|
||
- Wiki corporation names must match DB proper_name records (D-182 sync constraint)
|
||
- Chain completeness: every intermediate commodity has at least one production chain
|
||
- Commodity coverage: 3+ corporations per major commodity type (D-175)
|
||
- System coverage: 1+ corporation per inhabited system with population > 100K (D-175)
|
||
|
||
This file is the CLI entrypoint and single-transaction orchestrator; the import
|
||
steps live in the economy_import package (T-1067). The stamped source set is
|
||
defined in tooling/generator_sources.py.
|
||
|
||
Usage:
|
||
python3 tooling/economy-db/import_economics.py
|
||
python3 tooling/economy-db/import_economics.py --dry-run
|
||
python3 tooling/economy-db/import_economics.py --db path/to/systems.db
|
||
"""
|
||
|
||
import argparse
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
# tooling/economy-db is not a package (hyphenated dir) — put it on sys.path so
|
||
# the economy_import package resolves; its __init__ adds tooling/ for
|
||
# generator_sources and schema_version.
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
|
||
from economy_import import ( # noqa: E402
|
||
atlas,
|
||
bodies,
|
||
brands,
|
||
corporations,
|
||
db,
|
||
economy,
|
||
migration,
|
||
specialization,
|
||
stamp,
|
||
traits,
|
||
validators,
|
||
)
|
||
from economy_import.errors import ImportAborted # noqa: E402
|
||
from economy_import.paths import DB_PATH # noqa: E402
|
||
from generator_sources import IMPORT_ECONOMICS_SOURCES # noqa: E402
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="Import economics data into systems.db")
|
||
parser.add_argument("--db", default=str(DB_PATH), help="Path to systems.db")
|
||
parser.add_argument("--dry-run", action="store_true", help="Validate without writing")
|
||
parser.add_argument(
|
||
"--strict-specialization", action="store_true",
|
||
help="Treat D-237 completeness gates (V-SES-02, V-FAC-01) as hard errors. "
|
||
"Off by default until the #1014 fallback and #1016 content pass land.",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
db_path = Path(args.db)
|
||
if not db_path.exists():
|
||
print(f"error: {db_path} not found", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
print("\n Economics Import Pipeline")
|
||
print(f" DB: {db_path}")
|
||
if args.dry_run:
|
||
print(" Mode: DRY RUN")
|
||
print()
|
||
|
||
# Load wiki corps before opening DB — allows early exit on parse failures
|
||
print(" Loading wiki corporations...")
|
||
wiki_corps = corporations.load_wiki_corps()
|
||
print(f" {len(wiki_corps)} corporation files parsed")
|
||
|
||
# Regenerate generated_brands.toml via the Rust binary before the Python
|
||
# import reads it. Single pipeline, single stamp — resolves review T2/H3
|
||
# ("on-behalf stamping" coupling) by folding brand generation into
|
||
# import_economics' flow rather than having the caller (Makefile / user)
|
||
# remember to run it first. Skipped on --dry-run to avoid a disk
|
||
# side-effect during validation.
|
||
if not args.dry_run:
|
||
try:
|
||
brands.regenerate_brands()
|
||
except ImportAborted:
|
||
sys.exit(1)
|
||
|
||
conn = db.connect(db_path)
|
||
|
||
# The clear-then-reimport cycle below runs as a single explicit
|
||
# transaction. Any crash, validation error, or KeyboardInterrupt
|
||
# between the first DELETE and the final commit rolls everything
|
||
# back — the DB never ends up half-cleared with stale rows in some
|
||
# tables and empty rows in others. On success we commit exactly
|
||
# once, immediately after structural validation passes.
|
||
conn.execute("BEGIN")
|
||
try:
|
||
# 1. Migrate schema (idempotent, inside the tx so a crash here
|
||
# leaves no half-applied ALTER TABLE.)
|
||
print(" [1/10] Schema migration...")
|
||
migration.apply_schema_migrations(conn)
|
||
# Atlas index tables: apply canonical DDL + empty geometry (D-223, #951)
|
||
atlas.ensure_atlas_index_schema(conn, args.dry_run)
|
||
print(" tables and columns ready")
|
||
|
||
# Clear economics tables in FK-safe order (children before parents)
|
||
if not args.dry_run:
|
||
db.clear_economics_tables(conn)
|
||
|
||
# 2. Gate links
|
||
print(" [2/10] Importing gate links...")
|
||
n_links = economy.import_gate_links(conn, args.dry_run)
|
||
print(f" {n_links} rows (bidirectional)")
|
||
|
||
# 3. Commodities
|
||
print(" [3/10] Importing commodities...")
|
||
n_commodities = economy.import_commodities(conn, args.dry_run)
|
||
print(f" {n_commodities} commodities")
|
||
|
||
# 4. Production chains
|
||
print(" [4/10] Importing production chains...")
|
||
n_chains, n_inputs = economy.import_chains(conn, args.dry_run)
|
||
print(f" {n_chains} chains, {n_inputs} inputs")
|
||
|
||
# 4b. System specialization (D-237 authored layer) — after commodities
|
||
# (FK) and before currency zones. UPSERTs onto pre-existing
|
||
# system_economy / system_factions rows; unauthored systems stay NULL.
|
||
print(" [4b/10] Importing system specialization (D-237)...")
|
||
spec = specialization.import_system_specialization(
|
||
conn, args.dry_run, strict=args.strict_specialization
|
||
)
|
||
print(
|
||
f" vocab {spec['vocab']} | economic {spec['economic']} | "
|
||
f"cultural {spec['cultural']} | faction {spec['faction']}"
|
||
)
|
||
if spec["missing_economy_row"]:
|
||
print(
|
||
f" WARNING: {len(spec['missing_economy_row'])} authored "
|
||
f"system(s) lack a system_economy row (values dropped): "
|
||
f"{spec['missing_economy_row']}"
|
||
)
|
||
if spec["missing_faction_row"]:
|
||
print(
|
||
f" WARNING: {len(spec['missing_faction_row'])} authored "
|
||
f"system(s) lack a system_factions row (faction dropped): "
|
||
f"{spec['missing_faction_row']}"
|
||
)
|
||
|
||
# 5. Currency zones
|
||
print(" [5/10] Setting currency zones...")
|
||
zones = economy.set_currency_zones(conn, args.dry_run)
|
||
for zone, count in sorted(zones.items()):
|
||
print(f" {zone}: {count}")
|
||
|
||
# 6. Gate energy connectivity (D-186) — must run after currency zones
|
||
print(" [6/10] Setting gate energy connectivity...")
|
||
energy = economy.set_gate_energy(conn, args.dry_run)
|
||
for label, count in sorted(energy.items()):
|
||
print(f" {label}: {count}")
|
||
|
||
# 7. Sync corporations from wiki (D-182: hard error on name divergence)
|
||
print(" [7/10] Syncing corporations...")
|
||
corp_errors = corporations.sync_corporations(conn, wiki_corps, args.dry_run)
|
||
if corp_errors:
|
||
print(f" ERRORS ({len(corp_errors)}) — name divergence detected (D-182):")
|
||
for e in corp_errors:
|
||
print(f" - {e}")
|
||
print(" Fix: update wiki title or DB proper_name to match, then re-run.")
|
||
raise ImportAborted()
|
||
n_db_corps = conn.execute("SELECT COUNT(*) FROM corporations").fetchone()[0]
|
||
print(f" {n_db_corps} corporations in DB ({len(wiki_corps)} from wiki)")
|
||
|
||
# 8. Corp presence from wiki headquarters data
|
||
print(" [8/10] Importing corp presence...")
|
||
commodity_ids = {
|
||
r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()
|
||
}
|
||
n_presence = corporations.import_corp_presence(
|
||
conn, wiki_corps, commodity_ids, args.dry_run
|
||
)
|
||
print(f" {n_presence} corp_presence rows")
|
||
|
||
# 9. Brand products and inputs (D-189, #827)
|
||
print(" [9/10] Importing brand products and inputs...")
|
||
n_brands, n_brand_inputs = brands.import_brands(conn, args.dry_run)
|
||
print(f" {n_brands} brand_products, {n_brand_inputs} brand_inputs")
|
||
|
||
# 10. System fiscal parameters (D-189 section 6)
|
||
print(" [10/13] Populating system_fiscal...")
|
||
n_fiscal = economy.import_system_fiscal(conn, args.dry_run)
|
||
print(f" {n_fiscal} system_fiscal rows")
|
||
|
||
# 11. body_radius_km fallback from planet_class (D-204, #910)
|
||
print(" [11/13] Populating body_radius_km fallback...")
|
||
n_radius = bodies.populate_body_radius_km(conn, args.dry_run)
|
||
print(f" {n_radius} bodies updated")
|
||
|
||
# 12. atlas_city_names from wiki markers.json (D-207, #908)
|
||
print(" [12/13] Populating atlas_city_names from wiki content...")
|
||
n_cities = atlas.populate_atlas_city_names(conn, args.dry_run)
|
||
print(f" {n_cities} city name rows")
|
||
|
||
# 13. atlas_city_names corp HQ cross-reference (D-207, #909)
|
||
print(" [13/13] Cross-referencing corp HQ cities into atlas_city_names...")
|
||
n_updated, n_inserted = atlas.populate_atlas_city_names_corps(conn, args.dry_run)
|
||
print(f" {n_updated} rows updated, {n_inserted} reserved rows inserted")
|
||
|
||
# 14. Architecture-flavor trait templates (D-232, #993). Catalog first,
|
||
# then sparse per-body hero bias (FK -> trait_templates + bodies).
|
||
print(" [14/15] Baking trait_templates catalog (D-232)...")
|
||
n_templates = traits.populate_trait_templates(conn, args.dry_run)
|
||
print(f" {n_templates} trait templates")
|
||
print(" [15/15] Baking atlas_body_trait_bias hero pins (D-232)...")
|
||
n_bias = traits.populate_atlas_body_trait_bias(conn, args.dry_run)
|
||
print(f" {n_bias} body trait-bias rows")
|
||
|
||
# 16. axial_tilt_deg from body-def frontmatter (T-1024, D-239 §2)
|
||
print(" [16/16] Populating axial_tilt_deg from body-def frontmatter...")
|
||
n_tilt = bodies.populate_axial_tilt_deg(conn, args.dry_run)
|
||
print(f" {n_tilt} bodies updated with axial_tilt_deg")
|
||
|
||
# Validate structural integrity (FK, chain refs, chain completeness).
|
||
# These errors indicate broken imported data — do NOT commit.
|
||
print("\n Validating structural integrity...")
|
||
struct_errors = validators.validate(conn)
|
||
struct_errors.extend(brands.validate_brands(conn))
|
||
if struct_errors:
|
||
print(f" STRUCTURAL ERRORS ({len(struct_errors)}) — rolling back:")
|
||
for e in struct_errors:
|
||
print(f" - {e}")
|
||
raise ImportAborted()
|
||
print(" FK integrity, chain completeness, and brand layer (V-B01–V-B06) OK")
|
||
|
||
# Commit all imported data (corps, presence, etc.) before coverage check.
|
||
# Coverage validation is a Phase 2 gate (D-175) — data should be persisted
|
||
# so tools can query it and report gaps clearly.
|
||
if not args.dry_run:
|
||
conn.commit()
|
||
print(" Data committed.")
|
||
else:
|
||
# Dry-run: leave the transaction open so the coverage check below
|
||
# can still SELECT against the in-memory imported data. The
|
||
# transaction is discarded when conn.close() runs on exit.
|
||
print(" Dry run — no changes written.")
|
||
except ImportAborted:
|
||
conn.rollback()
|
||
conn.close()
|
||
sys.exit(1)
|
||
except BaseException:
|
||
# Any other exception (KeyboardInterrupt, MemoryError, DB error,
|
||
# programmer error) triggers a rollback so the DB is never left in
|
||
# a half-imported state. Re-raise so the user sees the traceback.
|
||
conn.rollback()
|
||
conn.close()
|
||
raise
|
||
|
||
# Stamp generator metadata (#855, #856): record source SHAs so the
|
||
# pre-push hook can detect stale DB snapshots. Written BEFORE the
|
||
# coverage gate — the stamp records generator execution (code version),
|
||
# not data completeness. Coverage gaps (#860) are pre-existing data
|
||
# issues and must not prevent the stamp from landing.
|
||
if not args.dry_run:
|
||
try:
|
||
stamp.write_stamp(conn, "import_economics", *IMPORT_ECONOMICS_SOURCES)
|
||
conn.commit()
|
||
print(" Stamped: import_economics (covers brand pipeline Rust sources)")
|
||
except Exception as exc: # noqa: BLE001
|
||
print(f" WARNING: failed to write generator stamp: {exc}", file=sys.stderr)
|
||
|
||
# Validate coverage (hard errors per D-175, but after commit so data is usable).
|
||
print("\n Validating coverage (D-175 Phase 2 gate)...")
|
||
coverage_errors: list[str] = []
|
||
commodity_ids_for_coverage = {
|
||
r[0] for r in conn.execute("SELECT commodity_id FROM commodities").fetchall()
|
||
}
|
||
coverage_errors.extend(
|
||
validators.validate_commodity_coverage(conn, wiki_corps, commodity_ids_for_coverage)
|
||
)
|
||
coverage_errors.extend(validators.validate_system_coverage(conn, wiki_corps))
|
||
|
||
if coverage_errors:
|
||
print(f" COVERAGE ERRORS ({len(coverage_errors)}) — Phase 2 gate not met:")
|
||
for e in coverage_errors:
|
||
print(f" - {e}")
|
||
print("\n Data committed but Phase 2 gate is NOT met. "
|
||
"Add corporations to meet coverage thresholds and re-run.")
|
||
conn.close()
|
||
sys.exit(2) # exit 2 = coverage warning (data+stamp committed); exit 1 = real error
|
||
else:
|
||
print(" All coverage thresholds met — Phase 2 gate PASSED.")
|
||
|
||
conn.close()
|
||
|
||
print(f"\n Done: {n_links} gate_links, {n_commodities} commodities, "
|
||
f"{n_chains} chains, {n_inputs} inputs, {n_presence} corp_presence, "
|
||
f"{n_brands} brand_products, {n_brand_inputs} brand_inputs, "
|
||
f"{n_fiscal} system_fiscal\n")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|