refactor(db): split import_economics.py; single generator-source registry (T-1067)
import_economics.py 2,620 → 309 lines — a thin orchestrator keeping the exact CLI, single-transaction/rollback contract, and exit codes. The 16 import steps, MIGRATION_SQL, brands shell-out, validators, and stamp write now live in tooling/economy-db/economy_import/ (db, migration, economy, corporations, brands, bodies, atlas, specialization, traits, validators, stamp, paths, errors). Full type hints throughout. tooling/generator_sources.py replaces the triplicated source registry (importer / stamp checker / pr-process watch list — the skill now derives its list via --list). The registry stamps itself, and economy_import/ modules are globbed fail-closed, so a future module is stamped the moment it exists — closing the silently-weakened-stamp failure mode. Rider: connector config helpers centralized in tooling/db/common.py. Byte-identical behavior proven: full-import table dump diff EMPTY over 107,843 lines / 37 tables (volatile timestamp fields excluded); dry-run output parity; generated_brands.toml sha unchanged. make test-tooling PASS; ruff clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
generator_sources — single source of truth for systems.db generator source sets.
|
||||
|
||||
Each generator that writes a stamp row to systems.db's ``meta`` table records the
|
||||
SHA-1 of the concatenated bytes of its source files (sorted by path). This module
|
||||
is the ONE place that defines which files belong to each generator's stamped set
|
||||
(T-1067). It replaces the formerly triplicated lists in:
|
||||
|
||||
- ``IMPORT_ECONOMICS_SOURCES`` in tooling/economy-db/import_economics.py
|
||||
- ``GENERATOR_SOURCES`` in tooling/check-systems-db-stamp
|
||||
- the hardcoded watch list in .claude/skills/pr-process/SKILL.md
|
||||
(now derived via ``python3 tooling/generator_sources.py --list``)
|
||||
|
||||
Consumers:
|
||||
- tooling/economy-db (the importer) — imports ``IMPORT_ECONOMICS_SOURCES`` and
|
||||
``file_sha1`` to write the stamp after a successful run.
|
||||
- tooling/check-systems-db-stamp — imports ``GENERATOR_SOURCES`` and
|
||||
``file_sha1`` to verify the stamp before a push.
|
||||
- /pr-process (skill) — shells out to the ``--list`` CLI for its watch list.
|
||||
|
||||
This file is itself a member of the stamped source set: a registry change (adding
|
||||
or removing a source) must flip the stamp and force ``make regen-db``, otherwise
|
||||
edits to the list would be invisible to staleness detection.
|
||||
|
||||
The economy_import package modules are collected by glob rather than listed by
|
||||
hand, so a module added during a future split is stamped automatically — a
|
||||
hand-maintained list that misses a module silently weakens staleness detection.
|
||||
|
||||
Usage:
|
||||
python3 tooling/generator_sources.py --list [generator_name]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# import_economics source set
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Entrypoint + split module package (T-1067).
|
||||
IMPORT_ECONOMICS_ENTRYPOINT: Path = (
|
||||
REPO_ROOT / "tooling" / "economy-db" / "import_economics.py"
|
||||
)
|
||||
ECONOMY_IMPORT_PACKAGE_DIR: Path = (
|
||||
REPO_ROOT / "tooling" / "economy-db" / "economy_import"
|
||||
)
|
||||
|
||||
# Rust sources for the generate_brands subroutine. import_economics shells out
|
||||
# to tooling/generate-brands as part of its normal flow (see economy_import/
|
||||
# brands.py), so all of these contribute to its effective source SHA: any change
|
||||
# to them must invalidate the meta stamp even though Python hasn't changed.
|
||||
GENERATE_BRANDS_RS: Path = (
|
||||
REPO_ROOT / "server" / "src" / "bin" / "generate_brands" / "main.rs"
|
||||
)
|
||||
GENERATE_BRANDS_NAMES_RS: Path = (
|
||||
REPO_ROOT / "server" / "src" / "bin" / "generate_brands" / "names.rs"
|
||||
)
|
||||
# Shared surname corpus extracted from the two names.rs copies (T-1064).
|
||||
GENERATE_BRANDS_SURNAMES_RS: Path = (
|
||||
REPO_ROOT / "server" / "src" / "bin" / "shared" / "surname_corpus.rs"
|
||||
)
|
||||
GENERATE_BRANDS_WRAPPER: Path = REPO_ROOT / "tooling" / "generate-brands"
|
||||
|
||||
# D-237 authored specialization layer: these data TOMLs feed the DB, so a change
|
||||
# to either must flip the stamp and force a regen (#1013).
|
||||
SPECIALIZATION_VOCAB_TOML: Path = (
|
||||
REPO_ROOT / "wiki" / "economics" / "specialization_vocabulary.toml"
|
||||
)
|
||||
SYSTEM_SPECIALIZATION_TOML: Path = (
|
||||
REPO_ROOT / "wiki" / "economics" / "system_specialization.toml"
|
||||
)
|
||||
# D-232 architecture-flavor trait-template catalog + sparse hero bias (#993).
|
||||
ARCHITECTURE_TRAIT_CATALOG_TOML: Path = (
|
||||
REPO_ROOT / "wiki" / "economics" / "architecture_trait_catalog.toml"
|
||||
)
|
||||
ARCHITECTURE_TRAIT_BIAS_TOML: Path = (
|
||||
REPO_ROOT / "wiki" / "economics" / "architecture_trait_bias.toml"
|
||||
)
|
||||
|
||||
|
||||
def _economy_import_modules() -> tuple[Path, ...]:
|
||||
"""All Python modules of the economy_import package, collected by glob.
|
||||
|
||||
Globbing (rather than hand-listing) guarantees a future module is stamped
|
||||
the moment it exists. Fail closed if the package is missing — an empty
|
||||
set here would silently weaken staleness detection.
|
||||
"""
|
||||
modules = tuple(sorted(ECONOMY_IMPORT_PACKAGE_DIR.glob("*.py")))
|
||||
if not modules:
|
||||
raise RuntimeError(
|
||||
f"economy_import package not found at {ECONOMY_IMPORT_PACKAGE_DIR} — "
|
||||
"the import_economics stamp source set would be incomplete"
|
||||
)
|
||||
return modules
|
||||
|
||||
|
||||
# Canonical source set for import_economics' meta stamp. Covers the Python
|
||||
# entrypoint and its module package, the Rust binary it invokes (generate_brands
|
||||
# main.rs + names.rs + surname corpus + wrapper script), the shared schema
|
||||
# version constant, the authored data TOMLs, and this registry itself.
|
||||
IMPORT_ECONOMICS_SOURCES: tuple[Path, ...] = (
|
||||
Path(__file__).resolve(), # registry changes must stale the stamp
|
||||
IMPORT_ECONOMICS_ENTRYPOINT,
|
||||
*_economy_import_modules(),
|
||||
GENERATE_BRANDS_RS,
|
||||
GENERATE_BRANDS_NAMES_RS,
|
||||
GENERATE_BRANDS_SURNAMES_RS,
|
||||
GENERATE_BRANDS_WRAPPER,
|
||||
REPO_ROOT / "tooling" / "schema_version.py",
|
||||
SPECIALIZATION_VOCAB_TOML,
|
||||
SYSTEM_SPECIALIZATION_TOML,
|
||||
ARCHITECTURE_TRAIT_CATALOG_TOML,
|
||||
ARCHITECTURE_TRAIT_BIAS_TOML,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Maps generator_name (as stored in meta.generator_name) to the source file(s)
|
||||
# whose SHA is stamped. The SHA is computed as SHA-1 of the concatenated bytes
|
||||
# of all files in sorted order.
|
||||
#
|
||||
# The atlas geometry generator (generate_atlas) was retired in #951 (D-223).
|
||||
# import_economics is now the sole regen-db generator that writes systems.db; it
|
||||
# owns the atlas index (names-only pool + empty geometry tables). The surviving
|
||||
# planet-gen importers (import_heightmaps, import_province_boundaries) are
|
||||
# one-time build imports baked into the committed DB, not part of regen-db, so
|
||||
# they are intentionally not stamped here.
|
||||
GENERATOR_SOURCES: dict[str, tuple[Path, ...]] = {
|
||||
"import_economics": IMPORT_ECONOMICS_SOURCES,
|
||||
}
|
||||
|
||||
|
||||
def file_sha1(*paths: Path) -> str:
|
||||
"""SHA-1 of concatenated file contents (sorted paths).
|
||||
|
||||
Missing files raise FileNotFoundError rather than silently contributing
|
||||
an empty-string hash (H2): a ghost SHA could mask real breakage when
|
||||
stored and current SHAs converge on the empty-bytes digest.
|
||||
"""
|
||||
h = hashlib.sha1()
|
||||
for p in sorted(paths):
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"generator source not found: {p}")
|
||||
h.update(p.read_bytes())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Single source of truth for systems.db generator source sets"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
action="store_true",
|
||||
help="Print the stamped source files, one repo-relative path per line",
|
||||
)
|
||||
parser.add_argument(
|
||||
"generator",
|
||||
nargs="?",
|
||||
default=None,
|
||||
help="Restrict --list to one generator (default: all)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.list:
|
||||
parser.print_help()
|
||||
sys.exit(2)
|
||||
|
||||
if args.generator is not None:
|
||||
if args.generator not in GENERATOR_SOURCES:
|
||||
print(
|
||||
f"error: unknown generator '{args.generator}' — "
|
||||
f"known: {sorted(GENERATOR_SOURCES)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
names = [args.generator]
|
||||
else:
|
||||
names = sorted(GENERATOR_SOURCES)
|
||||
|
||||
seen: set[Path] = set()
|
||||
for name in names:
|
||||
seen.update(GENERATOR_SOURCES[name])
|
||||
for path in sorted(seen):
|
||||
print(path.relative_to(REPO_ROOT).as_posix())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user