settled-reach/tooling/test_lazy_domains.py

#!/usr/bin/env python3
"""Units for lazy domain registration (T-1260).

`reach --help` must list every domain WITHOUT importing any of them. That is not
tidiness: an eager entrypoint pays for every domain's imports on every
invocation, and the expensive ones are already in this tree — scipy.ndimage
alone is 275 ms. The cost of `--help` has to stay flat as the registry grows
from one domain to a dozen.

The trap this guards is specific and quiet. `TyperGroup.format_commands` loops
over `list_commands` calling `get_command` on each, purely to read a short help
string off the loaded command. With lazy loading underneath, that imports every
domain in the registry to render `--help` — while looking entirely correct.
Nothing about the output changes; only the import graph does. So the assertion
has to be on `sys.modules`, not on what `--help` prints.

Three properties, and the third is what stops the first two passing vacuously:

  1. After `--help`, nothing under `tooling.domains` is imported.
  2. After `--help`, no heavy third-party module is imported.
  3. POSITIVE CONTROL: actually invoking a domain DOES import its service. If
     this fails, properties 1 and 2 are meaningless — they would also pass for a
     CLI whose lazy loader is broken and never imports anything at all.

Run: python3 tooling/test_lazy_domains.py
"""

import json
import subprocess
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent

# Modules that must never be dragged in by `--help`. pydantic is on the list
# because domain schemas use it: it must load with the domain, not with the CLI.
HEAVY = ("rich", "pygments", "numpy", "scipy", "pydantic", "PIL")

_HELP_PROBE = """
import contextlib, io, json, sys
from tooling.main import cli, DOMAINS

buf = io.StringIO()
try:
    with contextlib.redirect_stdout(buf):
        cli(args=["--help"])
except SystemExit:
    pass

print(json.dumps({
    "help": buf.getvalue(),
    "domain_modules": sorted(
        m for m in sys.modules
        if m == "tooling.domains" or m.startswith("tooling.domains.")
    ),
    "heavy": sorted(m for m in %(heavy)r if m in sys.modules),
    "declared": sorted(DOMAINS),
}))
"""

_INVOKE_PROBE = """
import contextlib, io, json, sys
from tooling.main import cli

buf = io.StringIO()
try:
    with contextlib.redirect_stdout(buf):
        cli(args=["check", "client-version"])
except SystemExit:
    pass

print(json.dumps({
    "domain_modules": sorted(
        m for m in sys.modules
        if m == "tooling.domains" or m.startswith("tooling.domains.")
    ),
}))
"""


def _probe(source: str) -> dict:
    """Run a snippet in a FRESH interpreter and return its JSON verdict.

    A subprocess, not an in-process import, because this test is entirely about
    a module graph — running it in the harness's own interpreter would inherit
    whatever the harness already imported and prove nothing.
    """
    result = subprocess.run(
        [sys.executable, "-c", source],
        capture_output=True,
        text=True,
        cwd=REPO_ROOT,
    )
    if result.returncode != 0:
        raise SystemExit(f"probe failed (exit {result.returncode}):\n{result.stderr}")
    return json.loads(result.stdout)


def main() -> int:
    failures = []

    helped = _probe(_HELP_PROBE % {"heavy": HEAVY})

    # Guard against a vacuous pass: an empty registry would satisfy every
    # assertion below while proving nothing at all.
    if not helped["declared"]:
        failures.append("DOMAINS registry is empty — every assertion here would pass vacuously")

    for name in helped["declared"]:
        if name not in helped["help"]:
            failures.append(f"`--help` does not list the declared domain {name!r}")

    if helped["domain_modules"]:
        failures.append(
            "`--help` imported domain modules, so registration is not lazy: "
            + ", ".join(helped["domain_modules"])
            + "\n  The usual cause is format_commands falling back to the base "
            "implementation,\n  which calls get_command on every subcommand to read its short help."
        )

    if helped["heavy"]:
        failures.append("`--help` imported heavy modules: " + ", ".join(helped["heavy"]))

    # Positive control. Without this, the assertions above would pass for a CLI
    # that is simply broken and imports nothing ever.
    invoked = _probe(_INVOKE_PROBE)
    if "tooling.domains.check.service" not in invoked["domain_modules"]:
        failures.append(
            "positive control FAILED: invoking `check client-version` did not import "
            "tooling.domains.check.service, so the lazy-import assertions above prove nothing"
        )

    if failures:
        print("test_lazy_domains: FAIL", file=sys.stderr)
        for failure in failures:
            print(f"  - {failure}", file=sys.stderr)
        return 1

    print(
        f"test_lazy_domains: OK — {len(helped['declared'])} domain(s) listed, "
        "none imported; positive control confirms the loader works"
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())