odysseus/docker/searxng/validate_image.py

"""Offline incident-chain checks and loopback boot smoke for the derived image."""

import hashlib
import asyncio
import json
import os
from pathlib import Path
import re
import signal
import ssl
import subprocess
import sys
import tempfile
import time
import urllib.request

import certifi

from repair_trust import check_bundle


def check_chains(baseline=False):
    incident = json.loads(Path(__file__).with_name("incident.json").read_text())
    system = Path(os.environ["SSL_CERT_FILE"])
    capath = os.environ["SSL_CERT_DIR"]
    stores = [["-CAfile", str(system), "-no-CApath"],
              ["-CAfile", certifi.where(), "-no-CApath"],
              ["-no-CAfile", "-CApath", capath]]
    if not baseline:
        check_bundle(system)
        check_bundle(Path(certifi.where()))
    with tempfile.TemporaryDirectory() as directory:
        directory = Path(directory)
        for fingerprint, pem in incident["certificates"].items():
            if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != fingerprint:
                raise ValueError(f"incident fixture fingerprint mismatch: {fingerprint}")
            (directory / fingerprint).write_text(pem)
        positive = directory / "affected.pem"
        positive.write_text("\n".join(incident["certificates"][fp] for fp in incident["roots"]))
        for hostname, chain in incident["chains"].items():
            untrusted = directory / "untrusted.pem"
            untrusted.write_text("\n".join(incident["certificates"][fp] for fp in chain[1:]))
            base = ["openssl", "verify", "-no-CAstore", "-attime", str(incident["verification_time"]),
                    "-purpose", "sslserver", "-verify_hostname", hostname,
                    "-x509_strict", "-partial_chain", "-trusted_first", "-untrusted", str(untrusted)]
            expectations = [(s, baseline and (index != 1 or "-aaa-" not in hostname))
                            for index, s in enumerate(stores)]
            for selection, expected in expectations + [
                    (["-CAfile", str(positive), "-no-CApath"], True)]:
                result = subprocess.run(base + selection + [str(directory / chain[0])],
                                        capture_output=True, text=True)
                if (result.returncode == 0) != expected:
                    raise RuntimeError(f"unexpected chain result: {hostname}: {selection}: {result.stderr}")
                # Accept issuer lookup, self-signed and explicitly untrusted
                # certificate errors, rather than tool or fixture failures.
                if not expected and not re.search(r"error (?:2|18|19|20|21|27) at \d+ depth lookup", result.stderr):
                    raise RuntimeError(f"not a certificate trust rejection: {hostname}: {result.stderr}")
    if baseline:
        total = len(incident["chains"])
        certifi_accepted = sum("-aaa-" not in host for host in incident["chains"])
        print(f"Affected base reproduced: system PEM and directory accept {total}/{total}; certifi accepts {certifi_accepted}/{total}")
    else:
        print(f"{len(incident['chains'])}/{len(incident['chains'])} incident chains rejected by system PEM, hashed directory and certifi; affected controls accepted")


def check_boot(live=False):
    with tempfile.TemporaryDirectory() as directory:
        settings = Path(directory) / "settings.yml"
        settings.write_text("use_default_settings: true\nserver:\n  secret_key: offline-build-validation-only\n  limiter: false\nsearch:\n  formats: [html, json]\n")
        cache = Path(directory) / "cache"
        cache.mkdir()
        env = dict(os.environ, SEARXNG_SETTINGS_PATH=str(settings),
                   __SEARXNG_CONFIG_PATH=directory, __SEARXNG_DATA_PATH=str(cache),
                   GRANIAN_HOST="127.0.0.1", GRANIAN_PORT="18888")
        # Do not expose a port or issue an outbound search during offline builds.
        with tempfile.TemporaryFile() as log:
            process = subprocess.Popen(["/usr/local/searxng/entrypoint.sh"],
                                       cwd="/usr/local/searxng", env=env, stdout=log,
                                       stderr=subprocess.STDOUT, start_new_session=True)
            try:
                opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
                healthy = False
                for _ in range(60):
                    if process.poll() is not None:
                        break
                    try:
                        with opener.open("http://127.0.0.1:18888/healthz", timeout=1) as response:
                            healthy = response.status == 200
                    except OSError:
                        pass
                    if healthy:
                        break
                    time.sleep(0.5)
                if not healthy:
                    log.seek(0)
                    raise RuntimeError("SearXNG failed boot smoke: " + log.read().decode(errors="replace")[-6000:])
                print("SearXNG loopback healthcheck passed")
                # Its root entrypoint regenerates system trust. Repeat the
                # behavioral checks after that real startup path completes.
                check_running(live=live, port=18888)
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGTERM)
                try:
                    process.wait(timeout=10)
                except subprocess.TimeoutExpired:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait()


def check_running(live=False, port=None):
    """Verify an existing service without repairing it or starting another one."""
    port = int(os.environ.get("GRANIAN_PORT", "8080")) if port is None else port
    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
    with opener.open(f"http://127.0.0.1:{port}/healthz", timeout=5) as response:
        if response.status != 200:
            raise RuntimeError("SearXNG running-service healthcheck failed")
    check_chains()
    for path in Path(os.environ["SSL_CERT_DIR"]).iterdir():
        if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name):
            check_bundle(path)
    if live:
        asyncio.run(check_https())
        with opener.open(f"http://127.0.0.1:{port}/search?q=python+documentation&format=json", timeout=90) as search:
            payload = json.load(search)
        results = payload.get("results", [])
        if not results:
            raise RuntimeError("live search returned no results")
        print(f"SearXNG live JSON search passed: {len(results)} results")


async def check_https():
    # Exercise the actual pinned SearXNG client factory with its default CA
    # selection, rather than a different HTTP library's trust configuration.
    sys.path.insert(0, "/usr/local/searxng")
    from curl_cffi.requests.exceptions import RequestException
    from searx.network.client import new_client

    client = new_client(enable_http=False, verify=True, enable_http2=True,
                        enable_http3=False, max_connections=2, proxies={},
                        local_address=None, max_redirects=3)
    try:
        response = await client.get("https://pypi.org/simple/certifi/", timeout=30)
        if response.status_code != 200:
            raise RuntimeError(f"ordinary HTTPS failed: {response.status_code}")
        print("SearXNG actual curl client ordinary HTTPS passed with verification enabled")
        incident = json.loads(Path(__file__).with_name("incident.json").read_text())
        for hostname in incident["chains"]:
            try:
                await client.get(f"https://{hostname}/", timeout=30, allow_redirects=False)
            except RequestException as exc:
                # libcurl code 60 is peer certificate verification failure.
                # DNS failures, timeouts and handshake EOF do not prove distrust.
                if exc.code != 60:
                    raise RuntimeError(f"inconclusive incident TLS failure: {hostname}: {exc}") from exc
            else:
                raise RuntimeError(f"incident HTTPS unexpectedly accepted: {hostname}")
        print(f"SearXNG actual curl client rejects {len(incident['chains'])}/{len(incident['chains'])} incident hosts with certificate errors after startup")
    finally:
        await client.aclose()


if __name__ == "__main__":
    baseline = "--baseline" in sys.argv
    live = "--live" in sys.argv
    if "--running" in sys.argv:
        if baseline:
            raise ValueError("--running cannot validate an affected baseline")
        check_running(live=live)
    else:
        check_chains(baseline=baseline)
        if not baseline:
            check_boot(live=live)