From df1c2c8b192f693fdbcad5020df4b1545db99910 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:27:05 +0000 Subject: [PATCH 1/5] fix: refresh outbound HTTPS trust and installer bootstrap --- .github/workflows/ci.yml | 15 ++- .github/workflows/dependency-review.yml | 4 + .github/workflows/docker-publish.yml | 1 + .github/workflows/workflow-security.yml | 4 + Dockerfile | 17 ++- build-windows-portable.ps1 | 3 +- docs/tls-trust.md | 34 +++++ launch-windows.ps1 | 3 +- requirements-bootstrap.txt | 8 ++ requirements.txt | 2 + scripts/check_ca_trust.py | 96 ++++++++++++++ start-macos.sh | 4 +- tests/fixtures/ca-trust/README.md | 5 + tests/test_ca_trust.py | 168 ++++++++++++++++++++++++ 14 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 docs/tls-trust.md create mode 100644 requirements-bootstrap.txt create mode 100644 scripts/check_ca_trust.py create mode 100644 tests/fixtures/ca-trust/README.md create mode 100644 tests/test_ca_trust.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42f0651f0..29f303ca4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: "20" + node-version: "22.23.4" - run: npm ci --ignore-scripts --no-audit --no-fund - name: Check frontend architecture run: node scripts/check_frontend.mjs @@ -160,15 +160,18 @@ jobs: - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 if: steps.docs-check.outputs.docs_only != 'true' with: + version: "0.13.4" enable-cache: true cache-dependency-glob: requirements.txt prune-cache: false - - run: uv pip install --system -r requirements.txt + - run: | + uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt + uv pip install --system -r requirements.txt if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 if: steps.docs-check.outputs.docs_only != 'true' with: - node-version: "20" + node-version: "22.23.4" cache: npm - run: npm ci if: steps.docs-check.outputs.docs_only != 'true' @@ -190,10 +193,14 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap + sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates command -v ffmpeg ffmpeg -version | head -n 1 + - name: Validate Python and system CA trust + if: steps.docs-check.outputs.docs_only != 'true' + run: python scripts/check_ca_trust.py --system --python + - name: Establish functional bubblewrap containment if: steps.docs-check.outputs.docs_only != 'true' shell: bash diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 0e4f9a1f1..e78bba158 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -70,5 +70,9 @@ jobs: - name: Run pip-audit on requirements run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install pip-audit==2.10.0 pip-audit -r requirements.txt -r requirements-optional.txt --strict diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index da41c7140..2cb8221ec 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,6 +65,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . + pull: true platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.arch }} diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index b00cd03a4..18a197eb0 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -76,5 +76,9 @@ jobs: - name: Run zizmor run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install zizmor==1.25.2 zizmor --offline --min-severity=low .github/workflows/ diff --git a/Dockerfile b/Dockerfile index d6e7ca48f..2e49c6ed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,13 @@ # the final image / Cookbook never has to compile the broken sdists. See # docker/build-realesrgan-wheels.sh for the full rationale. FROM python:3.14-slim AS realesrgan-wheels -RUN apt-get update && apt-get install -y --no-install-recommends curl \ +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels @@ -28,6 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cmake \ curl \ + ca-certificates \ git \ nodejs \ npm \ @@ -48,6 +54,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libmagic1 \ && rm -rf /var/lib/apt/lists/* +# Refresh both Python trust bundles before installing application dependencies. +# System roots also serve web fetches, webhooks, curl, and Node's OpenSSL mode. +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python +ENV NODE_OPTIONS=--use-openssl-ca + # Private browser automation wrapper used by the native `private_browser` tool. # Chromium is installed above, so agent-browser can drive the existing browser # binary without paying `npx` startup/install overhead on each tool call. diff --git a/build-windows-portable.ps1 b/build-windows-portable.ps1 index 7da912f73..634ae8068 100644 --- a/build-windows-portable.ps1 +++ b/build-windows-portable.ps1 @@ -47,7 +47,8 @@ if (-not $pyExe) { Write-Host ("Using Python: " + $pyExe) Write-Step "Installing build dependencies" -& $pyExe -m pip install --upgrade pip --quiet +& $pyExe -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS build dependency bootstrap failed." } & $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." } diff --git a/docs/tls-trust.md b/docs/tls-trust.md new file mode 100644 index 000000000..3d91e08cd --- /dev/null +++ b/docs/tls-trust.md @@ -0,0 +1,34 @@ +# Outbound HTTPS trust + +Odysseus uses both Python package CA bundles and system trust. HTTPX uses certifi by default; the guarded web/PDF fetcher and webhook delivery explicitly use Python's system SSL context. A configured `LLM_CA_BUNDLE` adds certificates to system trust for model calls. Upgrading certifi alone cannot repair every outbound HTTPS path. + +For the October 2026 eMudhra incident, use certifi 2026.10.19 or newer, pip 26.2.2 or newer, and uv 0.13.4 or newer wherever uv is used. Pip has its own vendored certifi and also loads system roots. Update the operating system's certificate package separately. See the [Python security announcement](https://discuss.python.org/t/109576) and [incident details](https://emudhra.we-have-a-problem.com/). + +Before other pip installs, bootstrap the selected environment with the integrity-pinned wheels: + +```sh +python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +python -m pip install -r requirements.txt +``` + +The pinned bootstrap protects package integrity even when the old installer's certificate bundle is affected. It is not an operating-system trust-store update. Native Windows/macOS trust and command-line OpenSSL PEM stores may differ; follow the platform vendor's remediation for every store in use. Restart Odysseus and its subprocesses after updating trust. + +On Linux/OpenSSL deployments, inspect both the effective CA file and hashed CA directory, plus the two Python bundles: + +```sh +python scripts/check_ca_trust.py --system --python +``` + +Explicit PEM bundles can also be passed as positional arguments. The check rejects the incident's known root fingerprints and malformed/empty bundles. It does not model native OS intermediate revocation, custom TLS implementations, or arbitrary certificates added by an operator. + +Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if affected roots remain. Node uses the corrected system store through `--use-openssl-ca`. Rebuild with a fresh base and dependency layers when remediating an existing deployment: + +```sh +docker compose build --pull --no-cache odysseus +docker compose up -d odysseus +docker compose exec odysseus python scripts/check_ca_trust.py --system --python +``` + +SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. + +For behavioral validation, all twelve [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. diff --git a/launch-windows.ps1 b/launch-windows.ps1 index ab0e3542b..31c45a5fb 100644 --- a/launch-windows.ps1 +++ b/launch-windows.ps1 @@ -131,7 +131,8 @@ if (-not (Test-Path $venvPy)) { # 3. Install / update dependencies Write-Step "Installing dependencies (first run can take a few minutes)" -& $venvPy -m pip install --upgrade pip --quiet +& $venvPy -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS dependency bootstrap failed." } & $venvPy -m pip install -r requirements.txt if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed. Scroll up for the pip error." } diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt new file mode 100644 index 000000000..8b39dc69b --- /dev/null +++ b/requirements-bootstrap.txt @@ -0,0 +1,8 @@ +# Integrity-pinned bootstrap for the October 2026 eMudhra CA incident. +# Install with --require-hashes --only-binary=:all: --no-deps before other pip work. +# pip vendors its own certifi; updating the application bundle cannot update it. +# Direct universal wheels avoid index resolution through the old trust store. +pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl \ + --hash=sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70 +certifi @ https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl \ + --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b diff --git a/requirements.txt b/requirements.txt index 7fabaae93..4d794537d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ uvicorn python-multipart python-dotenv httpx +# HTTPS clients using certifi must distrust the compromised eMudhra CA paths. +certifi>=2026.10.19 httpcore>=1.0,<2.0 pydantic>=2.13.4 pydantic-settings>=2.14.1 diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py new file mode 100644 index 000000000..c675dc6f3 --- /dev/null +++ b/scripts/check_ca_trust.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Reject the eMudhra incident's root certificates in PEM trust stores. + +This checks certificate contents, including OpenSSL's lazily loaded CA directory. +It does not model native OS certificate revocation or arbitrary custom clients. +Incident fingerprints: https://emudhra.we-have-a-problem.com/roots.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import os +from pathlib import Path +import re +import ssl +import sys + + +BLOCKED_ROOTS = { + "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b": "emSign ECC Root CA - G3", + "d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4": "AAA Certificate Services", + "7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d": "emSign Root TLS CA - G3", + "f2edaaff3d9b701fb4d14c9f285f8c0bad90fc3bcc1fa07d47f4718edaba34a5": "emSign Root CS CA - G3", +} + + +def check_bundle(path: Path) -> None: + """Raise on an unreadable, empty, malformed, or affected CA bundle.""" + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cafile=str(path)) + certificates = context.get_ca_certs(binary_form=True) + if not certificates: + raise ValueError("no CA certificates") + blocked = { + BLOCKED_ROOTS[fingerprint] + for certificate in certificates + if (fingerprint := hashlib.sha256(certificate).hexdigest()) in BLOCKED_ROOTS + } + if blocked: + raise ValueError("affected roots: " + ", ".join(sorted(blocked))) + + +def system_paths() -> list[Path]: + """Inspect both the effective OpenSSL CA file and hashed directory entries.""" + defaults = ssl.get_default_verify_paths() + paths = [Path(defaults.cafile)] if defaults.cafile else [] + # get_default_verify_paths() can report None for a colon-separated + # SSL_CERT_DIR even though OpenSSL consumes each directory in that list. + capath = os.environ.get(defaults.openssl_capath_env, defaults.capath) + if capath: + for directory in filter(None, capath.split(os.pathsep)): + paths.extend( + path + for path in Path(directory).iterdir() + if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name) and path.is_file() + ) + if not paths: + raise ValueError("no effective OpenSSL PEM CA store found") + return paths + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundles", nargs="*", type=Path) + parser.add_argument("--system", action="store_true", help="check effective OpenSSL PEM stores") + parser.add_argument("--python", action="store_true", help="check certifi and pip's separate vendored bundle") + args = parser.parse_args(argv) + if not (args.bundles or args.system or args.python): + parser.error("select --system, --python, or explicit PEM bundle paths") + try: + paths = list(args.bundles) + if args.system: + paths.extend(system_paths()) + if args.python: + for name in ("certifi", "pip._vendor.certifi"): + paths.append(Path(importlib.import_module(name).where())) + paths = list(dict.fromkeys(path.resolve() for path in paths)) + errors = [] + for path in paths: + try: + check_bundle(path) + except (OSError, ValueError, ssl.SSLError) as exc: + errors.append(f"{path}: {exc}") + if errors: + raise ValueError("\n".join(errors)) + except (ImportError, OSError, ValueError, ssl.SSLError) as exc: + print(str(exc), file=sys.stderr) + return 1 + print(f"Validated {len(paths)} CA bundle file(s): no eMudhra incident roots") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/start-macos.sh b/start-macos.sh index 3e9048547..b9178792f 100755 --- a/start-macos.sh +++ b/start-macos.sh @@ -140,11 +140,11 @@ if [ ! -x "$VENV_PY" ] || ! "$VENV_PY" -m pip --version >/dev/null 2>&1; then echo "▶ Creating Python environment…" "$PY" -m venv venv fi -REQ_HASH="$(md5 -q requirements.txt 2>/dev/null || md5sum requirements.txt | cut -d' ' -f1)" +REQ_HASH="$("$VENV_PY" -c 'import hashlib, pathlib; print(hashlib.sha256(b"".join(pathlib.Path(p).read_bytes() for p in ("requirements.txt", "requirements-bootstrap.txt"))).hexdigest())')" REQ_HASH_FILE="venv/.requirements_hash" if [ ! -f "$REQ_HASH_FILE" ] || [ "$REQ_HASH" != "$(cat "$REQ_HASH_FILE" 2>/dev/null)" ]; then echo "▶ Installing Python packages (first run downloads a few — can take a few minutes)…" - "$VENV_PY" -m pip install --quiet --upgrade pip + "$VENV_PY" -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt # Not --quiet: this is the slow step, so show progress (and any real errors). "$VENV_PY" -m pip install -r requirements.txt echo "$REQ_HASH" > "$REQ_HASH_FILE" diff --git a/tests/fixtures/ca-trust/README.md b/tests/fixtures/ca-trust/README.md new file mode 100644 index 000000000..47e440ac1 --- /dev/null +++ b/tests/fixtures/ca-trust/README.md @@ -0,0 +1,5 @@ +# CA trust regression fixtures + +The public CA certificates embedded in `tests/test_ca_trust.py` exercise the October 2026 eMudhra trust-store check. They contain no private keys. `emsign-g3.pem` and `aaa.pem` are the incident's affected trust anchors; `emsign-g1.pem` is an unaffected emSign anchor that must remain usable. The test creates temporary PEM files from these public constants. The certificate fingerprints are the factual identity of each fixture. + +Sources: the [incident root list](https://emudhra.we-have-a-problem.com/roots.json) and the public Mozilla root certificates distributed in Debian's `ca-certificates` package. Behavioral chain verification should also cover the incident site's separate [twelve published server chains](https://emudhra.we-have-a-problem.com/chains.zip). diff --git a/tests/test_ca_trust.py b/tests/test_ca_trust.py new file mode 100644 index 000000000..b8e7b1bb0 --- /dev/null +++ b/tests/test_ca_trust.py @@ -0,0 +1,168 @@ +"""Offline regressions for incident roots, unaffected roots, and CA directories.""" + +import contextlib +import importlib.util +import io +import os +from pathlib import Path +import ssl +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("check_ca_trust", ROOT / "scripts/check_ca_trust.py") +checker = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(checker) + + +class CATrustTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Public certificate data is embedded below. Tests create temporary + # PEM files from these constants; no fixture contains a private key. + cls.temporary = tempfile.TemporaryDirectory() + cls.fixtures = Path(cls.temporary.name) + for name, pem in PUBLIC_CERTIFICATES.items(): + (cls.fixtures / name).write_text(pem) + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def test_unaffected_emsign_root_remains_usable(self): + checker.check_bundle(self.fixtures / "emsign-g1.pem") + + def test_compromised_emsign_path_is_rejected(self): + with self.assertRaisesRegex(ValueError, "emSign ECC Root CA - G3"): + checker.check_bundle(self.fixtures / "emsign-g3.pem") + + def test_aaa_cross_sign_path_is_rejected(self): + with self.assertRaisesRegex(ValueError, "AAA Certificate Services"): + checker.check_bundle(self.fixtures / "aaa.pem") + + def test_mixed_bundle_does_not_hide_an_affected_root(self): + with tempfile.TemporaryDirectory() as tmp: + bundle = Path(tmp) / "mixed.pem" + bundle.write_bytes((self.fixtures / "emsign-g1.pem").read_bytes() + (self.fixtures / "aaa.pem").read_bytes()) + with self.assertRaisesRegex(ValueError, "AAA Certificate Services"): + checker.check_bundle(bundle) + + def test_empty_malformed_and_missing_bundles_fail_closed(self): + with tempfile.TemporaryDirectory() as tmp: + for content in ("", "not a certificate"): + path = Path(tmp) / "bad.pem" + path.write_text(content) + with self.subTest(content=content), self.assertRaises((ssl.SSLError, ValueError)): + checker.check_bundle(path) + with self.assertRaises(OSError): + checker.check_bundle(Path(tmp) / "missing.pem") + + def test_safe_cafile_does_not_hide_stale_hashed_directory(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + (directory / "14bc7599.0").symlink_to(self.fixtures / "emsign-g3.pem") + defaults = ssl.get_default_verify_paths()._replace( + cafile=str(self.fixtures / "emsign-g1.pem"), capath=str(directory) + ) + errors = io.StringIO() + with patch.object(checker.ssl, "get_default_verify_paths", return_value=defaults), patch.dict( + os.environ, {defaults.openssl_capath_env: str(directory)} + ): + with contextlib.redirect_stderr(errors): + result = checker.main(["--system"]) + self.assertEqual(result, 1) + self.assertIn("emSign ECC Root CA - G3", errors.getvalue()) + + def test_multiple_ca_directories_are_all_checked(self): + with tempfile.TemporaryDirectory() as tmp: + clean, stale = Path(tmp) / "clean", Path(tmp) / "stale" + clean.mkdir() + stale.mkdir() + (stale / "ee64a828.0").symlink_to(self.fixtures / "aaa.pem") + defaults = ssl.get_default_verify_paths()._replace( + cafile=str(self.fixtures / "emsign-g1.pem"), capath=None + ) + errors = io.StringIO() + with patch.object(checker.ssl, "get_default_verify_paths", return_value=defaults), patch.dict( + os.environ, {defaults.openssl_capath_env: os.pathsep.join(map(str, (clean, stale)))} + ): + with contextlib.redirect_stderr(errors): + result = checker.main(["--system"]) + self.assertEqual(result, 1) + self.assertIn("AAA Certificate Services", errors.getvalue()) + + +# Public Mozilla CA certificates; provenance: tests/fixtures/ca-trust/README.md. +PUBLIC_CERTIFICATES = { + 'aaa.pem': """-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- +""", + 'emsign-g1.pem': """-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- +""", + 'emsign-g3.pem': """-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- +""", +} + + +if __name__ == "__main__": + unittest.main() From c4fdaf59b3d74377b191938b028a710ee5aad5c7 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:59:02 +0000 Subject: [PATCH 2/5] fix(tls): accept hashed non-CA certificates during trust checks --- scripts/check_ca_trust.py | 7 +++++-- tests/fixtures/ca-trust/README.md | 2 +- tests/test_ca_trust.py | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py index c675dc6f3..36457fe17 100644 --- a/scripts/check_ca_trust.py +++ b/scripts/check_ca_trust.py @@ -30,9 +30,12 @@ def check_bundle(path: Path) -> None: """Raise on an unreadable, empty, malformed, or affected CA bundle.""" context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(cafile=str(path)) + # A hashed OpenSSL directory can include valid non-CA certificates (for + # example Ubuntu's snakeoil server certificate). get_ca_certs() omits them; + # use the total X.509 count to distinguish these from an empty store. + if not context.cert_store_stats()["x509"]: + raise ValueError("no certificates") certificates = context.get_ca_certs(binary_form=True) - if not certificates: - raise ValueError("no CA certificates") blocked = { BLOCKED_ROOTS[fingerprint] for certificate in certificates diff --git a/tests/fixtures/ca-trust/README.md b/tests/fixtures/ca-trust/README.md index 47e440ac1..98b57d82d 100644 --- a/tests/fixtures/ca-trust/README.md +++ b/tests/fixtures/ca-trust/README.md @@ -1,5 +1,5 @@ # CA trust regression fixtures -The public CA certificates embedded in `tests/test_ca_trust.py` exercise the October 2026 eMudhra trust-store check. They contain no private keys. `emsign-g3.pem` and `aaa.pem` are the incident's affected trust anchors; `emsign-g1.pem` is an unaffected emSign anchor that must remain usable. The test creates temporary PEM files from these public constants. The certificate fingerprints are the factual identity of each fixture. +The public certificates embedded in `tests/test_ca_trust.py` exercise the October 2026 eMudhra trust-store check. They contain no private keys. `emsign-g3.pem` and `aaa.pem` are the incident's affected trust anchors; `emsign-g1.pem` is an unaffected emSign anchor that must remain usable. `localhost-leaf.pem` is a synthetic self-signed Ed25519 server certificate with `CA:FALSE`, generated solely to reproduce hashed non-CA certificates found in Ubuntu's OpenSSL directory; its private key was discarded. The test creates temporary PEM files from these public constants. The certificate fingerprints are the factual identity of each fixture. Sources: the [incident root list](https://emudhra.we-have-a-problem.com/roots.json) and the public Mozilla root certificates distributed in Debian's `ca-certificates` package. Behavioral chain verification should also cover the incident site's separate [twelve published server chains](https://emudhra.we-have-a-problem.com/chains.zip). diff --git a/tests/test_ca_trust.py b/tests/test_ca_trust.py index b8e7b1bb0..88a5a343f 100644 --- a/tests/test_ca_trust.py +++ b/tests/test_ca_trust.py @@ -75,6 +75,20 @@ class CATrustTests(unittest.TestCase): self.assertEqual(result, 1) self.assertIn("emSign ECC Root CA - G3", errors.getvalue()) + def test_hashed_non_ca_certificate_does_not_reject_clean_store(self): + # Ubuntu's OpenSSL directory can also hash ssl-cert-snakeoil.pem. + # A valid server certificate has no CA entries in get_ca_certs(). + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + (directory / "ce275665.0").symlink_to(self.fixtures / "localhost-leaf.pem") + defaults = ssl.get_default_verify_paths()._replace( + cafile=str(self.fixtures / "emsign-g1.pem"), capath=str(directory) + ) + with patch.object(checker.ssl, "get_default_verify_paths", return_value=defaults), patch.dict( + os.environ, {defaults.openssl_capath_env: str(directory)} + ), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(checker.main(["--system"]), 0) + def test_multiple_ca_directories_are_all_checked(self): with tempfile.TemporaryDirectory() as tmp: clean, stale = Path(tmp) / "clean", Path(tmp) / "stale" @@ -96,6 +110,16 @@ class CATrustTests(unittest.TestCase): # Public Mozilla CA certificates; provenance: tests/fixtures/ca-trust/README.md. PUBLIC_CERTIFICATES = { + 'localhost-leaf.pem': """-----BEGIN CERTIFICATE----- +MIIBOTCB7KADAgECAhRcVxnOQNEFWf92K4dm4I8ChyjVMzAFBgMrZXAwFDESMBAG +A1UEAwwJbG9jYWxob3N0MB4XDTI2MTAyMTE4NTgwN1oXDTM2MTAxODE4NTgwN1ow +FDESMBAGA1UEAwwJbG9jYWxob3N0MCowBQYDK2VwAyEARglphvNgp9qRCESLHnuu +W8VC5urdZl+YgS+Ux1BoLryjUDBOMB0GA1UdDgQWBBSaEckyR/76lX3a5Ijh8B42 +oJwN1DAfBgNVHSMEGDAWgBSaEckyR/76lX3a5Ijh8B42oJwN1DAMBgNVHRMBAf8E +AjAAMAUGAytlcANBAJkq9ayy+d4AUZnjhWM31rFke5DT+l9QFcPl34+dpVhqJV/L +m4IpPHYagqrBRHn9Dv/C8qYyPBKnAwJRpNptsgw= +-----END CERTIFICATE----- +""", 'aaa.pem': """-----BEGIN CERTIFICATE----- MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow From 5d0a219edee63039f48cb7e2c98188a17d7571ec Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:09:54 +0000 Subject: [PATCH 3/5] fix: distrust incident certificates in Chromium and gRPC --- Dockerfile | 11 ++++++++- docker/chromium-ca-policy.md | 20 ++++++++++++++++ docker/chromium-emudhra.json | 12 ++++++++++ docs/tls-trust.md | 10 +++++--- scripts/check_ca_trust.py | 21 ++++++++++++++-- tests/test_ca_trust.py | 36 ++++++++++++++++++++++++++++ tests/test_chromium_ca_policy.py | 41 ++++++++++++++++++++++++++++++++ 7 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 docker/chromium-ca-policy.md create mode 100644 docker/chromium-emudhra.json create mode 100644 tests/test_chromium_ca_policy.py diff --git a/Dockerfile b/Dockerfile index 2e49c6ed6..28653ab7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,12 @@ RUN python /usr/local/bin/check-ca-trust.py --system COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ && python /usr/local/bin/check-ca-trust.py --python -ENV NODE_OPTIONS=--use-openssl-ca +ENV NODE_OPTIONS=--use-openssl-ca \ + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt + +# Chromium has a separate root store. Distrust the compromised intermediate +# public keys explicitly, including cross-signs, without waiting for CRL updates. +COPY docker/chromium-emudhra.json /etc/chromium/policies/managed/odysseus-emudhra.json # Private browser automation wrapper used by the native `private_browser` tool. # Chromium is installed above, so agent-browser can drive the existing browser @@ -123,6 +128,10 @@ COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/ RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \ && rm -rf /tmp/odysseus-wheels +# Check gRPC's effective defaults after all Python dependencies are installed. +# Its wheel can retain affected roots even when pip/certifi are current. +RUN python /usr/local/bin/check-ca-trust.py --system --python --grpc + # Copy app code COPY . . # Require the redistribution notices in the image build context. diff --git a/docker/chromium-ca-policy.md b/docker/chromium-ca-policy.md new file mode 100644 index 000000000..b4852ac3a --- /dev/null +++ b/docker/chromium-ca-policy.md @@ -0,0 +1,20 @@ +# Chromium incident distrust policy + +`chromium-emudhra.json` is installed in the Docker image at `/etc/chromium/policies/managed/odysseus-emudhra.json`. Chromium 132 and later support `CADistrustedCertificates`: a list of base64-encoded DER X.509 certificates whose public keys must not be accepted for TLS server authentication. Matching is by SubjectPublicKeyInfo (SPKI), so one certificate per compromised key also blocks its cross-signs. This policy adds no trust, disables no certificate checks, and does not require a revocation-list download. Native browsers and other browser distributions require separately managed installation of the appropriate policy. + +Policy semantics: [Chromium's version-155 definition](https://github.com/chromium/chromium/blob/155.0.8059.79/components/policy/resources/templates/policy_definitions/CertificateManagement/CADistrustedCertificates.yaml). Linux installation paths: [Chromium policy documentation](https://www.chromium.org/administrators/configuring-policy-for-extensions/#linux). + +Public certificate source, retrieved 2026-10-21: [incident intermediate inventory](https://emudhra.we-have-a-problem.com/impacted-intermediates.json). Its downloaded JSON has SHA-256 `f67b3abe9b4879f4e91874ed041beccd71257f1db41c588c724ccf12aa90c59e`. Each selected certificate's DER fingerprint and public-key fingerprint were checked against that inventory before inclusion. The eight unique keys below are covered, including the code-signing intermediate keys listed in the incident. The policy contains only public certificates; no private key is included. + +| Intermediate | SHA-256 of DER SPKI | +| --- | --- | +| EM OV TLS CA - G3B | `43bffa19d6ef651451f6841e019931e96f6f2eaeff28ab4b6f1ce4576de2717b` | +| EM DV TLS CA - G3C | `49dd353dc2b919a3770645117d0e60c16115e7d1bc259eefe734b00ea434fe5f` | +| EM EV TLS CA - G3C | `5c0660a2b164910d8ff589c18e14552e9e0ec237dcefe2cb736d41a4811f81a0` | +| EM OV TLS CA - G3C | `63ecefee431d49523bc1136c4e72d0206a477086caf606cd5cf025686c109cf8` | +| EM OV CS CA - G3C | `92c063d67e8f14db71d288e11266ffdaf69cbf115f7bab14f3bc2f68d25e0e30` | +| EM EV TLS CA - G3B | `e0254c7fcdc265e2f0f8a428dabb75ab4ae039b4872d62f5ee57f0877d5a8a41` | +| EM DV TLS CA - G3B | `ebed6475c2c2d80fcb199f3fe8dd6e6b3ce60b77eb15d4e42c3dec0c6f5b5105` | +| EM OV CS CA - G3B | `ec4ffa1ad0ac88081314b7aaffef6a91f97cd57291bb5ba686c5415f5eb4f6b6` | + +`tests/test_chromium_ca_policy.py` verifies that the policy distrusts exactly these eight keys using valid intermediate CA certificates and adds no trust settings. Changes to the encoded material need a fresh source/fingerprint check and behavioral validation: all twelve published incident HTTPS chains must be rejected by the actual Chromium binary while ordinary HTTPS still succeeds. Confirm loading and any policy errors at `chrome://policy`. Do not treat a passing PEM-store check as a browser-policy check. diff --git a/docker/chromium-emudhra.json b/docker/chromium-emudhra.json new file mode 100644 index 000000000..104f82f60 --- /dev/null +++ b/docker/chromium-emudhra.json @@ -0,0 +1,12 @@ +{ + "CADistrustedCertificates": [ + "MIIDZjCCAu2gAwIBAgIPbK2jZ/scEACIEg6iAtVoMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNjAwMTJaFw00MDA0MjgxODMwMDBaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJWMRswGQYDVQQDDBJFTSBPViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARziXqg6zz9yosSQC/68KvE4SErr43ud4rJ7i1tuD+sKAxeBWOK56Ks1xwyKo42HN659pxDArndjXPRzGyqsswK7k4hbOl53VGKSvS2JuVoTN1lgDftDIVOTMEvTPhkbQ2jggGWMIIBkjAfBgNVHSMEGDAWgBThOqshfPHjTVSu1HK3/YtZQiCQVDAdBgNVHQ4EFgQUm9hlUa/n5yw3rQjuaRnZvzeErgowHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMF0GA1UdIARWMFQwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTANBgsrBgEEAYOOIQECbjAIBgZngQwBAgIwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1PVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNnADBkAjBAk+5BizdJhrED3wb7F2Gm7bWI60dlNd075nghT2TrR4NWKZtJZcbc28xxIY/NONICMF6mPdsjosaqmvEMWqKcBh4vsIvXSKnQj5o8zrFB4oKS+accjYTe8Dz3bQjauvy9uw==", + "MIIDaDCCAu6gAwIBAgIPWIsuPCu4zaE+GhIJJUMhMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM1MDlaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOnp4YsjETTJN3WkDOmeCwO/u/kBV5ooSP6A3G6O9kZRRTuFypDPjYw8CvRiAN9nlFJ+bd9uUooHfmHsv6oiLdL3+j9pMEJnubKGVZv4+8OX2sJpVbMyLpQVh3xsqRB0Bo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFK5+EFw8KZbBqEcnBVnx4D3U5QrzMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMEN2HWMocbuWdy1d33pH6JpMvP/V/DIS5PtZbv1Q7TXEbxrcyxgyuddOJscYDJJlAIwB4sH1BpbScDzkdD4tJBSwrG/1Mx82vb8YaySehNuu1+1oXNdAEKwaJ0mVzScy2+6", + "MIIDaDCCAu6gAwIBAgIQAIAJkopkEHYBrwX6LxnxCTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTUzOTMxWhcNNDAwNDI4MTgzMDAwWjBAMQswCQYDVQQGEwJVUzEUMBIGA1UECgwLZU11ZGhyYSBJbmMxGzAZBgNVBAMMEkVNIEVWIFRMUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABC1W6Cmm+4LGSkaX2WCvlR6QZ66R+SbTQwbaSKiq63hptqZ+bSdCbgRCqgOzbhmI3/Y2UU7hzwf1uks/8qBmRa59MdWGatMao9JG6wsxd+6Zy3VmxRbOdICKZo+R8MR/yqOCAZUwggGRMB8GA1UdIwQYMBaAFOE6qyF88eNNVK7Ucrf9i1lCIJBUMB0GA1UdDgQWBBSmJV6AqU2385Z8G8eR35K87kvrXzAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwXAYDVR0gBFUwUzA5BgsrBgEEAYOOIQEAATAqMCgGCCsGAQUFBwIBFhxodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tMA0GCysGAQQBg44hAQJ4MAcGBWeBDAEBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRVZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwBy1wTnzH4lXDTy2PD91ysLjvnlVN3JbJmc1BIMEU8TOLkYVHibMc9HeyonikFSfmAjEA6uE3QRxlrZTz/1/lXPTEMUhoauAtycagtlNFp79kjhvNa2/fAal1GNxsOutLnr0Y", + "MIIDaDCCAu6gAwIBAgIPKWkmwiejoVJjkrGUgqUEMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM3NDFaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gT1YgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEE4JRzVKskpjByKrZdzId65fnCb0LTKCspV3IrVHc/J0ViaFZfjpRIOHv1jHGPhg1ysrimGmmLhYx6KKBgatWYW3EXZGKvAnzqw5jz3kqpNztc4e+N7ulaVSfHAKycO1no4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFDTq6gzryQra+dDF4u2beMmHnhtsMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAm4wCAYGZ4EMAQICMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMSZt2xsPeumRdeSMSQ3CbzIZeQ/q7GDCWfeE1Pq+mhgA6gRM44cNVF3xJ+wsxJk+wIwFJbaDMokD8Ei6eNOiA8yUMYvfzQyngmrhZQMdPCK1SewrR794rSb9vGqxZzelQth", + "MIIDWzCCAuGgAwIBAgIPVEt/kiwxriIKSpIpFvbmMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2Mzg0MFoXDTQwMDQyODE4MzAwMFowPzELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRowGAYDVQQDDBFFTSBPViBDUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABLnw4GNHAqacpqrdRDz9YsQbm6SyCSRpfXl052kaEm+5avs+NML9O8X0gwYfJl3CpYQ2NHMjUQQ2so8E84YjYwZ5gOJQPgJTLWVGI1pxZaJZoe9Jb3QbX97neHII5Lj676OCAYswggGHMB8GA1UdIwQYMBaAFGh0A/iFyo1UUFcdFuDI3kUNq80tMB0GA1UdDgQWBBTub3MpjW+/p7Mbw5U09hVWRHhmtzATBgNVHSUEDDAKBggrBgEFBQcDAzBeBgNVHSAEVzBVMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDgYMKwYBBAGDjiEBAoFIMAgGBmeBDAEEATB0BggrBgEFBQcBAQRoMGYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLWMuZW1zaWduLmNvbTA+BggrBgEFBQcwAoYyaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbS9jZXJ0cy9FTU9WQ1NDQUczQy5wN2MwNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290Y3NjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNoADBlAjEA8yPg8sntOhLwV2QXJ5LEaKSpo/fnE3uOgTadK73fDhyjiqr0Io9GVuQTYqBPbQ2TAjBgeuApmBS486+iei03lRrp139dPw6k28jjGNvWsAU7A1N86J1zPJVDEyVH+nMAWC0=", + "MIIDaDCCAu2gAwIBAgIQAMfEHv6S2CXwXq6EGkqCwTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTYwMzMxWhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRVYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEJduoWmTTl9oLEFLp6lRuSHvv6pydnWigxerc/u91ppu11vwB9Cv7ym60C81kwHZ+Nv/QXZeoE8WqDGyVMOec9fCGZGjxOAZCCqXIUYQLHiUlQOeQ5LZUw4Uu1krx6GV9o4IBlTCCAZEwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFD7cINmJRpiHe4c/2IFHslwV2EcGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBcBgNVHSAEVTBTMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAngwBwYFZ4EMAQEwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1FVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNpADBmAjEA9Jq999oaOUKhFwopw33iCmt0OHIn3gjT43GfuOHR/YckXdwoNdDAlIEhz0NgRO3/AjEAvY/a3OMvktIZLYjn+WLKmp0QhrIMV+e6EpS7j4GVeOKouNp6MN/P+ByGeiRMghzB", + "MIIDaDCCAu6gAwIBAgIQAKSu5njyM2K0wORgt3LyFDAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTU1NDA4WhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE557ZwEqXc8RzXlERHahawr1eZKs3S4DhLLwgOOweTw47gS/JhZ4eZ6+wTi+LyX90hbplqpEe6KTkW2LVzAPVa+Z4s8OMtNyTfKJGDiOEU8iSk10p1E+R4VYGtjzO2rRoo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFOIeL+ccGTkOVUtiwSAkVzUkiiUUMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQi5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwLmsGXi6oFMJ2klkcEMXMug6gvxQWZWkewiDWVzQyl2L6z6DuQfPOIbh1grvdULDCAjEAxZvLlxUavG69J6wnSjC8XBbqNIQsJ6CE2QdNOe84Ptxb6VG8gt2TlfJbHii+KNvK", + "MIIDWzCCAuCgAwIBAgIPeBiTP4+FtGGaKzX7qD0KMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2MDgyN1oXDTQwMDQyODE4MzAwMFowPjELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGjAYBgNVBAMMEUVNIE9WIENTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEqrsPsGZqz88B+tUr/P1MK/LLPDSX3nOVDlgZth/mlQmHQcjEkUObNKBDHK/+W/m0wcjorgAB+fTcitgsWfNjnpbNbD1NsM7+wmw9YlF7N3YTcnQ/jordd7EAZPKr5eBQo4IBizCCAYcwHwYDVR0jBBgwFoAUaHQD+IXKjVRQVx0W4MjeRQ2rzS0wHQYDVR0OBBYEFHn66OzP2w11jaRY1p+27dx8Sd7mMBMGA1UdJQQMMAoGCCsGAQUFBwMDMF4GA1UdIARXMFUwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTAOBgwrBgEEAYOOIQECgUgwCAYGZ4EMAQQBMHQGCCsGAQUFBwEBBGgwZjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD4GCCsGAQUFBzAChjJodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZDU0NBRzNCLnA3YzA2BgNVHR8ELzAtMCugKaAnhiVodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3Rjc2NhZzMuY3JsMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMAoGCCqGSM49BAMDA2kAMGYCMQC/zr03YEELsNKqzUtH2dVFi4BaKFdyrr/6G1dIWp8KAVgsw2/u4lyyboUDaC9jDvICMQC9HTxku84tNX9dmHOtH7aC2Nu/JM0jTf6KjZAvovCJzzv1IFWSZpVLZibynMLXOrM=" + ] +} diff --git a/docs/tls-trust.md b/docs/tls-trust.md index 3d91e08cd..ed89d2011 100644 --- a/docs/tls-trust.md +++ b/docs/tls-trust.md @@ -21,14 +21,18 @@ python scripts/check_ca_trust.py --system --python Explicit PEM bundles can also be passed as positional arguments. The check rejects the incident's known root fingerprints and malformed/empty bundles. It does not model native OS intermediate revocation, custom TLS implementations, or arbitrary certificates added by an operator. -Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if affected roots remain. Node uses the corrected system store through `--use-openssl-ca`. Rebuild with a fresh base and dependency layers when remediating an existing deployment: +Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if affected roots remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of the incident roots: gRPC can otherwise fall back to its affected bundled defaults. + +Chromium uses a separate trust store. The Docker image installs a managed `CADistrustedCertificates` policy for the eight compromised intermediate public keys, covering their cross-signs without depending on background revocation updates. It adds no trusted certificate and does not disable certificate verification. The policy and public-certificate provenance are documented in [the browser policy note](../docker/chromium-ca-policy.md). Native browser installations and independently managed profiles require their own platform policy or vendor update; this image policy does not configure host browsers. + +Rebuild with a fresh base and dependency layers when remediating an existing deployment: ```sh docker compose build --pull --no-cache odysseus docker compose up -d odysseus -docker compose exec odysseus python scripts/check_ca_trust.py --system --python +docker compose exec odysseus python scripts/check_ca_trust.py --system --python --grpc ``` SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. -For behavioral validation, all twelve [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. +For behavioral validation, all twelve [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. In Chromium, confirm the managed policy is loaded at `chrome://policy` and test the incident endpoints through the actual browser path with certificate verification enabled. The PEM checker cannot inspect Chromium's policy or its active verifier. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py index 36457fe17..c1db2f2cc 100644 --- a/scripts/check_ca_trust.py +++ b/scripts/check_ca_trust.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse import hashlib import importlib +import importlib.util import os from pathlib import Path import re @@ -64,14 +65,28 @@ def system_paths() -> list[Path]: return paths +def grpc_roots_path() -> Path: + """Select gRPC's override or its independent Python-wheel default bundle.""" + override = os.environ.get("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH") + if override: + # gRPC can fall back to its bundled roots when this file is unreadable. + # Check the configured file strictly instead of blessing that fallback. + return Path(override) + spec = importlib.util.find_spec("grpc") + if spec is None or spec.origin is None: + raise ImportError("grpcio is not installed; cannot check its default roots") + return Path(spec.origin).parent / "_cython" / "_credentials" / "roots.pem" + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("bundles", nargs="*", type=Path) parser.add_argument("--system", action="store_true", help="check effective OpenSSL PEM stores") parser.add_argument("--python", action="store_true", help="check certifi and pip's separate vendored bundle") + parser.add_argument("--grpc", action="store_true", help="check gRPC's selected default CA file") args = parser.parse_args(argv) - if not (args.bundles or args.system or args.python): - parser.error("select --system, --python, or explicit PEM bundle paths") + if not (args.bundles or args.system or args.python or args.grpc): + parser.error("select --system, --python, --grpc, or explicit PEM bundle paths") try: paths = list(args.bundles) if args.system: @@ -79,6 +94,8 @@ def main(argv: list[str] | None = None) -> int: if args.python: for name in ("certifi", "pip._vendor.certifi"): paths.append(Path(importlib.import_module(name).where())) + if args.grpc: + paths.append(grpc_roots_path()) paths = list(dict.fromkeys(path.resolve() for path in paths)) errors = [] for path in paths: diff --git a/tests/test_ca_trust.py b/tests/test_ca_trust.py index 88a5a343f..b2ab32d40 100644 --- a/tests/test_ca_trust.py +++ b/tests/test_ca_trust.py @@ -7,6 +7,7 @@ import os from pathlib import Path import ssl import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -107,6 +108,41 @@ class CATrustTests(unittest.TestCase): self.assertEqual(result, 1) self.assertIn("AAA Certificate Services", errors.getvalue()) + def test_grpc_bundled_roots_are_checked_without_an_override(self): + with tempfile.TemporaryDirectory() as temporary: + package = Path(temporary) / "grpc" + bundle = package / "_cython" / "_credentials" / "roots.pem" + bundle.parent.mkdir(parents=True) + bundle.write_bytes((self.fixtures / "emsign-g3.pem").read_bytes()) + errors = io.StringIO() + with patch.dict(os.environ, {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": ""}), patch.object( + checker.importlib.util, "find_spec", return_value=SimpleNamespace(origin=str(package / "__init__.py")) + ), contextlib.redirect_stderr(errors): + self.assertEqual(checker.main(["--grpc"]), 1) + self.assertIn("emSign ECC Root CA - G3", errors.getvalue()) + + def test_grpc_explicit_clean_store_replaces_the_independent_bundle(self): + with patch.dict(os.environ, {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": str(self.fixtures / "emsign-g1.pem")}), patch.object( + checker.importlib.util, "find_spec", side_effect=AssertionError("must use the explicit store") + ), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(checker.main(["--grpc"]), 0) + + def test_grpc_invalid_override_does_not_fall_back_silently(self): + with tempfile.TemporaryDirectory() as temporary: + for name, content in (("empty.pem", ""), ("malformed.pem", "not a certificate"), ("affected.pem", PUBLIC_CERTIFICATES["aaa.pem"])): + bundle = Path(temporary) / name + bundle.write_text(content) + with self.subTest(name=name), patch.dict(os.environ, {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": str(bundle)}), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--grpc"]), 1) + with patch.dict(os.environ, {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": str(Path(temporary) / "missing.pem")}), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--grpc"]), 1) + + def test_grpc_check_reports_a_missing_installation(self): + with patch.dict(os.environ, {"GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": ""}), patch.object( + checker.importlib.util, "find_spec", return_value=None + ), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--grpc"]), 1) + # Public Mozilla CA certificates; provenance: tests/fixtures/ca-trust/README.md. PUBLIC_CERTIFICATES = { diff --git a/tests/test_chromium_ca_policy.py b/tests/test_chromium_ca_policy.py new file mode 100644 index 000000000..a8b8a47e1 --- /dev/null +++ b/tests/test_chromium_ca_policy.py @@ -0,0 +1,41 @@ +"""The managed browser policy must distrust exactly the incident's eight keys.""" + +import base64 +import hashlib +import json +from pathlib import Path +import unittest + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization + + +ROOT = Path(__file__).resolve().parents[1] +# Public incident inventory: impacted-intermediates.json, 2026-10-21. +COMPROMISED_SPKIS = { + "43bffa19d6ef651451f6841e019931e96f6f2eaeff28ab4b6f1ce4576de2717b", + "49dd353dc2b919a3770645117d0e60c16115e7d1bc259eefe734b00ea434fe5f", + "5c0660a2b164910d8ff589c18e14552e9e0ec237dcefe2cb736d41a4811f81a0", + "63ecefee431d49523bc1136c4e72d0206a477086caf606cd5cf025686c109cf8", + "92c063d67e8f14db71d288e11266ffdaf69cbf115f7bab14f3bc2f68d25e0e30", + "e0254c7fcdc265e2f0f8a428dabb75ab4ae039b4872d62f5ee57f0877d5a8a41", + "ebed6475c2c2d80fcb199f3fe8dd6e6b3ce60b77eb15d4e42c3dec0c6f5b5105", + "ec4ffa1ad0ac88081314b7aaffef6a91f97cd57291bb5ba686c5415f5eb4f6b6", +} + + +class ChromiumCATrustTests(unittest.TestCase): + def test_policy_distrusts_each_compromised_key_without_adding_trust(self): + policy = json.loads((ROOT / "docker" / "chromium-emudhra.json").read_text()) + self.assertEqual(set(policy), {"CADistrustedCertificates"}) + self.assertEqual(len(policy["CADistrustedCertificates"]), 8) + keys = set() + for encoded in policy["CADistrustedCertificates"]: + certificate = x509.load_der_x509_certificate(base64.b64decode(encoded, validate=True)) + self.assertTrue(certificate.extensions.get_extension_for_class(x509.BasicConstraints).value.ca) + self.assertNotEqual(certificate.subject, certificate.issuer) + public_key = certificate.public_key().public_bytes( + serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo + ) + keys.add(hashlib.sha256(public_key).hexdigest()) + self.assertEqual(keys, COMPROMISED_SPKIS) From cd9f1f734fc12d3479de3039558902cf421c5454 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:48:34 +0000 Subject: [PATCH 4/5] fix(security): detect incident CAs in custom trust stores Reject the published compromised intermediates and cross-signed G3 issuer when supplied as trust anchors. Add explicit CA-override inspection and offline chain regressions with a fixed verification time and affected positive control. --- docs/tls-trust.md | 12 ++- scripts/check_ca_trust.py | 81 ++++++++++++++--- tests/fixtures/ca-trust/README.md | 12 ++- tests/fixtures/ca-trust/incident.json | 121 ++++++++++++++++++++++++++ tests/test_ca_trust.py | 119 +++++++++++++++++++++++++ 5 files changed, 327 insertions(+), 18 deletions(-) create mode 100644 tests/fixtures/ca-trust/incident.json diff --git a/docs/tls-trust.md b/docs/tls-trust.md index ed89d2011..8e0351c68 100644 --- a/docs/tls-trust.md +++ b/docs/tls-trust.md @@ -16,12 +16,16 @@ The pinned bootstrap protects package integrity even when the old installer's ce On Linux/OpenSSL deployments, inspect both the effective CA file and hashed CA directory, plus the two Python bundles: ```sh -python scripts/check_ca_trust.py --system --python +python scripts/check_ca_trust.py --system --python --overrides ``` -Explicit PEM bundles can also be passed as positional arguments. The check rejects the incident's known root fingerprints and malformed/empty bundles. It does not model native OS intermediate revocation, custom TLS implementations, or arbitrary certificates added by an operator. +Explicit PEM bundles can also be passed as positional arguments. The check rejects the four published root fingerprints, all sixteen published intermediate certificates (including their cross-signs), the cross-signed G3 issuer in the published test chains, and malformed/empty bundles. Checking intermediates matters because adding one as a custom trust anchor can restore an affected chain even after its roots have been removed. -Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if affected roots remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of the incident roots: gRPC can otherwise fall back to its affected bundled defaults. +`--overrides` checks every nonempty `LLM_CA_BUNDLE`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `PIP_CERT`, `NODE_EXTRA_CA_CERTS` and `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` setting in the checker's process environment, including configured fallback settings. Requests/pip hashed-directory overrides are inspected too; `--system` covers `SSL_CERT_DIR`. Run with the application's environment and filesystem access. The script does not load `.env` or application configuration files; pass any CA paths configured only there explicitly, for example `python scripts/check_ca_trust.py --system --python /path/to/private-ca.pem`. Missing or invalid configured files fail the check. Selecting only `--overrides` with no configured paths also fails instead of reporting that stores were validated. + +This is an offline inventory check of known certificate fingerprints. It does not modify stores, reject requests at runtime, model native OS revocation, or detect arbitrary reissued certificates sharing the compromised keys. Chromium's policy separately matches public keys. Check mounted or replaced CA files again after deployment, remove affected trust through the owning platform/client configuration, and restart clients that cache SSL contexts. A passing build cannot validate custom files introduced later. + +Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if known affected CA certificates remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of known affected CA certificates: gRPC can otherwise fall back to its affected bundled defaults. Upstream gRPC has [merged a bundle update](https://github.com/grpc/grpc/commit/8b32af3bf498c2ba9667410eaa8c836e700e97e6); as checked on 2026-10-21, its [release backport](https://github.com/grpc/grpc/pull/43700) is still open and the latest published grpcio wheel is 1.84.0. Retain the checked override until a released replacement's actual trust behavior has been verified. Chromium uses a separate trust store. The Docker image installs a managed `CADistrustedCertificates` policy for the eight compromised intermediate public keys, covering their cross-signs without depending on background revocation updates. It adds no trusted certificate and does not disable certificate verification. The policy and public-certificate provenance are documented in [the browser policy note](../docker/chromium-ca-policy.md). Native browser installations and independently managed profiles require their own platform policy or vendor update; this image policy does not configure host browsers. @@ -30,7 +34,7 @@ Rebuild with a fresh base and dependency layers when remediating an existing dep ```sh docker compose build --pull --no-cache odysseus docker compose up -d odysseus -docker compose exec odysseus python scripts/check_ca_trust.py --system --python --grpc +docker compose exec odysseus python scripts/check_ca_trust.py --system --python --grpc --overrides ``` SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py index c1db2f2cc..7f017fe3d 100644 --- a/scripts/check_ca_trust.py +++ b/scripts/check_ca_trust.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Reject the eMudhra incident's root certificates in PEM trust stores. +"""Reject the eMudhra incident's known CA certificates in PEM trust stores. This checks certificate contents, including OpenSSL's lazily loaded CA directory. It does not model native OS certificate revocation or arbitrary custom clients. Incident fingerprints: https://emudhra.we-have-a-problem.com/roots.json +and https://emudhra.we-have-a-problem.com/impacted-intermediates.json +Cross-signed issuer: https://emudhra.we-have-a-problem.com/chains.zip """ from __future__ import annotations @@ -19,12 +21,35 @@ import ssl import sys -BLOCKED_ROOTS = { +BLOCKED_CERTIFICATES = { "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b": "emSign ECC Root CA - G3", "d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4": "AAA Certificate Services", "7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d": "emSign Root TLS CA - G3", "f2edaaff3d9b701fb4d14c9f285f8c0bad90fc3bcc1fa07d47f4718edaba34a5": "emSign Root CS CA - G3", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd": "emSign Root TLS CA - G3 (cross-signed)", + # Intermediates can themselves be operator-supplied trust anchors. Include + # both published signatures for each key, without parsing SPKI in stdlib. + "42c4accd12c4f59597925e1a08a7a90102608543809308d50ec427e118ba1493": "EM DV TLS CA - G3B (eMudhra)", + "45a883f349aedfd5f3a649b5498ffb121712ba8b448f3f2a91dbc5a766fcddbe": "EM DV TLS CA - G3B (Sectigo)", + "98b896625c0c230adeec3eb2cf6aad5001ecf9156a68644e82374ce5833bc25b": "EM OV TLS CA - G3B (eMudhra)", + "5f1ded4dd590af74631290cf073b122a2aac6a1fa479778a0bd43579bd832699": "EM OV TLS CA - G3B (Sectigo)", + "e7568fe6e12331f4fd7de61da4c7559d0f8eb192a218d4741a3ba75184333838": "EM EV TLS CA - G3B (eMudhra)", + "579480aea166290bd3c643589e795fc0d0e54e86505d180f1dff99efd3199512": "EM EV TLS CA - G3B (Sectigo)", + "0164d7ceae5bb909c272af151ff44b1177ae9b47d9424b94f1d9558c30934d19": "EM DV TLS CA - G3C (eMudhra)", + "62b7f9868263f72f6911fc78ceaf72c1979158b7368bf648683bb2aa39b02e72": "EM DV TLS CA - G3C (Sectigo)", + "cb6e15165ac802e9b3f993f16c25d41f4e81c21ebde71ae1b012ac00bf9c7c3f": "EM OV TLS CA - G3C (eMudhra)", + "7485b51c0ed192878fa1d2f948fb529ae1842cc5086c4b67f7be5b2039de9489": "EM OV TLS CA - G3C (Sectigo)", + "432ec25f0d28de9006d97bf752f17bbb2cd41f669933fcb072d9a1148d0261f3": "EM EV TLS CA - G3C (eMudhra)", + "fd9d5c2dbbd4995ab69b8d50c439d9527b00529199c2fd01b2174eb58f07021f": "EM EV TLS CA - G3C (Sectigo)", + "9e39de2881273acc225a149c72ba3e0f51424963669ab633ee1dc7df5db067b4": "EM OV CS CA - G3B (eMudhra)", + "a496e23ac0cab8155fc9b117b97eae368efcbb9041aec8804d5a308d2d01c2d8": "EM OV CS CA - G3B (Sectigo)", + "4b2460a466f95fd40113bcdcca2ceb29b3b093921de45474da7df48874c630b1": "EM OV CS CA - G3C (eMudhra)", + "84112972b8e59d14560ae7e7119654474aad119990cf3722bfc1e3b91615e1fd": "EM OV CS CA - G3C (Sectigo)", } +OVERRIDE_ENV_VARS = ( + "LLM_CA_BUNDLE", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", + "PIP_CERT", "NODE_EXTRA_CA_CERTS", "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH", +) def check_bundle(path: Path) -> None: @@ -38,12 +63,20 @@ def check_bundle(path: Path) -> None: raise ValueError("no certificates") certificates = context.get_ca_certs(binary_form=True) blocked = { - BLOCKED_ROOTS[fingerprint] + BLOCKED_CERTIFICATES[fingerprint] for certificate in certificates - if (fingerprint := hashlib.sha256(certificate).hexdigest()) in BLOCKED_ROOTS + if (fingerprint := hashlib.sha256(certificate).hexdigest()) in BLOCKED_CERTIFICATES } if blocked: - raise ValueError("affected roots: " + ", ".join(sorted(blocked))) + raise ValueError("affected CA certificates: " + ", ".join(sorted(blocked))) + + +def ca_directory_paths(directory: Path) -> list[Path]: + """Return the hashed certificate files consumed by OpenSSL's CApath.""" + return [ + path for path in directory.iterdir() + if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name) and path.is_file() + ] def system_paths() -> list[Path]: @@ -55,16 +88,33 @@ def system_paths() -> list[Path]: capath = os.environ.get(defaults.openssl_capath_env, defaults.capath) if capath: for directory in filter(None, capath.split(os.pathsep)): - paths.extend( - path - for path in Path(directory).iterdir() - if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name) and path.is_file() - ) + paths.extend(ca_directory_paths(Path(directory))) if not paths: raise ValueError("no effective OpenSSL PEM CA store found") return paths +def override_paths() -> list[Path]: + """Inspect every configured override, including client fallback settings.""" + paths = [] + for name in OVERRIDE_ENV_VARS: + value = os.environ.get(name, "") + if name == "LLM_CA_BUNDLE": + value = value.strip() # Match src/tls_overrides.py. + if not value: + continue + path = Path(value) + # Requests (also used by pip) accepts either a file or a hashed directory. + if name in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "PIP_CERT") and path.is_dir(): + entries = ca_directory_paths(path) + if not entries: + raise ValueError(f"{name}: no hashed certificates in {path}") + paths.extend(entries) + else: + paths.append(path) + return paths + + def grpc_roots_path() -> Path: """Select gRPC's override or its independent Python-wheel default bundle.""" override = os.environ.get("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH") @@ -84,9 +134,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--system", action="store_true", help="check effective OpenSSL PEM stores") parser.add_argument("--python", action="store_true", help="check certifi and pip's separate vendored bundle") parser.add_argument("--grpc", action="store_true", help="check gRPC's selected default CA file") + parser.add_argument("--overrides", action="store_true", help="check configured LLM/HTTPX/Requests/pip/Node/gRPC CA overrides") args = parser.parse_args(argv) - if not (args.bundles or args.system or args.python or args.grpc): - parser.error("select --system, --python, --grpc, or explicit PEM bundle paths") + if not (args.bundles or args.system or args.python or args.grpc or args.overrides): + parser.error("select --system, --python, --grpc, --overrides, or explicit PEM bundle paths") try: paths = list(args.bundles) if args.system: @@ -96,7 +147,11 @@ def main(argv: list[str] | None = None) -> int: paths.append(Path(importlib.import_module(name).where())) if args.grpc: paths.append(grpc_roots_path()) + if args.overrides: + paths.extend(override_paths()) paths = list(dict.fromkeys(path.resolve() for path in paths)) + if not paths: + raise ValueError("no CA bundle files selected; also select --system, --python, or an explicit path") errors = [] for path in paths: try: @@ -108,7 +163,7 @@ def main(argv: list[str] | None = None) -> int: except (ImportError, OSError, ValueError, ssl.SSLError) as exc: print(str(exc), file=sys.stderr) return 1 - print(f"Validated {len(paths)} CA bundle file(s): no eMudhra incident roots") + print(f"Validated {len(paths)} CA bundle file(s): no known eMudhra incident CA certificates") return 0 diff --git a/tests/fixtures/ca-trust/README.md b/tests/fixtures/ca-trust/README.md index 98b57d82d..3feef0f40 100644 --- a/tests/fixtures/ca-trust/README.md +++ b/tests/fixtures/ca-trust/README.md @@ -2,4 +2,14 @@ The public certificates embedded in `tests/test_ca_trust.py` exercise the October 2026 eMudhra trust-store check. They contain no private keys. `emsign-g3.pem` and `aaa.pem` are the incident's affected trust anchors; `emsign-g1.pem` is an unaffected emSign anchor that must remain usable. `localhost-leaf.pem` is a synthetic self-signed Ed25519 server certificate with `CA:FALSE`, generated solely to reproduce hashed non-CA certificates found in Ubuntu's OpenSSL directory; its private key was discarded. The test creates temporary PEM files from these public constants. The certificate fingerprints are the factual identity of each fixture. -Sources: the [incident root list](https://emudhra.we-have-a-problem.com/roots.json) and the public Mozilla root certificates distributed in Debian's `ca-certificates` package. Behavioral chain verification should also cover the incident site's separate [twelve published server chains](https://emudhra.we-have-a-problem.com/chains.zip). +Sources: the [incident root list](https://emudhra.we-have-a-problem.com/roots.json) and the public Mozilla root certificates distributed in Debian's `ca-certificates` package. + +`incident.json` contains the four published roots, sixteen [compromised intermediate certificates and cross-signs](https://emudhra.we-have-a-problem.com/impacted-intermediates.json), and the [twelve published server chains](https://emudhra.we-have-a-problem.com/chains.zip), retrieved on 2026-10-21. Certificates are deduplicated by SHA-256 of DER; each chain lists its leaf first. The additional cross-signed `emSign Root TLS CA - G3` issuer comes from those chains and has fingerprint `3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd`. Like the intermediates, it must not be accepted as a custom trust anchor. All data is public certificate material, with no private keys. + +Downloaded source SHA-256 values: + +- `roots.json`: `119f32cd81a6b5a4e3c4c2df5fe07daa90dc43cafa123a62f0394f1895d92380` +- `impacted-intermediates.json`: `f67b3abe9b4879f4e91874ed041beccd71257f1db41c588c724ccf12aa90c59e` +- `chains.zip`: `8a2edc81b6c51e3ad27d45ca1e441ec84c961e2dee0fc547f2a02c0f74315f95` + +The offline OpenSSL regression verifies server purpose, hostname and strict certificate rules with partial-chain trust, at `2026-10-21T12:00:00Z` (`1792584000`). Fixing the verification date keeps later certificate expiry from masking a broken mitigation. All twelve chains must verify against the affected positive-control roots and fail against the unaffected G1-only store. Adding the compromised DV G3B intermediate restores two paths, which the checker must reject. The regression makes no network requests and skips chain verification if the OpenSSL executable is unavailable; certificate-inventory tests still run using Python's standard library. diff --git a/tests/fixtures/ca-trust/incident.json b/tests/fixtures/ca-trust/incident.json new file mode 100644 index 000000000..771a341f2 --- /dev/null +++ b/tests/fixtures/ca-trust/incident.json @@ -0,0 +1,121 @@ +{ + "verification_time": 1792584000, + "roots": { + "7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d": "emSign Root TLS CA - G3", + "f2edaaff3d9b701fb4d14c9f285f8c0bad90fc3bcc1fa07d47f4718edaba34a5": "emSign Root CS CA - G3", + "d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4": "AAA Certificate Services", + "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b": "emSign ECC Root CA - G3" + }, + "intermediates": { + "42c4accd12c4f59597925e1a08a7a90102608543809308d50ec427e118ba1493": "EM DV TLS CA - G3B (eMudhra)", + "45a883f349aedfd5f3a649b5498ffb121712ba8b448f3f2a91dbc5a766fcddbe": "EM DV TLS CA - G3B (Sectigo)", + "98b896625c0c230adeec3eb2cf6aad5001ecf9156a68644e82374ce5833bc25b": "EM OV TLS CA - G3B (eMudhra)", + "5f1ded4dd590af74631290cf073b122a2aac6a1fa479778a0bd43579bd832699": "EM OV TLS CA - G3B (Sectigo)", + "e7568fe6e12331f4fd7de61da4c7559d0f8eb192a218d4741a3ba75184333838": "EM EV TLS CA - G3B (eMudhra)", + "579480aea166290bd3c643589e795fc0d0e54e86505d180f1dff99efd3199512": "EM EV TLS CA - G3B (Sectigo)", + "0164d7ceae5bb909c272af151ff44b1177ae9b47d9424b94f1d9558c30934d19": "EM DV TLS CA - G3C (eMudhra)", + "62b7f9868263f72f6911fc78ceaf72c1979158b7368bf648683bb2aa39b02e72": "EM DV TLS CA - G3C (Sectigo)", + "cb6e15165ac802e9b3f993f16c25d41f4e81c21ebde71ae1b012ac00bf9c7c3f": "EM OV TLS CA - G3C (eMudhra)", + "7485b51c0ed192878fa1d2f948fb529ae1842cc5086c4b67f7be5b2039de9489": "EM OV TLS CA - G3C (Sectigo)", + "432ec25f0d28de9006d97bf752f17bbb2cd41f669933fcb072d9a1148d0261f3": "EM EV TLS CA - G3C (eMudhra)", + "fd9d5c2dbbd4995ab69b8d50c439d9527b00529199c2fd01b2174eb58f07021f": "EM EV TLS CA - G3C (Sectigo)", + "9e39de2881273acc225a149c72ba3e0f51424963669ab633ee1dc7df5db067b4": "EM OV CS CA - G3B (eMudhra)", + "a496e23ac0cab8155fc9b117b97eae368efcbb9041aec8804d5a308d2d01c2d8": "EM OV CS CA - G3B (Sectigo)", + "4b2460a466f95fd40113bcdcca2ceb29b3b093921de45474da7df48874c630b1": "EM OV CS CA - G3C (eMudhra)", + "84112972b8e59d14560ae7e7119654474aad119990cf3722bfc1e3b91615e1fd": "EM OV CS CA - G3C (Sectigo)" + }, + "certificates": { + "0164d7ceae5bb909c272af151ff44b1177ae9b47d9424b94f1d9558c30934d19": "-----BEGIN CERTIFICATE-----\nMIIDaDCCAu6gAwIBAgIPWIsuPCu4zaE+GhIJJUMhMAoGCCqGSM49BAMDMFYxCzAJ\nBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVk\nMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM1\nMDlaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVk\naHJhIEluYzEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0C\nAQYFK4EEACIDYgAEOnp4YsjETTJN3WkDOmeCwO/u/kBV5ooSP6A3G6O9kZRRTuFy\npDPjYw8CvRiAN9nlFJ+bd9uUooHfmHsv6oiLdL3+j9pMEJnubKGVZv4+8OX2sJpV\nbMyLpQVh3xsqRB0Bo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2L\nWUIgkFQwHQYDVR0OBBYEFK5+EFw8KZbBqEcnBVnx4D3U5QrzMB0GA1UdJQQWMBQG\nCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCow\nKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYB\nBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYY\naHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVw\nb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQy5wN2MwNwYDVR0f\nBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5j\ncmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0E\nAwMDaAAwZQIxAMEN2HWMocbuWdy1d33pH6JpMvP/V/DIS5PtZbv1Q7TXEbxrcyxg\nyuddOJscYDJJlAIwB4sH1BpbScDzkdD4tJBSwrG/1Mx82vb8YaySehNuu1+1oXNd\nAEKwaJ0mVzScy2+6\n-----END CERTIFICATE-----\n", + "21f57f83631a93c98e867050075746b3361e0433c9e0e969462852476784f6f0": "-----BEGIN CERTIFICATE-----\nMIICPzCCAcagAwIBAgIUH0gWfBRnOiSjcduti/cukjDGAjIwCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIE9WIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMjIyNDJaFw0yNzA1MDMwMjI3NDJa\nMDMxMTAvBgNVBAMMKG92LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxl\nbS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQ3EfjVE+6fUm9gr7UoUv6N\nYPttIdILR9OMZnAdTj+RWT1Z4fp7dpC5mat1gbIo4J6EDelcOQQz5/oGeB3gFJKw\no4GrMIGoMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoG\nCCsGAQUFBwMBMDMGA1UdEQQsMCqCKG92LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZl\nLWEtcHJvYmxlbS5jb20wHQYDVR0OBBYEFKzQPp6gTBOWb0TBxZbkL5zsQn3oMB8G\nA1UdIwQYMBaAFJvYZVGv5+csN60I7mkZ2b83hK4KMAoGCCqGSM49BAMCA2cAMGQC\nMARKrRRBEWY2Ix1d80DGUvC7bWENP1Q9pf6BOGF4+84UiwV5+nAuht/+hdpAanWB\nsQIwJF2dW8tpPN6lTm1kDerzG6yJAJOz4drotaTthMF2+efuv5W2FL7Dr8qZSd1z\n201o\n-----END CERTIFICATE-----\n", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd": "-----BEGIN CERTIFICATE-----\nMIIDdDCCAvmgAwIBAgIQAJYaPvsTWVdu0EAFtGhKWzAKBggqhkjOPQQDAzBrMQsw\nCQYDVQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhy\nYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290\nIENBIC0gRzMwHhcNMjQwNzExMTU0MzM2WhcNMzkwNzA5MTgzMDAwWjBWMQswCQYD\nVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEg\nMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwdjAQBgcqhkjOPQIBBgUr\ngQQAIgNiAARfD9w/q1cWxL6D9bMGH6CQ4u+gAV6GN9s+PwQx9Bl4hC+0iU8QOk8m\nWO/RwXQy10DCSxT8vSIJClHKp+Ao4qJKdYeV0SwZ7LMQziyLfxIUOY6Grja/sQFw\nLKYf8EdDXAyjggF1MIIBcTAfBgNVHSMEGDAWgBR8XQKEE9TMipuBzhccLikenEhj\nQjAdBgNVHQ4EFgQU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0lBBYwFAYIKwYB\nBQUHAwEGCCsGAQUFBwMCMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIB\nFhxodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tMHgGCCsGAQUFBwEBBGwwajAi\nBggrBgEFBQcwAYYWaHR0cDovL29jc3AuZW1zaWduLmNvbTBEBggrBgEFBQcwAoY4\naHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbS9jZXJ0cy9lbVNpZ25FQ0NSb290\nQ0FHMy5jcnQwMwYDVR0fBCwwKjAooCagJIYiaHR0cDovL2NybC5lbXNpZ24uY29t\nP1Jvb3RDQUczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIB\nATAKBggqhkjOPQQDAwNpADBmAjEAt9CIpRGp46QHQYDUBnLKw+vvK0/h9vmilEAg\nTroUl/anH4x2rXHqiCtBTBfgKDegAjEA8WpG1LTZdAIL0FSYqxXeREaeTSxEofM0\n1jJH3jGvCXC79LU9UPJ2o3TU+UIj2gMV\n-----END CERTIFICATE-----\n", + "346cdd962e97189b59aee5526d1ae8fa714bb1a64094dfa67bef768ee93f36b5": "-----BEGIN CERTIFICATE-----\nMIICODCCAb+gAwIBAgIUbWvtwSNK/Qro223vAqyz323+VBAwCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBEViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDA1OTM2WhcNMjcwNTAzMDEwNDM2\nWjAvMS0wKwYDVQQDDCRkdi1nM2MtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxlbS5j\nb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAScdBQpNUOOCcag6YEVFccNRSFe\nUIl02iH53vDfoWPJFS005WzqF3CPDclB/kygP2gWF4pg/r5gPYdnI3/6Vdzro4Gn\nMIGkMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsG\nAQUFBwMBMC8GA1UdEQQoMCaCJGR2LWczYy1lbXVkaHJhLndlLWhhdmUtYS1wcm9i\nbGVtLmNvbTAdBgNVHQ4EFgQUBj2yGO/TzDOE/u/6QeUvcW8y1iMwHwYDVR0jBBgw\nFoAUrn4QXDwplsGoRycFWfHgPdTlCvMwCgYIKoZIzj0EAwIDZwAwZAIwBQOOQqG9\nccdzM/7XIlXLtota1esALaLmgEN/ZweosBIvk4xabHcERZInWVIwO45NAjBos8Ue\nNwDe1X3ahdK2lkxqBEoBMtUHJN18TnJKs2nK1gmXpMbmY5oKnAQQgCEZPA4=\n-----END CERTIFICATE-----\n", + "42c4accd12c4f59597925e1a08a7a90102608543809308d50ec427e118ba1493": "-----BEGIN CERTIFICATE-----\nMIIDaDCCAu6gAwIBAgIQAKSu5njyM2K0wORgt3LyFDAKBggqhkjOPQQDAzBWMQsw\nCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRl\nZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTU1\nNDA4WhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11\nZGhyYSBCVjEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0C\nAQYFK4EEACIDYgAE557ZwEqXc8RzXlERHahawr1eZKs3S4DhLLwgOOweTw47gS/J\nhZ4eZ6+wTi+LyX90hbplqpEe6KTkW2LVzAPVa+Z4s8OMtNyTfKJGDiOEU8iSk10p\n1E+R4VYGtjzO2rRoo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2L\nWUIgkFQwHQYDVR0OBBYEFOIeL+ccGTkOVUtiwSAkVzUkiiUUMB0GA1UdJQQWMBQG\nCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCow\nKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYB\nBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYY\naHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVw\nb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQi5wN2MwNwYDVR0f\nBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5j\ncmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0E\nAwMDaAAwZQIwLmsGXi6oFMJ2klkcEMXMug6gvxQWZWkewiDWVzQyl2L6z6DuQfPO\nIbh1grvdULDCAjEAxZvLlxUavG69J6wnSjC8XBbqNIQsJ6CE2QdNOe84Ptxb6VG8\ngt2TlfJbHii+KNvK\n-----END CERTIFICATE-----\n", + "4314941cf242a4ea4149aa7e9cdf77a15de2771a79984b809fc04443fea013c8": "-----BEGIN CERTIFICATE-----\nMIICOTCCAb+gAwIBAgIUXcJ4+VeNEICJ9JbBANtIyfP96EowCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBPViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDA1OTM3WhcNMjcwNTAzMDEwNDM3\nWjAvMS0wKwYDVQQDDCRvdi1nM2MtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxlbS5j\nb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATLEvRHdoPc0Z9ovgqlYmoKTHPU\nApwd6lAp0ou1G63SzuESyBzIW6Tl64CsKzlV9ggkZjX+L9/tOx97/1wmSTYvo4Gn\nMIGkMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsG\nAQUFBwMBMC8GA1UdEQQoMCaCJG92LWczYy1lbXVkaHJhLndlLWhhdmUtYS1wcm9i\nbGVtLmNvbTAdBgNVHQ4EFgQUrytBb+px+b58Df+d5vP8GpTW+/wwHwYDVR0jBBgw\nFoAUNOrqDOvJCtr50MXi7Zt4yYeeG2wwCgYIKoZIzj0EAwIDaAAwZQIwNdoOUOKz\nMLV1OFEoMjBeUqvkQAwsajvVWu0A23qyyvYlosyMp8aFv0gxURW6Yj6DAjEAhe0z\nkOQmmqb0eyhjh7weF4fHtwMtLi4jNFA6M/EfPTfkqE91MN9fSin5uhua5xhk\n-----END CERTIFICATE-----\n", + "432ec25f0d28de9006d97bf752f17bbb2cd41f669933fcb072d9a1148d0261f3": "-----BEGIN CERTIFICATE-----\nMIIDaDCCAu6gAwIBAgIQAIAJkopkEHYBrwX6LxnxCTAKBggqhkjOPQQDAzBWMQsw\nCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRl\nZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTUz\nOTMxWhcNNDAwNDI4MTgzMDAwWjBAMQswCQYDVQQGEwJVUzEUMBIGA1UECgwLZU11\nZGhyYSBJbmMxGzAZBgNVBAMMEkVNIEVWIFRMUyBDQSAtIEczQzB2MBAGByqGSM49\nAgEGBSuBBAAiA2IABC1W6Cmm+4LGSkaX2WCvlR6QZ66R+SbTQwbaSKiq63hptqZ+\nbSdCbgRCqgOzbhmI3/Y2UU7hzwf1uks/8qBmRa59MdWGatMao9JG6wsxd+6Zy3Vm\nxRbOdICKZo+R8MR/yqOCAZUwggGRMB8GA1UdIwQYMBaAFOE6qyF88eNNVK7Ucrf9\ni1lCIJBUMB0GA1UdDgQWBBSmJV6AqU2385Z8G8eR35K87kvrXzAdBgNVHSUEFjAU\nBggrBgEFBQcDAQYIKwYBBQUHAwIwXAYDVR0gBFUwUzA5BgsrBgEEAYOOIQEAATAq\nMCgGCCsGAQUFBwIBFhxodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tMA0GCysG\nAQQBg44hAQJ4MAcGBWeBDAEBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYY\naHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVw\nb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRVZUTFNDQUczQy5wN2MwNwYDVR0f\nBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5j\ncmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0E\nAwMDaAAwZQIwBy1wTnzH4lXDTy2PD91ysLjvnlVN3JbJmc1BIMEU8TOLkYVHibMc\n9HeyonikFSfmAjEA6uE3QRxlrZTz/1/lXPTEMUhoauAtycagtlNFp79kjhvNa2/f\nAal1GNxsOutLnr0Y\n-----END CERTIFICATE-----\n", + "45a883f349aedfd5f3a649b5498ffb121712ba8b448f3f2a91dbc5a766fcddbe": "-----BEGIN CERTIFICATE-----\nMIIDzjCCAragAwIBAgIRAKAk/fS6Zlx4XOSgIEg4WLcwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJW\nMRswGQYDVQQDDBJFTSBEViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAATnntnASpdzxHNeUREdqFrCvV5kqzdLgOEsvCA47B5PDjuBL8mFnh5nr7BO\nL4vJf3SFumWqkR7opORbYtXMA9Vr5nizw4y03JN8okYOI4RTyJKTXSnUT5HhVga2\nPM7atGijggE1MIIBMTAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd\nBgNVHQ4EFgQU4h4v5xwZOQ5VS2LBICRXNSSKJRQwDgYDVR0PAQH/BAQDAgEGMBIG\nA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC\nMDEGA1UdIAQqMCgwCAYGZ4EMAQIBMA0GCysGAQQBg44hAQABMA0GCysGAQQBg44h\nAQJkMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0FB\nQUNlcnRpZmljYXRlU2VydmljZXMuY3JsMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEF\nBQcwAYYYaHR0cDovL29jc3AuY29tb2RvY2EuY29tMA0GCSqGSIb3DQEBCwUAA4IB\nAQBzQ1IoqpdotM6gFrR4Q3YrKifOLeaj5XXunldpVE/vZn2TxC2y0s8oYiLR5ICE\nwqVXSB7N6BM1Mp+Nif7Jbhb2IswAfSR4Iik8+B0iRmi7lYoNlycCgHjmsjOmZaxM\nQkPmjXAJbG5f4V41OsvJZmS7lI3r0IuvEDVX3GAZ13on6IImSs9ybNpHQ7NCqkMQ\n7wH4uAO1dtpFycgJSdHaCOhHjrox2BTYGLdpUrYwY3EOAjlZ8UGgsxs7mG+xquMl\nbN1LFURwfnkVMP3T/DBIZ/ys2+lNFyxK3FlArrV4/000uTnVB1lO8oGORYdXoT5V\nGUyRCEIDhF5TBUuKF4E3kf1h\n-----END CERTIFICATE-----\n", + "4b2460a466f95fd40113bcdcca2ceb29b3b093921de45474da7df48874c630b1": "-----BEGIN CERTIFICATE-----\nMIIDWzCCAuGgAwIBAgIPVEt/kiwxriIKSpIpFvbmMAoGCCqGSM49BAMDMFUxCzAJ\nBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVk\nMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2Mzg0\nMFoXDTQwMDQyODE4MzAwMFowPzELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRo\ncmEgSW5jMRowGAYDVQQDDBFFTSBPViBDUyBDQSAtIEczQzB2MBAGByqGSM49AgEG\nBSuBBAAiA2IABLnw4GNHAqacpqrdRDz9YsQbm6SyCSRpfXl052kaEm+5avs+NML9\nO8X0gwYfJl3CpYQ2NHMjUQQ2so8E84YjYwZ5gOJQPgJTLWVGI1pxZaJZoe9Jb3Qb\nX97neHII5Lj676OCAYswggGHMB8GA1UdIwQYMBaAFGh0A/iFyo1UUFcdFuDI3kUN\nq80tMB0GA1UdDgQWBBTub3MpjW+/p7Mbw5U09hVWRHhmtzATBgNVHSUEDDAKBggr\nBgEFBQcDAzBeBgNVHSAEVzBVMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEW\nHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDgYMKwYBBAGDjiEBAoFIMAgG\nBmeBDAEEATB0BggrBgEFBQcBAQRoMGYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3Nw\nLWMuZW1zaWduLmNvbTA+BggrBgEFBQcwAoYyaHR0cDovL3JlcG9zaXRvcnkuZW1z\naWduLmNvbS9jZXJ0cy9FTU9WQ1NDQUczQy5wN2MwNgYDVR0fBC8wLTAroCmgJ4Yl\naHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290Y3NjYWczLmNybDAOBgNVHQ8BAf8E\nBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNoADBlAjEA8yPg\n8sntOhLwV2QXJ5LEaKSpo/fnE3uOgTadK73fDhyjiqr0Io9GVuQTYqBPbQ2TAjBg\neuApmBS486+iei03lRrp139dPw6k28jjGNvWsAU7A1N86J1zPJVDEyVH+nMAWC0=\n-----END CERTIFICATE-----\n", + "579480aea166290bd3c643589e795fc0d0e54e86505d180f1dff99efd3199512": "-----BEGIN CERTIFICATE-----\nMIIDzTCCArWgAwIBAgIRANj4aF5PK9nYJAKoWhwetzEwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJW\nMRswGQYDVQQDDBJFTSBFViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAAQl26haZNOX2gsQUunqVG5Ie+/qnJ2daKDF6tz+73Wmm7XW/AH0K/vKbrQL\nzWTAdn42/9Bdl6gTxaoMbJUw55z18IZkaPE4BkIKpchRhAseJSVA55DktlTDhS7W\nSvHoZX2jggE0MIIBMDAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd\nBgNVHQ4EFgQUPtwg2YlGmId7hz/YgUeyXBXYRwYwDgYDVR0PAQH/BAQDAgEGMBIG\nA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC\nMDAGA1UdIAQpMCcwBwYFZ4EMAQEwDQYLKwYBBAGDjiEBAAEwDQYLKwYBBAGDjiEB\nAngwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20vQUFB\nQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUF\nBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB\nAJNFuAFW9Gnw/8+kCO4KIlZdnM1Sh2u+8cK0EYAHhXpb5FlGvYkcARigo5oS7VoM\n7FOYlTueV6yICZKNAYawZPJPhNCGK+rp3cQBEz6d0n8OZEYZov9lb3ll8ORgSDuf\nFVIW6Gagg/UnOamLBTOR8eD5V73SCme/B2cHUDw+m5E8cc+0Ujzz3Cn3ex5JrhXk\n2T3s8PoVzEmhTgGboFfcWO1EYhPPrYwdxPYRQIXAXUXhLEOVNL9ASbZJLFI8E1IM\nm7NwxVN2s38jWpaCUaaF0tg7hfiy7oe13rzJfHsdqldJVTTUTPWVx8TC4ju0VLet\nbAlwhwFT6AP8uIVMGUHNw24=\n-----END CERTIFICATE-----\n", + "59cf0386b56bc2414a0c06241ea9e25e01a9c93bac7a596648747f9d0ded4906": "-----BEGIN CERTIFICATE-----\nMIICQTCCAcegAwIBAgIULuWnWW6kFV6QZxp2ls8r18NCjXkwCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBPViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDIyMjQyWhcNMjcwNTAzMDIyNzQy\nWjAzMTEwLwYDVQQDDChvdi1nM2MtYWFhLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEb7+5g2YHNYwH2eQERrZc\nTmVuf/RmvzsGGMSwXmZke6vn35Ut5HME7UOOU3Muy+BSfjfpo6fjL5KOs3Ndz67d\nOKOBqzCBqDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAK\nBggrBgEFBQcDATAzBgNVHREELDAqgihvdi1nM2MtYWFhLWVtdWRocmEud2UtaGF2\nZS1hLXByb2JsZW0uY29tMB0GA1UdDgQWBBQ7oBzq3jpTuvntbIkzMV12Ps3O0TAf\nBgNVHSMEGDAWgBQ06uoM68kK2vnQxeLtm3jJh54bbDAKBggqhkjOPQQDAgNoADBl\nAjEA/Y7AZ4M2MYbN5Od9tREaUfFjZKoOjczfIKPjKNMjtGxTdL9WUXvAT4QDgktn\n/IJdAjAbORTkAXD5hggyG6K7p0MBz3lFZJb2l/LndZlrttJWBLP8QGrc44hrZZtK\nIq0d1dE=\n-----END CERTIFICATE-----\n", + "5f1ded4dd590af74631290cf073b122a2aac6a1fa479778a0bd43579bd832699": "-----BEGIN CERTIFICATE-----\nMIIDzjCCAragAwIBAgIRAOlCI73pL5FOWrmsq/2sNxAwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJW\nMRswGQYDVQQDDBJFTSBPViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAARziXqg6zz9yosSQC/68KvE4SErr43ud4rJ7i1tuD+sKAxeBWOK56Ks1xwy\nKo42HN659pxDArndjXPRzGyqsswK7k4hbOl53VGKSvS2JuVoTN1lgDftDIVOTMEv\nTPhkbQ2jggE1MIIBMTAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd\nBgNVHQ4EFgQUm9hlUa/n5yw3rQjuaRnZvzeErgowDgYDVR0PAQH/BAQDAgEGMBIG\nA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC\nMDEGA1UdIAQqMCgwCAYGZ4EMAQICMA0GCysGAQQBg44hAQABMA0GCysGAQQBg44h\nAQJuMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0FB\nQUNlcnRpZmljYXRlU2VydmljZXMuY3JsMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEF\nBQcwAYYYaHR0cDovL29jc3AuY29tb2RvY2EuY29tMA0GCSqGSIb3DQEBCwUAA4IB\nAQALRoOSvkzRa+tN5Rje9W5GiXd/qsicd9N6Gnh0SWC7VyWH+HycIqAnFCMYZznj\nCeGusomFYHE42SHWr85JDIbgzy6y3dO4i9n9w0Bx7yrXWBbDzFS8VgA3+EY4Cg7y\nmtoC3RPOJVtaXXPZ+68X804g1Mj+3vCSHj2lNPVgco0OzSIOJ/KMg1R9MLBkiKzb\nbERAGZHejY0HBhYd76IINZN8z6bBNah2UdFYxsOisLynPQUkol6gRLum024/FzVl\nCCDfnUusrkr+mAfPAvD/Vb8bhpA7jYtepuU+2mIdShj5lLlvAwZphn+ypWycuo2x\n0k9mldz2dqI9nhCiY+TTIOOR\n-----END CERTIFICATE-----\n", + "62b7f9868263f72f6911fc78ceaf72c1979158b7368bf648683bb2aa39b02e72": "-----BEGIN CERTIFICATE-----\nMIIDzzCCAregAwIBAgIRAJHzR/Glt9OZYrleDuTimdEwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIElu\nYzEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EE\nACIDYgAEOnp4YsjETTJN3WkDOmeCwO/u/kBV5ooSP6A3G6O9kZRRTuFypDPjYw8C\nvRiAN9nlFJ+bd9uUooHfmHsv6oiLdL3+j9pMEJnubKGVZv4+8OX2sJpVbMyLpQVh\n3xsqRB0Bo4IBNTCCATEwHwYDVR0jBBgwFoAUoBEKIz6W8Qfs4q8p74Klf9AwpLQw\nHQYDVR0OBBYEFK5+EFw8KZbBqEcnBVnx4D3U5QrzMA4GA1UdDwEB/wQEAwIBBjAS\nBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD\nAjAxBgNVHSAEKjAoMAgGBmeBDAECATANBgsrBgEEAYOOIQEAATANBgsrBgEEAYOO\nIQECZDBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9B\nQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDA0BggrBgEFBQcBAQQoMCYwJAYIKwYB\nBQUHMAGGGGh0dHA6Ly9vY3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQsFAAOC\nAQEAr7b6Mx11TIJgVzHt6EH5qGNWeiiqoMrp8E5/k8s+jvupx1n083pZ5XOzUzND\nwEUe0W4C4FIGUHNQ4XDwqOw0yK42xqm2dvCKgsko2UvDlT0haZVtTCeWMQOe5J/J\nziaMfZwsZNfZ3aaI6XPtWDZtwY8AaNfyaeCpbtm5GcGnyNWk4jSNpbtpo6NjfXuP\nCFOm3rTjJ4zytP7Tso6iMm0L01cYFOlTmoPv4515R2r3SPnKDtijoB/D/vgxzojv\nDafgyG9afX8+laO9NbMMz4sqC1vZx3984i/sBNFiugm3Vvdq5OdJRjuGC1QILIs6\ndg/K3i8VQqRpbfaM9kIlMzZ3dQ==\n-----END CERTIFICATE-----\n", + "66374ae64962c437bd815a2aae546c76e30c2eafac8335192c8e517c15a246ed": "-----BEGIN CERTIFICATE-----\nMIICNzCCAb6gAwIBAgIUPfthO0PYGAYKs9lQ4gp1IOkOr2MwCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIEVWIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMDU5MzZaFw0yNzA1MDMwMTA0MzZa\nMC8xLTArBgNVBAMMJGV2LWczYi1lbXVkaHJhLndlLWhhdmUtYS1wcm9ibGVtLmNv\nbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPZPwlA3io/AX/pMwu0dNl9WMxrr\nC9j/xMLHgG72cWVzoX62REmvp9zyXQ04kHP9/sW6KnCcm5ZZ5i4YoJbDPASjgacw\ngaQwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB\nBQUHAwEwLwYDVR0RBCgwJoIkZXYtZzNiLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMB0GA1UdDgQWBBSvX4QR4ERW5uNUaMq0yYzXCZf0ZjAfBgNVHSMEGDAW\ngBQ+3CDZiUaYh3uHP9iBR7JcFdhHBjAKBggqhkjOPQQDAgNnADBkAjACy/uJR01M\n/O9IEZyRHX1LSU28bmg542K0IX9jDRTrZXUqkDWfiZIL7xfnrRrBpHoCMFbBIaOa\nWwWtFtDg9ZqQvktsWS950u5qfW1sChgWNya/E3IMKFgtoHm2xrZsmfwLSQ==\n-----END CERTIFICATE-----\n", + "7485b51c0ed192878fa1d2f948fb529ae1842cc5086c4b67f7be5b2039de9489": "-----BEGIN CERTIFICATE-----\nMIIDzzCCAregAwIBAgIRAI1aSBGTnvRH13ERzdmEmukwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIElu\nYzEbMBkGA1UEAwwSRU0gT1YgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EE\nACIDYgAEE4JRzVKskpjByKrZdzId65fnCb0LTKCspV3IrVHc/J0ViaFZfjpRIOHv\n1jHGPhg1ysrimGmmLhYx6KKBgatWYW3EXZGKvAnzqw5jz3kqpNztc4e+N7ulaVSf\nHAKycO1no4IBNTCCATEwHwYDVR0jBBgwFoAUoBEKIz6W8Qfs4q8p74Klf9AwpLQw\nHQYDVR0OBBYEFDTq6gzryQra+dDF4u2beMmHnhtsMA4GA1UdDwEB/wQEAwIBBjAS\nBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD\nAjAxBgNVHSAEKjAoMAgGBmeBDAECAjANBgsrBgEEAYOOIQEAATANBgsrBgEEAYOO\nIQECbjBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9B\nQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDA0BggrBgEFBQcBAQQoMCYwJAYIKwYB\nBQUHMAGGGGh0dHA6Ly9vY3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQsFAAOC\nAQEAHlbhZpjhkPEAQVhPHRtBDaHqiisVD20pQ1TUeSx3l62Qe7NQ0b7vWQy7DN0i\nKOu7BGpP5EnIjvXbz9FhpfPJ4FhKEt3tQ51rqXz2oD8vRaryvExWPobDUz3j7HN6\nOzBsVMIe3PD7AAkpky0x8zMSi/ep5R9OMzybmNptbtppyuB5ytgq+h7lQdRIP+CQ\nTaJX3qTgfMqzntYre5GljmEB5hU2/rSn8zoSU2APkJidlZFpghtgbD4I2VRHZvCw\nFFr/SrFM9pwrgbJNZ1MtyTSVgI5NlNb0aMfLR+/r6jJt3ROeY2yh2QQ2dx+Yqgdb\nuL5ChFWOQU+ErSDnUq7yr+pjBQ==\n-----END CERTIFICATE-----\n", + "7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d": "-----BEGIN CERTIFICATE-----\nMIICKTCCAa+gAwIBAgIQDnYGcvFDRZ/I/gqwvAXjlDAKBggqhkjOPQQDAzBWMQsw\nCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRl\nZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjQwMjA4MDgx\nOTMyWhcNNDkwMjA3MDgxOTMyWjBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11\nZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3Qg\nVExTIENBIC0gRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARfD9w/q1cWxL6D9bMG\nH6CQ4u+gAV6GN9s+PwQx9Bl4hC+0iU8QOk8mWO/RwXQy10DCSxT8vSIJClHKp+Ao\n4qJKdYeV0SwZ7LMQziyLfxIUOY6Grja/sQFwLKYf8EdDXAyjQjBAMB0GA1UdDgQW\nBBThOqshfPHjTVSu1HK3/YtZQiCQVDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/\nBAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjBhCFTkly7Yl7+s4hiw+k6ZJTXT2AiI\nrvmsmfqPr5E7ecwXb6kXA5LTFd6rjwTuV3sCMQC9w6ZSbVq1mveMO6dKFziOzClT\nKCT2RFT6yXtq6nzgRICgBmiXbJAXsmZ8yAwjyWc=\n-----END CERTIFICATE-----\n", + "7e02622ab6d745c1edb4b431bb1c551e667305c3dd3eb0c7cc6dfdb523807424": "-----BEGIN CERTIFICATE-----\nMIICQDCCAcagAwIBAgIUU3mgjls/8ZJ8upmukdlSQR1IF/UwCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIERWIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMjIyNDFaFw0yNzA1MDMwMjI3NDFa\nMDMxMTAvBgNVBAMMKGR2LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxl\nbS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATK3sfcOxxtM2Vdq9Qbw/GR\nNsHlJcnqINEInXO3fkWhw/FYT80hkqGk0jSN/EKib3YZJsr5GyfkYcpEiA2rwMgm\no4GrMIGoMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoG\nCCsGAQUFBwMBMDMGA1UdEQQsMCqCKGR2LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZl\nLWEtcHJvYmxlbS5jb20wHQYDVR0OBBYEFDggfJFz1eeToKmzFBSYkMCvpAu0MB8G\nA1UdIwQYMBaAFOIeL+ccGTkOVUtiwSAkVzUkiiUUMAoGCCqGSM49BAMCA2gAMGUC\nMDqDDOt7K313QIH0rhn/ZKnzFu8+/9W8OlTuFtDfVCTRDfRmqdyfsCfLsEk7xqHq\nuQIxAJqyLERABmU98U46OCuT5bp5h0ytuSEIGM7orTpKci5S1u1B/AOmwv162U7R\nTcVPHA==\n-----END CERTIFICATE-----\n", + "84112972b8e59d14560ae7e7119654474aad119990cf3722bfc1e3b91615e1fd": "-----BEGIN CERTIFICATE-----\nMIIDxTCCAq2gAwIBAgIRANal72E7OQ8y75zfNJ93KEEwDQYJKoZIhvcNAQELBQAw\nezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G\nA1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV\nBAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y\nODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIElu\nYzEaMBgGA1UEAwwRRU0gT1YgQ1MgQ0EgLSBHM0MwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAAS58OBjRwKmnKaq3UQ8/WLEG5uksgkkaX15dOdpGhJvuWr7PjTC/TvF9IMG\nHyZdwqWENjRzI1EENrKPBPOGI2MGeYDiUD4CUy1lRiNacWWiWaHvSW90G1/e53hy\nCOS4+u+jggEsMIIBKDAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd\nBgNVHQ4EFgQU7m9zKY1vv6ezG8OVNPYVVkR4ZrcwDgYDVR0PAQH/BAQDAgEGMBIG\nA1UdEwEB/wQIMAYBAf8CAQAwEwYDVR0lBAwwCgYIKwYBBQUHAwMwMgYDVR0gBCsw\nKTAIBgZngQwBBAEwDQYLKwYBBAGDjiEBAAEwDgYMKwYBBAGDjiEBAoFIMEMGA1Ud\nHwQ8MDowOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0FBQUNlcnRpZmlj\nYXRlU2VydmljZXMuY3JsMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEFBQcwAYYYaHR0\ncDovL29jc3AuY29tb2RvY2EuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBgGbZ+NDkn\n2Fr5dCVs/51+6cT5iEPP1stglX4TI2UaNh9LmPdltxXw3crV8gK5L8i+d+ScGibF\ny0obtBHNTldXDgLnAHIzY3n+pceyKj7O2ae9CYnyuu5RBifeI3XYKowWE+oXnVGF\nOy0CvCJ/JsdP6aYgoCa9hGnAYicYQQL6MOMQZ5PuhcSM8t4giIzrpgrEkUP93GYE\nemDG5Z8BYwKs80lLeQNBnwg/VHJ3iEF/o84RXYwrlrKqJ+33C094o6LNikMJCqOK\n/WYHZSR8Xyhf8GXFAj5pZuI6Bcpug2V/E7i42j8OBhc25aNYDqK+B0qmb90sjNUB\ng3tJeSIb/599\n-----END CERTIFICATE-----\n", + "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b": "-----BEGIN CERTIFICATE-----\nMIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG\nEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo\nbm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g\nRzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ\nTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s\nb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw\ndjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0\nWXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS\nfvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB\nzhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq\nhkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB\nCUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD\n+JbNR6iC8hZVdyR+EhCVBCyj\n-----END CERTIFICATE-----\n", + "9200d153c3cac272cc06bf68f9ea163fc24659fc3331e77668236fe00645ae52": "-----BEGIN CERTIFICATE-----\nMIICQjCCAcegAwIBAgIUMH3z62Z6agEULBZqrfhpw9/ZpCAwCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBEViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDIyMjQxWhcNMjcwNTAzMDIyNzQx\nWjAzMTEwLwYDVQQDDChkdi1nM2MtYWFhLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAMPpHsQqEqIerrj/durO\nEN7gkUVJSVgIPN6+VJ2sEGXDyqXAN1fnNzGuASP/4WEr0RIhht3cYvuUYp2SYofr\nw6OBqzCBqDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAK\nBggrBgEFBQcDATAzBgNVHREELDAqgihkdi1nM2MtYWFhLWVtdWRocmEud2UtaGF2\nZS1hLXByb2JsZW0uY29tMB0GA1UdDgQWBBT8CNQz4OLVnIwrQYZBF+++9jTCvzAf\nBgNVHSMEGDAWgBSufhBcPCmWwahHJwVZ8eA91OUK8zAKBggqhkjOPQQDAgNpADBm\nAjEAp3ZLdfBvKcZxgR2p8LZaxWzP6uB7zU/EZ9BA49Ds2P5ByYkRodwK5qcwt4J/\nVL/vAjEAnnxrcrDX7SzoJxuho604GBh6eyBLJLCdmcy+qb+LA3u5oxP/OjAzCA4x\nv1kw6Tm4\n-----END CERTIFICATE-----\n", + "98b896625c0c230adeec3eb2cf6aad5001ecf9156a68644e82374ce5833bc25b": "-----BEGIN CERTIFICATE-----\nMIIDZjCCAu2gAwIBAgIPbK2jZ/scEACIEg6iAtVoMAoGCCqGSM49BAMDMFYxCzAJ\nBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVk\nMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNjAw\nMTJaFw00MDA0MjgxODMwMDBaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVk\naHJhIEJWMRswGQYDVQQDDBJFTSBPViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIB\nBgUrgQQAIgNiAARziXqg6zz9yosSQC/68KvE4SErr43ud4rJ7i1tuD+sKAxeBWOK\n56Ks1xwyKo42HN659pxDArndjXPRzGyqsswK7k4hbOl53VGKSvS2JuVoTN1lgDft\nDIVOTMEvTPhkbQ2jggGWMIIBkjAfBgNVHSMEGDAWgBThOqshfPHjTVSu1HK3/YtZ\nQiCQVDAdBgNVHQ4EFgQUm9hlUa/n5yw3rQjuaRnZvzeErgowHQYDVR0lBBYwFAYI\nKwYBBQUHAwEGCCsGAQUFBwMCMF0GA1UdIARWMFQwOQYLKwYBBAGDjiEBAAEwKjAo\nBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTANBgsrBgEE\nAYOOIQECbjAIBgZngQwBAgIwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhho\ndHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBv\nc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1PVlRMU0NBRzNCLnA3YzA3BgNVHR8E\nMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNy\nbDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQD\nAwNnADBkAjBAk+5BizdJhrED3wb7F2Gm7bWI60dlNd075nghT2TrR4NWKZtJZcbc\n28xxIY/NONICMF6mPdsjosaqmvEMWqKcBh4vsIvXSKnQj5o8zrFB4oKS+accjYTe\n8Dz3bQjauvy9uw==\n-----END CERTIFICATE-----\n", + "9dcb21b1841d75313b3c888f4bb9cc01f8f51379a76fbaa241ca492b668839e7": "-----BEGIN CERTIFICATE-----\nMIICOTCCAb6gAwIBAgIUY1R/PHrtXShV0uhLJ5EyjsHvMi0wCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIERWIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMDU5MzZaFw0yNzA1MDMwMTA0MzZa\nMC8xLTArBgNVBAMMJGR2LWczYi1lbXVkaHJhLndlLWhhdmUtYS1wcm9ibGVtLmNv\nbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFpE24ACBFHQLzpg+7D767thJ3+Q\nNIoJJugUMT/OftEz6G6uDdxlS+9VfXYBDglHjJirjEPFNglOGJoQNQuiz7Gjgacw\ngaQwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB\nBQUHAwEwLwYDVR0RBCgwJoIkZHYtZzNiLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMB0GA1UdDgQWBBT4r5v4etYNuG21BqBqs273OkcHiDAfBgNVHSMEGDAW\ngBTiHi/nHBk5DlVLYsEgJFc1JIolFDAKBggqhkjOPQQDAgNpADBmAjEAlxNF+/8l\nAPpgibawACy15yn5bcvTksz2Zw0R9z9nUK9e3MSSk+hXagg7LVb7V4NtAjEA4qyR\nlaFPkaokrnTjqQieq8CgDQ6tIz3FjBvj3D7vkhKXHaml6zD2zT5ih2Y+lD2h\n-----END CERTIFICATE-----\n", + "9e39de2881273acc225a149c72ba3e0f51424963669ab633ee1dc7df5db067b4": "-----BEGIN CERTIFICATE-----\nMIIDWzCCAuCgAwIBAgIPeBiTP4+FtGGaKzX7qD0KMAoGCCqGSM49BAMDMFUxCzAJ\nBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVk\nMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2MDgy\nN1oXDTQwMDQyODE4MzAwMFowPjELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRo\ncmEgQlYxGjAYBgNVBAMMEUVNIE9WIENTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYF\nK4EEACIDYgAEqrsPsGZqz88B+tUr/P1MK/LLPDSX3nOVDlgZth/mlQmHQcjEkUOb\nNKBDHK/+W/m0wcjorgAB+fTcitgsWfNjnpbNbD1NsM7+wmw9YlF7N3YTcnQ/jord\nd7EAZPKr5eBQo4IBizCCAYcwHwYDVR0jBBgwFoAUaHQD+IXKjVRQVx0W4MjeRQ2r\nzS0wHQYDVR0OBBYEFHn66OzP2w11jaRY1p+27dx8Sd7mMBMGA1UdJQQMMAoGCCsG\nAQUFBwMDMF4GA1UdIARXMFUwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYc\naHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTAOBgwrBgEEAYOOIQECgUgwCAYG\nZ4EMAQQBMHQGCCsGAQUFBwEBBGgwZjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3At\nYi5lbXNpZ24uY29tMD4GCCsGAQUFBzAChjJodHRwOi8vcmVwb3NpdG9yeS5lbXNp\nZ24uY29tL2NlcnRzL0VNT1ZDU0NBRzNCLnA3YzA2BgNVHR8ELzAtMCugKaAnhiVo\ndHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3Rjc2NhZzMuY3JsMA4GA1UdDwEB/wQE\nAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMAoGCCqGSM49BAMDA2kAMGYCMQC/zr03\nYEELsNKqzUtH2dVFi4BaKFdyrr/6G1dIWp8KAVgsw2/u4lyyboUDaC9jDvICMQC9\nHTxku84tNX9dmHOtH7aC2Nu/JM0jTf6KjZAvovCJzzv1IFWSZpVLZibynMLXOrM=\n-----END CERTIFICATE-----\n", + "a496e23ac0cab8155fc9b117b97eae368efcbb9041aec8804d5a308d2d01c2d8": "-----BEGIN CERTIFICATE-----\nMIIDwzCCAqugAwIBAgIQQeMrc6wYXWzg10gLrPhOaTANBgkqhkiG9w0BAQsFADB7\nMQswCQYDVQQGEwJHQjEbMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYD\nVQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UE\nAwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTI1MDUxNDAwMDAwMFoXDTI4\nMTIzMTIzNTk1OVowPjELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYx\nGjAYBgNVBAMMEUVNIE9WIENTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACID\nYgAEqrsPsGZqz88B+tUr/P1MK/LLPDSX3nOVDlgZth/mlQmHQcjEkUObNKBDHK/+\nW/m0wcjorgAB+fTcitgsWfNjnpbNbD1NsM7+wmw9YlF7N3YTcnQ/jordd7EAZPKr\n5eBQo4IBLDCCASgwHwYDVR0jBBgwFoAUoBEKIz6W8Qfs4q8p74Klf9AwpLQwHQYD\nVR0OBBYEFHn66OzP2w11jaRY1p+27dx8Sd7mMA4GA1UdDwEB/wQEAwIBBjASBgNV\nHRMBAf8ECDAGAQH/AgEAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMDIGA1UdIAQrMCkw\nCAYGZ4EMAQQBMA0GCysGAQQBg44hAQABMA4GDCsGAQQBg44hAQKBSDBDBgNVHR8E\nPDA6MDigNqA0hjJodHRwOi8vY3JsLmNvbW9kb2NhLmNvbS9BQUFDZXJ0aWZpY2F0\nZVNlcnZpY2VzLmNybDA0BggrBgEFBQcBAQQoMCYwJAYIKwYBBQUHMAGGGGh0dHA6\nLy9vY3NwLmNvbW9kb2NhLmNvbTANBgkqhkiG9w0BAQsFAAOCAQEAIh0OzWsdBYhY\n7+yxPGyosRL+uUSN6/Mrd30fBuougs5FVWQ3cRgLhImnUUgXADY9buugWhMmfT3c\nyU5T1MFXZXClCILm5mfg+oqm/ikUWJl3Ayf2GvYf2O++6vzqvkn418nI5Lpdt0O7\nLi524YS/3CZHDA+zZQ7rrHpZXgQ9Hmp8AUQ4FneaGjNanFoYxiZxwCyW5z4u7AoK\n0bmOVqKtcDFoVNz5++G9TYDJdhWOXmKKgzqRo1MK0PjazNT7MGHqknP93SqMVJ1g\nmX/qNWxYteAjlvFanQxBNX/+TRxDjuFsEHO/7BvqSpIIck0UEEfizhN8zNpWLbcl\nEqVgl0g39A==\n-----END CERTIFICATE-----\n", + "bc88231a840e54cced148da48806169e8d0b1c0a96267a427aeccda9d10e6145": "-----BEGIN CERTIFICATE-----\nMIICQDCCAcagAwIBAgIUUYu3O8bpN1GPvCIHqWpT/FYUNDowCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIEVWIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMjIyNDFaFw0yNzA1MDMwMjI3NDFa\nMDMxMTAvBgNVBAMMKGV2LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxl\nbS5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASjDLpnVdmvXM6R3UB2o42a\n1GnrYfxRXo7UII36mBl9mIS7p9MkAryhPdxitFHkgTrrbypVtzpydNtJIGGGXUwE\no4GrMIGoMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoG\nCCsGAQUFBwMBMDMGA1UdEQQsMCqCKGV2LWczYi1hYWEtZW11ZGhyYS53ZS1oYXZl\nLWEtcHJvYmxlbS5jb20wHQYDVR0OBBYEFEVBY1ecTN9bLMDl4ifdnN2WZL+WMB8G\nA1UdIwQYMBaAFD7cINmJRpiHe4c/2IFHslwV2EcGMAoGCCqGSM49BAMCA2gAMGUC\nMQCyvr9gktgmtsuTuFleFV+YExNNKLf3GZieuUprQcERrzh798oYKavTqZeYf9Kz\nRhECMCVs1B7KPh7/lgHRly14Z8RJA8ApR7e2cRREW5uPKFOSDlZ/RIezxsfEzsQ4\nlAODeA==\n-----END CERTIFICATE-----\n", + "bcd6ab59ea95fd520bfe7709679b1f682abbd79c77367e84e25415b91de37753": "-----BEGIN CERTIFICATE-----\nMIICQDCCAcegAwIBAgIUTMUrGzSU+0BySFdrN3ALZq9Spv8wCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBFViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDIyMjQyWhcNMjcwNTAzMDIyNzQy\nWjAzMTEwLwYDVQQDDChldi1nM2MtYWFhLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglClDbotLFFeY9SpryT4\nk958tvriY2DDnaPsmjQaecRhB6+L/8hLw6FMt+HT8SLvjIZnFY+AH92cBjedcNST\nAKOBqzCBqDAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAK\nBggrBgEFBQcDATAzBgNVHREELDAqgihldi1nM2MtYWFhLWVtdWRocmEud2UtaGF2\nZS1hLXByb2JsZW0uY29tMB0GA1UdDgQWBBSVtGbn6xUT1iosUeaGA15Nq640aTAf\nBgNVHSMEGDAWgBSmJV6AqU2385Z8G8eR35K87kvrXzAKBggqhkjOPQQDAgNnADBk\nAjAg7b5bIK9X50QEX0U6G8fQh12BnFQ079Iluzd0Y2TJWCjtVXZi3nOciM8RzNKM\ntZECMCBf0JOyMt8RNS93Wj8kYbWkCtoKoUrqx7G3JTfGReKci+DdTvkF8/3F6doi\n9KB22Q==\n-----END CERTIFICATE-----\n", + "cb6e15165ac802e9b3f993f16c25d41f4e81c21ebde71ae1b012ac00bf9c7c3f": "-----BEGIN CERTIFICATE-----\nMIIDaDCCAu6gAwIBAgIPKWkmwiejoVJjkrGUgqUEMAoGCCqGSM49BAMDMFYxCzAJ\nBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVk\nMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM3\nNDFaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVk\naHJhIEluYzEbMBkGA1UEAwwSRU0gT1YgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0C\nAQYFK4EEACIDYgAEE4JRzVKskpjByKrZdzId65fnCb0LTKCspV3IrVHc/J0ViaFZ\nfjpRIOHv1jHGPhg1ysrimGmmLhYx6KKBgatWYW3EXZGKvAnzqw5jz3kqpNztc4e+\nN7ulaVSfHAKycO1no4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2L\nWUIgkFQwHQYDVR0OBBYEFDTq6gzryQra+dDF4u2beMmHnhtsMB0GA1UdJQQWMBQG\nCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCow\nKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYB\nBAGDjiEBAm4wCAYGZ4EMAQICMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYY\naHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVw\nb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZUTFNDQUczQy5wN2MwNwYDVR0f\nBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5j\ncmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0E\nAwMDaAAwZQIxAMSZt2xsPeumRdeSMSQ3CbzIZeQ/q7GDCWfeE1Pq+mhgA6gRM44c\nNVF3xJ+wsxJk+wIwFJbaDMokD8Ei6eNOiA8yUMYvfzQyngmrhZQMdPCK1SewrR79\n4rSb9vGqxZzelQth\n-----END CERTIFICATE-----\n", + "ccd80467f9cac38f115dcb6e4f57e4d74f5efa519814ea934aeb3be53b83a191": "-----BEGIN CERTIFICATE-----\nMIICOTCCAb+gAwIBAgIUOcw5N4XGLiB/3GvB05bJuGoZPR0wCgYIKoZIzj0EAwIw\nQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRswGQYDVQQDDBJF\nTSBFViBUTFMgQ0EgLSBHM0MwHhcNMjYxMDIwMDA1OTM3WhcNMjcwNTAzMDEwNDM3\nWjAvMS0wKwYDVQQDDCRldi1nM2MtZW11ZGhyYS53ZS1oYXZlLWEtcHJvYmxlbS5j\nb20wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQo7U02MAUO8EEMomXwIl+kSQzm\nb+nvXKIZ/hC4rZ1GpXchv4SzTrV1SB/Qg+pD6OBEXbBBnPmQNkePNQTlujqRo4Gn\nMIGkMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsG\nAQUFBwMBMC8GA1UdEQQoMCaCJGV2LWczYy1lbXVkaHJhLndlLWhhdmUtYS1wcm9i\nbGVtLmNvbTAdBgNVHQ4EFgQUqxe+7qS+MxNyioJZhQsL5MEEw6swHwYDVR0jBBgw\nFoAUpiVegKlNt/OWfBvHkd+SvO5L618wCgYIKoZIzj0EAwIDaAAwZQIwC/CxSIov\nwcar3sV98B8cL+QuR2kkJdLdXsFKzg/HDSJ1MOjqTGm924LB6r9nl0f7AjEAzcwr\nWQOYY9eycqIz+LG2daYS6LOctj+CjoAk/CLugWPxGxp4Aerfau2P1z1h5jAA\n-----END CERTIFICATE-----\n", + "d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4": "-----BEGIN CERTIFICATE-----\nMIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb\nMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow\nGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj\nYXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL\nMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE\nBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM\nGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP\nADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua\nBtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe\n3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4\nYgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR\nrOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm\nez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU\noBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF\nMAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v\nQUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t\nb2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF\nAAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q\nGE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz\nRt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2\nG9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi\nl2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3\nsmPi9WIsgtRqAEFQ8TmDn5XpNpaYbg==\n-----END CERTIFICATE-----\n", + "e7568fe6e12331f4fd7de61da4c7559d0f8eb192a218d4741a3ba75184333838": "-----BEGIN CERTIFICATE-----\nMIIDaDCCAu2gAwIBAgIQAMfEHv6S2CXwXq6EGkqCwTAKBggqhkjOPQQDAzBWMQsw\nCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRl\nZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTYw\nMzMxWhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11\nZGhyYSBCVjEbMBkGA1UEAwwSRU0gRVYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0C\nAQYFK4EEACIDYgAEJduoWmTTl9oLEFLp6lRuSHvv6pydnWigxerc/u91ppu11vwB\n9Cv7ym60C81kwHZ+Nv/QXZeoE8WqDGyVMOec9fCGZGjxOAZCCqXIUYQLHiUlQOeQ\n5LZUw4Uu1krx6GV9o4IBlTCCAZEwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2L\nWUIgkFQwHQYDVR0OBBYEFD7cINmJRpiHe4c/2IFHslwV2EcGMB0GA1UdJQQWMBQG\nCCsGAQUFBwMBBggrBgEFBQcDAjBcBgNVHSAEVTBTMDkGCysGAQQBg44hAQABMCow\nKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYB\nBAGDjiEBAngwBwYFZ4EMAQEwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhho\ndHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBv\nc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1FVlRMU0NBRzNCLnA3YzA3BgNVHR8E\nMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNy\nbDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQD\nAwNpADBmAjEA9Jq999oaOUKhFwopw33iCmt0OHIn3gjT43GfuOHR/YckXdwoNdDA\nlIEhz0NgRO3/AjEAvY/a3OMvktIZLYjn+WLKmp0QhrIMV+e6EpS7j4GVeOKouNp6\nMN/P+ByGeiRMghzB\n-----END CERTIFICATE-----\n", + "f2edaaff3d9b701fb4d14c9f285f8c0bad90fc3bcc1fa07d47f4718edaba34a5": "-----BEGIN CERTIFICATE-----\nMIICKDCCAa2gAwIBAgIQAY/OySfBFvWcmX7cDM0uKzAKBggqhkjOPQQDAzBVMQsw\nCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRl\nZDEfMB0GA1UEAwwWZW1TaWduIFJvb3QgQ1MgQ0EgLSBHMzAeFw0yNDAyMDgwODMz\nMTRaFw00OTAyMDcwODMzMTRaMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVk\naHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBD\nUyBDQSAtIEczMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAESl9kM5D0VA/Og76dz5Xz\nw1hnGQ4fSR9993/qBkkvI48pXQVKOdyHCgAZThgWejQ8OpFRiTISjpabn4tfRFOY\nuzhApR/XxGbkg9X+2cEtbyghF9CdoF91W22V/R2EVe4xo0IwQDAdBgNVHQ4EFgQU\naHQD+IXKjVRQVx0W4MjeRQ2rzS0wDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF\nMAMBAf8wCgYIKoZIzj0EAwMDaQAwZgIxAK9OlACrFod3UgWY1ai+DF9sGFhahU59\nrv4jJyV01QaxGn0HErZ8gjHkngqe+tcFogIxAPTdW0hBmrGjhxBWdv9mZNdVFiJ4\nlrZ3ahvimsLhTQpnDBgiGKv6Ov1Fx/CE6yjIAg==\n-----END CERTIFICATE-----\n", + "f3c3c15bc7edf2b3ebdc922af112f2a0c414eb509901099ef0488c0942e77efe": "-----BEGIN CERTIFICATE-----\nMIICODCCAb6gAwIBAgIUPXjC97k7buQHZ7p5ehDU8QNyrR4wCgYIKoZIzj0EAwIw\nPzELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGzAZBgNVBAMMEkVN\nIE9WIFRMUyBDQSAtIEczQjAeFw0yNjEwMjAwMDU5MzdaFw0yNzA1MDMwMTA0Mzda\nMC8xLTArBgNVBAMMJG92LWczYi1lbXVkaHJhLndlLWhhdmUtYS1wcm9ibGVtLmNv\nbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABA1haD+95Albk0UN6YtDCPpqevTv\nYUmEhsIVpb/hnhsbBlH0yLthSE2JLH14ywiQJXttoc/pDWC9eoICW6CJXL2jgacw\ngaQwDAYDVR0TAQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB\nBQUHAwEwLwYDVR0RBCgwJoIkb3YtZzNiLWVtdWRocmEud2UtaGF2ZS1hLXByb2Js\nZW0uY29tMB0GA1UdDgQWBBQUhzc4s202Hn62G0srmgguYdY2MTAfBgNVHSMEGDAW\ngBSb2GVRr+fnLDetCO5pGdm/N4SuCjAKBggqhkjOPQQDAgNoADBlAjEAoJ6d4zzA\nxE349yGHHrwKZpVCC/HBFutcOpOfzQoe8ikbmaaBTdtKai5TEllwt3dBAjBeHGiX\nVheDhJfewbei99cw1MfYB5tPwdfuY5pe1MJz0Tg9zJ4JgxBvF6LTT0OOXZ4=\n-----END CERTIFICATE-----\n", + "fd9d5c2dbbd4995ab69b8d50c439d9527b00529199c2fd01b2174eb58f07021f": "-----BEGIN CERTIFICATE-----\nMIIDzTCCArWgAwIBAgIQBfB4dS3s56G7hZXTAxZxcTANBgkqhkiG9w0BAQsFADB7\nMQswCQYDVQQGEwJHQjEbMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYD\nVQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UE\nAwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTI1MDUxNDAwMDAwMFoXDTI4\nMTIzMTIzNTk1OVowQDELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5j\nMRswGQYDVQQDDBJFTSBFViBUTFMgQ0EgLSBHM0MwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAAQtVugppvuCxkpGl9lgr5UekGeukfkm00MG2kioqut4abamfm0nQm4EQqoD\ns24ZiN/2NlFO4c8H9bpLP/KgZkWufTHVhmrTGqPSRusLMXfumct1ZsUWznSAimaP\nkfDEf8qjggE0MIIBMDAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd\nBgNVHQ4EFgQUpiVegKlNt/OWfBvHkd+SvO5L618wDgYDVR0PAQH/BAQDAgEGMBIG\nA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC\nMDAGA1UdIAQpMCcwBwYFZ4EMAQEwDQYLKwYBBAGDjiEBAAEwDQYLKwYBBAGDjiEB\nAngwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20vQUFB\nQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUF\nBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQELBQADggEB\nAIFfHiaMSjnrdZygRYV4ekRBmG7ybcFk4hGQWMMyMFi0cAEk2+dGfz5bl8IaIbLq\nz+uVJzhOgVEX7WqIfgbWfFsfPYHYjPHBmy7SlZ3BwA86IQ6qF/etpX2/qL9A5yRN\nN3nVowUpaqV6H3aCpkAh3wR2ZNkvAbLDvPz3J0l+sEWjWv+ZISOV4R+hXDTdPXV0\n1lKxnbwqEsdo/sE2QkCDKwFrZh/0uKjWspydJj+jSaDLTrGNx50Dgmn7VEcD1ZI5\nFXjknVUiO25YlmS1dtVZSBbyUKSjlrP/yDwPcZunoAsmCAAtCKsFLviLaymzvsKI\neSSDGxpSySp2VCxompGDk8g=\n-----END CERTIFICATE-----\n" + }, + "chains": { + "dv-g3b-aaa-emudhra.we-have-a-problem.com": [ + "7e02622ab6d745c1edb4b431bb1c551e667305c3dd3eb0c7cc6dfdb523807424", + "45a883f349aedfd5f3a649b5498ffb121712ba8b448f3f2a91dbc5a766fcddbe" + ], + "dv-g3b-emudhra.we-have-a-problem.com": [ + "9dcb21b1841d75313b3c888f4bb9cc01f8f51379a76fbaa241ca492b668839e7", + "42c4accd12c4f59597925e1a08a7a90102608543809308d50ec427e118ba1493", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ], + "dv-g3c-aaa-emudhra.we-have-a-problem.com": [ + "9200d153c3cac272cc06bf68f9ea163fc24659fc3331e77668236fe00645ae52", + "62b7f9868263f72f6911fc78ceaf72c1979158b7368bf648683bb2aa39b02e72" + ], + "dv-g3c-emudhra.we-have-a-problem.com": [ + "346cdd962e97189b59aee5526d1ae8fa714bb1a64094dfa67bef768ee93f36b5", + "0164d7ceae5bb909c272af151ff44b1177ae9b47d9424b94f1d9558c30934d19", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ], + "ev-g3b-aaa-emudhra.we-have-a-problem.com": [ + "bc88231a840e54cced148da48806169e8d0b1c0a96267a427aeccda9d10e6145", + "579480aea166290bd3c643589e795fc0d0e54e86505d180f1dff99efd3199512" + ], + "ev-g3b-emudhra.we-have-a-problem.com": [ + "66374ae64962c437bd815a2aae546c76e30c2eafac8335192c8e517c15a246ed", + "e7568fe6e12331f4fd7de61da4c7559d0f8eb192a218d4741a3ba75184333838", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ], + "ev-g3c-aaa-emudhra.we-have-a-problem.com": [ + "bcd6ab59ea95fd520bfe7709679b1f682abbd79c77367e84e25415b91de37753", + "fd9d5c2dbbd4995ab69b8d50c439d9527b00529199c2fd01b2174eb58f07021f" + ], + "ev-g3c-emudhra.we-have-a-problem.com": [ + "ccd80467f9cac38f115dcb6e4f57e4d74f5efa519814ea934aeb3be53b83a191", + "432ec25f0d28de9006d97bf752f17bbb2cd41f669933fcb072d9a1148d0261f3", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ], + "ov-g3b-aaa-emudhra.we-have-a-problem.com": [ + "21f57f83631a93c98e867050075746b3361e0433c9e0e969462852476784f6f0", + "5f1ded4dd590af74631290cf073b122a2aac6a1fa479778a0bd43579bd832699" + ], + "ov-g3b-emudhra.we-have-a-problem.com": [ + "f3c3c15bc7edf2b3ebdc922af112f2a0c414eb509901099ef0488c0942e77efe", + "98b896625c0c230adeec3eb2cf6aad5001ecf9156a68644e82374ce5833bc25b", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ], + "ov-g3c-aaa-emudhra.we-have-a-problem.com": [ + "59cf0386b56bc2414a0c06241ea9e25e01a9c93bac7a596648747f9d0ded4906", + "7485b51c0ed192878fa1d2f948fb529ae1842cc5086c4b67f7be5b2039de9489" + ], + "ov-g3c-emudhra.we-have-a-problem.com": [ + "4314941cf242a4ea4149aa7e9cdf77a15de2771a79984b809fc04443fea013c8", + "cb6e15165ac802e9b3f993f16c25d41f4e81c21ebde71ae1b012ac00bf9c7c3f", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd" + ] + }, + "cross_signed_roots": { + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd": "emSign Root TLS CA - G3 (emSign ECC Root CA - G3 cross-sign)" + } +} diff --git a/tests/test_ca_trust.py b/tests/test_ca_trust.py index b2ab32d40..24855ea8e 100644 --- a/tests/test_ca_trust.py +++ b/tests/test_ca_trust.py @@ -1,11 +1,15 @@ """Offline regressions for incident roots, unaffected roots, and CA directories.""" import contextlib +import hashlib import importlib.util import io +import json import os from pathlib import Path +import shutil import ssl +import subprocess import tempfile from types import SimpleNamespace import unittest @@ -16,6 +20,7 @@ ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location("check_ca_trust", ROOT / "scripts/check_ca_trust.py") checker = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(checker) +INCIDENT = json.loads((ROOT / "tests/fixtures/ca-trust/incident.json").read_text()) class CATrustTests(unittest.TestCase): @@ -27,6 +32,10 @@ class CATrustTests(unittest.TestCase): cls.fixtures = Path(cls.temporary.name) for name, pem in PUBLIC_CERTIFICATES.items(): (cls.fixtures / name).write_text(pem) + for fingerprint, pem in INCIDENT["certificates"].items(): + if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != fingerprint: + raise ValueError("incident fixture fingerprint mismatch") + (cls.fixtures / f"{fingerprint}.pem").write_text(pem) @classmethod def tearDownClass(cls): @@ -50,6 +59,74 @@ class CATrustTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "AAA Certificate Services"): checker.check_bundle(bundle) + def test_all_published_intermediates_and_cross_signs_are_rejected(self): + self.assertEqual(len(INCIDENT["intermediates"]), 16) + for fingerprint, name in INCIDENT["intermediates"].items(): + with self.subTest(certificate=name), self.assertRaisesRegex(ValueError, "affected CA certificates"): + checker.check_bundle(self.fixtures / f"{fingerprint}.pem") + + def test_published_roots_and_cross_signed_issuer_are_rejected(self): + certificates = INCIDENT["roots"] | INCIDENT["cross_signed_roots"] + self.assertEqual(len(certificates), 5) + for fingerprint, name in certificates.items(): + with self.subTest(certificate=name), self.assertRaises(ValueError): + checker.check_bundle(self.fixtures / f"{fingerprint}.pem") + + def test_clean_roots_do_not_hide_an_added_compromised_intermediate(self): + fingerprint = next(iter(INCIDENT["intermediates"])) + with tempfile.TemporaryDirectory() as tmp: + bundle = Path(tmp) / "custom.pem" + bundle.write_text(PUBLIC_CERTIFICATES["emsign-g1.pem"] + INCIDENT["certificates"][fingerprint]) + with self.assertRaisesRegex(ValueError, "EM DV TLS CA - G3B"): + checker.check_bundle(bundle) + + @unittest.skipUnless(shutil.which("openssl"), "offline chain verification requires OpenSSL") + def test_published_chains_and_custom_anchor_with_affected_positive_control(self): + # Freeze time inside the public fixtures' validity interval. Otherwise + # expiry could make an ineffective trust-store fix appear successful. + help_result = subprocess.run(["openssl", "verify", "-help"], capture_output=True, text=True) + no_store = ["-no-CAstore"] if "-no-CAstore" in help_result.stdout + help_result.stderr else [] + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + affected = directory / "affected.pem" + affected.write_text("".join(INCIDENT["certificates"][key] for key in INCIDENT["roots"])) + clean = self.fixtures / "emsign-g1.pem" + custom = directory / "custom.pem" + fingerprint = next(iter(INCIDENT["intermediates"])) + custom.write_text(clean.read_text() + INCIDENT["certificates"][fingerprint]) + self.assertEqual(len(INCIDENT["chains"]), 12) + custom_accepted = [] + for hostname, chain in INCIDENT["chains"].items(): + leaf, intermediates = directory / "leaf.pem", directory / "chain.pem" + leaf.write_text(INCIDENT["certificates"][chain[0]]) + intermediates.write_text("".join(INCIDENT["certificates"][key] for key in chain[1:])) + + def verify(bundle): + return subprocess.run([ + "openssl", "verify", "-no-CApath", *no_store, + "-attime", str(INCIDENT["verification_time"]), + "-partial_chain", "-x509_strict", "-purpose", "sslserver", + "-verify_hostname", hostname, "-CAfile", str(bundle), + "-untrusted", str(intermediates), str(leaf), + ], capture_output=True, text=True) + + with self.subTest(hostname=hostname): + positive = verify(affected) + self.assertEqual(positive.returncode, 0, positive.stderr) + negative = verify(clean) + self.assertNotEqual(negative.returncode, 0) + self.assertIn("unable to get local issuer certificate", negative.stderr) + if verify(custom).returncode == 0: + custom_accepted.append(hostname) + self.assertEqual(custom_accepted, [ + "dv-g3b-aaa-emudhra.we-have-a-problem.com", + "dv-g3b-emudhra.we-have-a-problem.com", + ]) + checker.check_bundle(clean) + for bundle in (affected, custom): + with self.subTest(bundle=bundle.name), self.assertRaises(ValueError): + checker.check_bundle(bundle) + def test_empty_malformed_and_missing_bundles_fail_closed(self): with tempfile.TemporaryDirectory() as tmp: for content in ("", "not a certificate"): @@ -60,6 +137,48 @@ class CATrustTests(unittest.TestCase): with self.assertRaises(OSError): checker.check_bundle(Path(tmp) / "missing.pem") + def test_each_configured_override_is_checked(self): + fingerprint = next(iter(INCIDENT["intermediates"])) + affected = self.fixtures / f"{fingerprint}.pem" + clean = self.fixtures / "emsign-g1.pem" + for name in ("LLM_CA_BUNDLE", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", + "PIP_CERT", "NODE_EXTRA_CA_CERTS", "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH"): + for bundle, expected in ((clean, 0), (affected, 1)): + with self.subTest(variable=name, bundle=bundle.name), patch.dict( + os.environ, {name: str(bundle)}, clear=True + ), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--overrides", str(clean)]), expected) + + def test_invalid_override_is_not_hidden_by_a_clean_explicit_bundle(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + for name, content in (("empty.pem", ""), ("malformed.pem", "invalid")): + (directory / name).write_text(content) + for name in ("missing.pem", "empty.pem", "malformed.pem"): + with self.subTest(file=name), patch.dict( + os.environ, {"LLM_CA_BUNDLE": str(directory / name)}, clear=True + ), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--overrides", str(self.fixtures / "emsign-g1.pem")]), 1) + + def test_no_configured_overrides_does_not_report_validated_stores(self): + with patch.dict(os.environ, {"LLM_CA_BUNDLE": " "}, clear=True), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--overrides"]), 1) + + def test_requests_directory_override_checks_hashed_intermediates(self): + fingerprint = next(iter(INCIDENT["intermediates"])) + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + for name in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "PIP_CERT"): + with self.subTest(variable=name), patch.dict(os.environ, {name: str(directory)}, clear=True): + for content, expected in ((PUBLIC_CERTIFICATES["emsign-g1.pem"], 0), + (INCIDENT["certificates"][fingerprint], 1)): + (directory / "12345678.0").write_text(content) + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--overrides"]), expected) + (directory / "12345678.0").unlink() + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(checker.main(["--overrides"]), 1) + def test_safe_cafile_does_not_hide_stale_hashed_directory(self): with tempfile.TemporaryDirectory() as tmp: directory = Path(tmp) From 77a02c30ffd3f86a34f1bc9e2843589acad76e67 Mon Sep 17 00:00:00 2001 From: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:40:05 +0000 Subject: [PATCH 5/5] fix(security): verify deployed trust before dependency work --- .github/workflows/ci.yml | 22 +++++++------ docs/tls-trust.md | 31 +++++++++++++++--- scripts/check_ca_trust.py | 9 ++++++ tests/test_ca_trust.py | 38 +++++++++++++++++++++++ tests/test_ci_authoritative_validation.py | 13 ++++++++ 5 files changed, 99 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29f303ca4..93dcb5e37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,17 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' with: python-version: "3.11" + # Establish system trust before dependency installers or their hooks run. + # Keep one apt pass for FFmpeg and bubblewrap; containment is checked later. + - name: Install FFmpeg and bubblewrap and validate system CA trust + if: steps.docs-check.outputs.docs_only != 'true' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates + python scripts/check_ca_trust.py --system + command -v ffmpeg + ffmpeg -version | head -n 1 + # uv resolves the same versions as pip but installs ~5x faster, and its # cache keeps each shard from re-downloading the whole requirement set. - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 @@ -166,6 +177,7 @@ jobs: prune-cache: false - run: | uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt + python scripts/check_ca_trust.py --python uv pip install --system -r requirements.txt if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -187,16 +199,6 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' - run: mkdir -p data # sqlite DB lives at ./data/app.db if: steps.docs-check.outputs.docs_only != 'true' - # One apt pass for FFmpeg (media integration tests) and bubblewrap - # (containment, verified in the next step). - - name: Install FFmpeg and bubblewrap - if: steps.docs-check.outputs.docs_only != 'true' - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates - command -v ffmpeg - ffmpeg -version | head -n 1 - - name: Validate Python and system CA trust if: steps.docs-check.outputs.docs_only != 'true' run: python scripts/check_ca_trust.py --system --python diff --git a/docs/tls-trust.md b/docs/tls-trust.md index 8e0351c68..fd5d9776d 100644 --- a/docs/tls-trust.md +++ b/docs/tls-trust.md @@ -16,27 +16,50 @@ The pinned bootstrap protects package integrity even when the old installer's ce On Linux/OpenSSL deployments, inspect both the effective CA file and hashed CA directory, plus the two Python bundles: ```sh -python scripts/check_ca_trust.py --system --python --overrides +python scripts/check_ca_trust.py --system --python --overrides --verbose ``` Explicit PEM bundles can also be passed as positional arguments. The check rejects the four published root fingerprints, all sixteen published intermediate certificates (including their cross-signs), the cross-signed G3 issuer in the published test chains, and malformed/empty bundles. Checking intermediates matters because adding one as a custom trust anchor can restore an affected chain even after its roots have been removed. `--overrides` checks every nonempty `LLM_CA_BUNDLE`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `PIP_CERT`, `NODE_EXTRA_CA_CERTS` and `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` setting in the checker's process environment, including configured fallback settings. Requests/pip hashed-directory overrides are inspected too; `--system` covers `SSL_CERT_DIR`. Run with the application's environment and filesystem access. The script does not load `.env` or application configuration files; pass any CA paths configured only there explicitly, for example `python scripts/check_ca_trust.py --system --python /path/to/private-ca.pem`. Missing or invalid configured files fail the check. Selecting only `--overrides` with no configured paths also fails instead of reporting that stores were validated. +`--verbose` reports the Python executable/prefix, user home, effective UID/GID where available, and resolved CA paths, including bundles selected from user-installed packages. Compare these with the running application's identity and interpreter. It does not dump environment variables or credentials. + This is an offline inventory check of known certificate fingerprints. It does not modify stores, reject requests at runtime, model native OS revocation, or detect arbitrary reissued certificates sharing the compromised keys. Chromium's policy separately matches public keys. Check mounted or replaced CA files again after deployment, remove affected trust through the owning platform/client configuration, and restart clients that cache SSL contexts. A passing build cannot validate custom files introduced later. Both Docker build stages refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if known affected CA certificates remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of known affected CA certificates: gRPC can otherwise fall back to its affected bundled defaults. Upstream gRPC has [merged a bundle update](https://github.com/grpc/grpc/commit/8b32af3bf498c2ba9667410eaa8c836e700e97e6); as checked on 2026-10-21, its [release backport](https://github.com/grpc/grpc/pull/43700) is still open and the latest published grpcio wheel is 1.84.0. Retain the checked override until a released replacement's actual trust behavior has been verified. Chromium uses a separate trust store. The Docker image installs a managed `CADistrustedCertificates` policy for the eight compromised intermediate public keys, covering their cross-signs without depending on background revocation updates. It adds no trusted certificate and does not disable certificate verification. The policy and public-certificate provenance are documented in [the browser policy note](../docker/chromium-ca-policy.md). Native browser installations and independently managed profiles require their own platform policy or vendor update; this image policy does not configure host browsers. -Rebuild with a fresh base and dependency layers when remediating an existing deployment: +To replace only Odysseus in an already running Compose stack, build from the corrected checkout with a distinct local image tag. Use the same Compose files and project name as the existing deployment for every command below: ```sh +export ODYSSEUS_IMAGE="odysseus-local:ca-$(git rev-parse --short=12 HEAD)" docker compose build --pull --no-cache odysseus -docker compose up -d odysseus -docker compose exec odysseus python scripts/check_ca_trust.py --system --python --grpc --overrides +docker compose up -d --no-deps --pull never --no-build odysseus +docker image inspect --format '{{.Id}}' "$ODYSSEUS_IMAGE" +docker inspect --format '{{.Image}}' "$(docker compose ps -q odysseus)" ``` +The two image IDs must match. Keep the selected `ODYSSEUS_IMAGE` in the deployment configuration for subsequent recreations. Compose can prefer the registry image when both `image` and `build` are present; the explicit pull/build flags keep this replacement on the local build. `--no-deps` leaves the existing sidecars running and is intended for an existing stack. For registry deployments, select a verified corrected immutable tag or digest. A merge to `dev` publishes `:dev`; it does not update `:latest`, which is published from `main`. + +Check the standard image as its configured app account, using the image's Python interpreter and the same user-bin PATH prefix as the entrypoint: + +```sh +docker compose exec -T --user 0 odysseus sh -ec ' + gosu_bin=$(command -v gosu) + app_user=$(getent passwd "${PUID:-1000}" | cut -d: -f1) + test -n "$app_user" + exec "$gosu_bin" "$app_user" env PATH="/app/.local/bin:$PATH" \ + /usr/local/bin/python /app/scripts/check_ca_trust.py \ + --system --python --grpc --overrides --verbose +' +``` + +`gosu` selects the account's HOME and drops privileges before Python runs, so the check includes its persisted user packages. The entrypoint itself is not rerun. Plain `docker compose exec ... python` uses the image's default root user and can miss `/app/.local` packages. A custom service user, entrypoint, virtual environment or launch command requires matching that deployment's actual user, HOME and Python interpreter instead of copying the standard command unchanged. Docker exec inherits container-creation environment, not variables exported later by the app or entrypoint: reproduce any additional Python/TLS settings with exec's `--env` options, and pass CA paths configured only in application files explicitly. Confirm the verbose identity and selected paths before accepting the result. + +For containers started non-root or without the SETUID/SETGID capabilities needed by `gosu`, use `docker compose exec --user UID:GID --env HOME=/actual/app/home` with the deployment's actual numeric IDs, home, PATH and interpreter instead. Do not add capabilities merely to run this check. + SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. For behavioral validation, all twelve [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. In Chromium, confirm the managed policy is loaded at `chrome://policy` and test the incident endpoints through the actual browser path with certificate verification enabled. The PEM checker cannot inspect Chromium's policy or its active verifier. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py index 7f017fe3d..d34fb53df 100644 --- a/scripts/check_ca_trust.py +++ b/scripts/check_ca_trust.py @@ -135,9 +135,16 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--python", action="store_true", help="check certifi and pip's separate vendored bundle") parser.add_argument("--grpc", action="store_true", help="check gRPC's selected default CA file") parser.add_argument("--overrides", action="store_true", help="check configured LLM/HTTPX/Requests/pip/Node/gRPC CA overrides") + parser.add_argument("--verbose", action="store_true", help="show the runtime identity, Python executable and selected CA paths") args = parser.parse_args(argv) if not (args.bundles or args.system or args.python or args.grpc or args.overrides): parser.error("select --system, --python, --grpc, --overrides, or explicit PEM bundle paths") + if args.verbose: + print(f"Python executable: {sys.executable}") + print(f"Python prefix: {sys.prefix}") + print(f"User home: {Path.home()}") + if hasattr(os, "geteuid"): + print(f"Effective user: uid={os.geteuid()} gid={os.getegid()}") try: paths = list(args.bundles) if args.system: @@ -154,6 +161,8 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("no CA bundle files selected; also select --system, --python, or an explicit path") errors = [] for path in paths: + if args.verbose: + print(f"CA bundle: {path}") try: check_bundle(path) except (OSError, ValueError, ssl.SSLError) as exc: diff --git a/tests/test_ca_trust.py b/tests/test_ca_trust.py index 24855ea8e..0eb35695f 100644 --- a/tests/test_ca_trust.py +++ b/tests/test_ca_trust.py @@ -10,6 +10,7 @@ from pathlib import Path import shutil import ssl import subprocess +import sys import tempfile from types import SimpleNamespace import unittest @@ -44,6 +45,43 @@ class CATrustTests(unittest.TestCase): def test_unaffected_emsign_root_remains_usable(self): checker.check_bundle(self.fixtures / "emsign-g1.pem") + def test_verbose_python_check_reports_and_rejects_affected_user_bundle(self): + # A fresh interpreter must see persisted user packages, independently + # of pytest's imported certifi/pip modules and any active test venv. + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) + env = dict(os.environ, HOME=str(home), PYTHONUSERBASE=str(home / ".local")) + env.pop("PYTHONNOUSERSITE", None) + env.pop("PYTHONPATH", None) + python = getattr(sys, "_base_executable", sys.executable) + site_result = subprocess.run( + [python, "-m", "site", "--user-site"], env=env, capture_output=True, text=True, + ) + self.assertEqual(site_result.returncode, 0, site_result.stderr) + user_site = Path(site_result.stdout.strip()) + for package in ("certifi", "pip", "pip/_vendor", "pip/_vendor/certifi"): + directory = user_site / package + directory.mkdir(parents=True, exist_ok=True) + (directory / "__init__.py").write_text( + "from pathlib import Path\ndef where(): return str(Path(__file__).with_name('cacert.pem'))\n" + ) + application_bundle = user_site / "certifi/cacert.pem" + pip_bundle = user_site / "pip/_vendor/certifi/cacert.pem" + for bundle in (application_bundle, pip_bundle): + bundle.write_text(PUBLIC_CERTIFICATES["emsign-g1.pem"]) + command = [python, str(ROOT / "scripts/check_ca_trust.py"), "--python", "--verbose"] + clean = subprocess.run(command, env=env, capture_output=True, text=True) + self.assertEqual(clean.returncode, 0, clean.stderr) + self.assertIn(str(application_bundle), clean.stdout) + self.assertIn(str(pip_bundle), clean.stdout) + self.assertIn(str(home), clean.stdout) + self.assertIn("Python executable:", clean.stdout) + application_bundle.write_text(PUBLIC_CERTIFICATES["emsign-g3.pem"]) + affected = subprocess.run(command, env=env, capture_output=True, text=True) + self.assertEqual(affected.returncode, 1, affected.stderr) + self.assertIn(str(application_bundle), affected.stderr) + self.assertIn("emSign ECC Root CA - G3", affected.stderr) + def test_compromised_emsign_path_is_rejected(self): with self.assertRaisesRegex(ValueError, "emSign ECC Root CA - G3"): checker.check_bundle(self.fixtures / "emsign-g3.pem") diff --git a/tests/test_ci_authoritative_validation.py b/tests/test_ci_authoritative_validation.py index 6de26817d..cf6994f39 100644 --- a/tests/test_ci_authoritative_validation.py +++ b/tests/test_ci_authoritative_validation.py @@ -33,3 +33,16 @@ def test_python_tests_are_authoritative(): assert "python -m pytest -q" in python_tests assert "continue-on-error:" not in python_tests + + +def test_ci_checks_system_trust_before_project_installation(): + python_tests = _indented_block(_WORKFLOW.read_text(), "python-tests", 2) + system_check = python_tests.index("python scripts/check_ca_trust.py --system\n") + system_update = python_tests.index("sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates") + bootstrap = python_tests.index("uv pip install --system --require-hashes") + python_check = python_tests.index("python scripts/check_ca_trust.py --python\n") + application_install = python_tests.index("uv pip install --system -r requirements.txt") + + assert system_update < system_check < bootstrap < python_check < application_install + assert system_check < python_tests.index("npm ci") + assert system_check < python_tests.index("npx playwright install")