fix(security): refresh client trust and publish validated SearXNG images

Repair SearXNG system and certifi stores, validate native amd64/arm64 runtime images through Compose, and publish the exact verified artifacts through a trusted CI job. Adopt released Python, certifi, grpcio, uv and Docker CLI trust fixes, with strict controls for all fourteen public incident chains. Part of #6784 and #6790. Co-authored-by: Léo <leograndcontact@gmail.com>
2026-10-24 15:52:20 +02:00 · 2026-10-23 22:32:33 +00:00
parent f8e043e8ca
commit c6d76b9a9b
23 changed files with 1200 additions and 17 deletions
@@ -0,0 +1,118 @@
+"""Describe a validated sidecar release without executing image or repository code."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+
+
+IMAGE = "ghcr.io/odysseus-dev/odysseus-searxng"
+REPOSITORY = "odysseus-dev/odysseus"
+SOURCE_FILES = (
+    "docker/searxng/Dockerfile",
+    "docker/searxng/repair_trust.py",
+    "docker/searxng/blocked-certificates.json",
+    "docker/searxng/validate_image.py",
+    "tests/fixtures/ca-trust/incident.json",
+)
+
+
+def revision(value):
+    if not re.fullmatch(r"[0-9a-f]{40}", value):
+        raise ValueError("invalid source revision")
+    return value
+
+
+def digest(value):
+    if not isinstance(value, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", value):
+        raise ValueError("invalid image digest")
+    return value
+
+
+def source_snapshot(root, source_revision, repository):
+    revision(source_revision)
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository):
+        raise ValueError("invalid source repository")
+    dockerfile = (root / SOURCE_FILES[0]).read_text()
+    match = re.search(r"^FROM (\S+) AS base$", dockerfile, re.M)
+    if not match or not re.fullmatch(r"docker\.io/searxng/searxng:[^\s@]+@sha256:[0-9a-f]{64}", match[1]):
+        raise ValueError("SearXNG base must have an immutable upstream digest")
+    return {
+        "schema_version": 1,
+        "repository": repository,
+        "source_revision": source_revision,
+        "upstream_image": match[1],
+        "source_inputs": {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in SOURCE_FILES},
+    }
+
+
+def release_descriptor(amd64, arm64, manifest, source_revision, run_id, tag, arch_digests):
+    revision(source_revision)
+    if amd64 != arm64:
+        raise ValueError("native architecture builds used different source inputs")
+    if amd64.get("repository") != REPOSITORY or amd64.get("source_revision") != source_revision:
+        raise ValueError("release source does not match the canonical workflow revision")
+    if amd64.get("schema_version") != 1 or set(amd64.get("source_inputs", {})) != set(SOURCE_FILES):
+        raise ValueError("incomplete source inventory")
+    if any(not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) for value in amd64["source_inputs"].values()):
+        raise ValueError("invalid source checksum")
+    if not str(run_id).isdigit() or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", tag):
+        raise ValueError("invalid workflow identity or image tag")
+    if manifest.get("mediaType") not in (
+        "application/vnd.oci.image.index.v1+json",
+        "application/vnd.docker.distribution.manifest.list.v2+json",
+    ):
+        raise ValueError("release is not a multiarch image index")
+    children = manifest.get("manifests", [])
+    platforms = sorted(f"{entry.get('platform', {}).get('os')}/{entry.get('platform', {}).get('architecture')}" for entry in children)
+    if platforms != ["linux/amd64", "linux/arm64"]:
+        raise ValueError("release must contain exactly the two validated Linux architectures")
+    if len({digest(entry.get("digest")) for entry in children}) != 2:
+        raise ValueError("architecture digests must be distinct")
+    expected = {arch: digest(value) for arch, value in arch_digests.items()}
+    actual = {entry["platform"]["architecture"]: entry["digest"] for entry in children}
+    if actual != expected:
+        raise ValueError("index does not reference the exact validated architecture images")
+    return {
+        **amd64,
+        "image": f"{IMAGE}@{digest(manifest.get('digest'))}",
+        "tag": tag,
+        "platforms": platforms,
+        "architecture_digests": actual,
+        "workflow_run": f"https://github.com/{REPOSITORY}/actions/runs/{run_id}",
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+    inputs = sub.add_parser("inputs")
+    inputs.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
+    inputs.add_argument("--revision", required=True)
+    inputs.add_argument("--repository", required=True)
+    inputs.add_argument("--output", type=Path, required=True)
+    release = sub.add_parser("release")
+    release.add_argument("--amd64", type=Path, required=True)
+    release.add_argument("--arm64", type=Path, required=True)
+    release.add_argument("--manifest", type=Path, required=True)
+    release.add_argument("--amd64-digest", required=True)
+    release.add_argument("--arm64-digest", required=True)
+    release.add_argument("--revision", required=True)
+    release.add_argument("--run-id", required=True)
+    release.add_argument("--tag", required=True)
+    release.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.command == "inputs":
+        value = source_snapshot(args.root, args.revision, args.repository)
+    else:
+        value = release_descriptor(
+            json.loads(args.amd64.read_text()), json.loads(args.arm64.read_text()),
+            json.loads(args.manifest.read_text()), args.revision, args.run_id, args.tag,
+            {"amd64": args.amd64_digest, "arm64": args.arm64_digest},
+        )
+    args.output.write_text(json.dumps(value, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Exercise an already-built image through the normal Compose startup path.
+set -euo pipefail
+
+searxng_repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)
+searxng_image=${1:?usage: validate-searxng-image.sh IMAGE}
+[[ "$searxng_image" =~ ^[a-zA-Z0-9_./:@+-]+$ ]] || {
+  echo "Invalid image reference" >&2
+  exit 2
+}
+searxng_checks=$(mktemp -d "${TMPDIR:-/tmp}/searxng-ci-XXXXXXXX")
+searxng_project=$(basename "$searxng_checks" | tr '[:upper:]' '[:lower:]')
+export SEARXNG_VALIDATION_IMAGE="$searxng_image"
+export SEARXNG_VALIDATION_CHECKS="$searxng_checks"
+export SEARXNG_VALIDATION_VOLUME="${searxng_project}-settings"
+# Never forward an operator's existing secret or load their .env for a CI probe.
+export SEARXNG_SECRET=searxng-ci-validation-only
+searxng_volume_created=0
+searxng_started=0
+searxng_compose=(docker compose --env-file /dev/null --project-name "$searxng_project"
+  -f "$searxng_repo/docker-compose.yml"
+  -f "$searxng_repo/.github/searxng-validation.compose.yml")
+
+searxng_cleanup() {
+  local searxng_exit=$?
+  if [ "$searxng_started" = 1 ]; then
+    "${searxng_compose[@]}" down --remove-orphans >/dev/null 2>&1 || true
+  fi
+  if [ "$searxng_volume_created" = 1 ]; then
+    docker volume rm "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1 || true
+  fi
+  rm -rf -- "$searxng_checks"
+  return "$searxng_exit"
+}
+trap searxng_cleanup EXIT
+
+install -m 0644 "$searxng_repo/docker/searxng/repair_trust.py" \
+  "$searxng_repo/docker/searxng/blocked-certificates.json" \
+  "$searxng_repo/docker/searxng/validate_image.py" "$searxng_checks/"
+install -m 0644 "$searxng_repo/tests/fixtures/ca-trust/incident.json" "$searxng_checks/incident.json"
+install -m 0644 "$searxng_repo/config/searxng/settings.yml" "$searxng_checks/settings.yml"
+install -m 0644 "$searxng_repo/scripts/migrate_searxng_settings.py" "$searxng_checks/migrate_searxng_settings.py"
+if docker volume inspect "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1; then
+  echo "Refusing to reuse an existing validation volume" >&2
+  exit 1
+fi
+docker volume create --label "odysseus.searxng-validation=$searxng_project" \
+  "$SEARXNG_VALIDATION_VOLUME" >/dev/null
+searxng_volume_created=1
+searxng_started=1
+if ! "${searxng_compose[@]}" up --detach --no-deps --no-build --pull never \
+  --wait --wait-timeout 150 searxng; then
+  "${searxng_compose[@]}" logs --no-color searxng >&2 || true
+  exit 1
+fi
+"${searxng_compose[@]}" exec -T searxng /usr/local/searxng/.venv/bin/python \
+  /opt/odysseus-checks/validate_image.py --running --live
@@ -0,0 +1,21 @@
+# CI-only isolation; retain the service's entrypoint, healthcheck and capabilities.
+services:
+  searxng:
+    image: ${SEARXNG_VALIDATION_IMAGE:?set the image being validated}
+    ports: !reset []
+    volumes: !override
+      - searxng-validation-data:/etc/searxng
+      - ${SEARXNG_VALIDATION_CHECKS:?}/settings.yml:/tmp/searxng-settings.yml.template:ro
+      - ${SEARXNG_VALIDATION_CHECKS:?}/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro
+      - ${SEARXNG_VALIDATION_CHECKS:?}:/opt/odysseus-checks:ro
+    networks: !override
+      - default
+    restart: "no"
+
+networks:
+  default: {}
+
+volumes:
+  searxng-validation-data:
+    external: true
+    name: ${SEARXNG_VALIDATION_VOLUME:?}
@@ -171,7 +171,7 @@ jobs:
      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7  # v10.2.0
        if: steps.docs-check.outputs.docs_only != 'true'
        with:
-          version: "0.13.4"
+          version: "0.13.5"
          enable-cache: true
          cache-dependency-glob: requirements.txt
          prune-cache: false
@@ -0,0 +1,243 @@
+name: ci / SearXNG image
+
+on:
+  pull_request:
+    paths:
+      - 'docker/searxng/**'
+      - 'tests/fixtures/ca-trust/**'
+      - 'tests/test_searxng*'
+      - 'docker-compose*.yml'
+      - 'config/searxng/**'
+      - 'scripts/migrate_searxng_settings.py'
+      - '.github/scripts/*searxng*'
+      - '.github/searxng-validation.compose.yml'
+      - '.github/workflows/searxng-image.yml'
+  push:
+    branches: [dev, main, release/searxng-ca-trust]
+    paths:
+      - 'docker/searxng/**'
+      - 'tests/fixtures/ca-trust/**'
+      - 'docker-compose*.yml'
+      - 'config/searxng/**'
+      - 'scripts/migrate_searxng_settings.py'
+      - '.github/scripts/*searxng*'
+      - '.github/searxng-validation.compose.yml'
+      - '.github/workflows/searxng-image.yml'
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: Publish after native validation (canonical trusted branches only)
+        type: boolean
+        default: false
+
+concurrency:
+  group: searxng-image-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: SearXNG validate (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            platform: linux/amd64
+            runner: ubuntu-latest
+          - arch: arm64
+            platform: linux/arm64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+      - name: Reproduce affected base
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: docker/searxng/Dockerfile
+          target: baseline-validation
+          network: none
+          platforms: ${{ matrix.platform }}
+          no-cache-filters: baseline-validation
+          cache-from: type=gha,scope=searxng-${{ matrix.arch }}
+      - name: Validate repaired stores and upstream startup
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: docker/searxng/Dockerfile
+          target: validation
+          network: none
+          platforms: ${{ matrix.platform }}
+          no-cache-filters: validation
+          cache-from: type=gha,scope=searxng-${{ matrix.arch }}
+      - name: Build runtime without registry credentials
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: docker/searxng/Dockerfile
+          target: runtime
+          network: none
+          platforms: ${{ matrix.platform }}
+          load: true
+          tags: localhost/odysseus-searxng:ci-${{ matrix.arch }}
+          provenance: false
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          cache-from: type=gha,scope=searxng-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=searxng-${{ matrix.arch }}
+      - name: Validate actual runtime through Compose and live HTTPS
+        env:
+          SEARXNG_TEST_IMAGE: localhost/odysseus-searxng:ci-${{ matrix.arch }}
+        run: bash .github/scripts/validate-searxng-image.sh "$SEARXNG_TEST_IMAGE"
+      - name: Export validated runtime and input checksums
+        env:
+          ARCH: ${{ matrix.arch }}
+        run: |
+          docker save --output "$RUNNER_TEMP/runtime-${ARCH}.tar" "localhost/odysseus-searxng:ci-${ARCH}"
+          python3 .github/scripts/searxng_release.py inputs --revision "$GITHUB_SHA" \
+            --repository "$GITHUB_REPOSITORY" --output "$RUNNER_TEMP/inputs-${ARCH}.json"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: searxng-runtime-${{ matrix.arch }}
+          path: |
+            ${{ runner.temp }}/runtime-${{ matrix.arch }}.tar
+            ${{ runner.temp }}/inputs-${{ matrix.arch }}.json
+          compression-level: 0
+          if-no-files-found: error
+          retention-days: 3
+
+  publish:
+    name: Publish validated SearXNG architectures and index
+    needs: validate
+    if: >-
+      github.repository == 'odysseus-dev/odysseus' &&
+      (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish)) &&
+      contains(fromJSON('["refs/heads/dev","refs/heads/main","refs/heads/release/searxng-ca-trust"]'), github.ref)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      image: ${{ steps.release.outputs.image }}
+    env:
+      SEARXNG_IMAGE: ghcr.io/odysseus-dev/odysseus-searxng
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: ${{ runner.temp }}/searxng-artifacts
+          pattern: searxng-runtime-*
+          merge-multiple: true
+      - name: Load and inspect the validated artifacts before registry login
+        run: |
+          for arch in amd64 arm64; do
+            docker load --input "$RUNNER_TEMP/searxng-artifacts/runtime-${arch}.tar"
+            image="localhost/odysseus-searxng:ci-${arch}"
+            test "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image")" = "linux/${arch}"
+            test "$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" = "$GITHUB_SHA"
+          done
+      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish the exact validated artifacts
+        id: release
+        run: |
+          tag="sha-${GITHUB_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          refs=()
+          for arch in amd64 arm64; do
+            remote="${SEARXNG_IMAGE}:${tag}-${arch}"
+            docker tag "localhost/odysseus-searxng:ci-${arch}" "$remote"
+            docker push "$remote"
+            digests=$(docker image inspect --format '{{json .RepoDigests}}' "$remote")
+            ref=$(jq -er --arg prefix "${SEARXNG_IMAGE}@" '.[] | select(startswith($prefix))' <<< "$digests")
+            [[ "$ref" =~ ^ghcr.io/odysseus-dev/odysseus-searxng@sha256:[0-9a-f]{64}$ ]]
+            refs+=("$ref")
+          done
+          docker buildx imagetools create --tag "${SEARXNG_IMAGE}:${tag}" "${refs[@]}"
+          docker buildx imagetools inspect "${SEARXNG_IMAGE}:${tag}" --format '{{json .Manifest}}' > "$RUNNER_TEMP/searxng-manifest.json"
+          python3 .github/scripts/searxng_release.py release \
+            --amd64 "$RUNNER_TEMP/searxng-artifacts/inputs-amd64.json" \
+            --arm64 "$RUNNER_TEMP/searxng-artifacts/inputs-arm64.json" \
+            --manifest "$RUNNER_TEMP/searxng-manifest.json" --revision "$GITHUB_SHA" \
+            --amd64-digest "${refs[0]#*@}" --arm64-digest "${refs[1]#*@}" \
+            --run-id "$GITHUB_RUN_ID" --tag "$tag" --output "$RUNNER_TEMP/searxng-release.json"
+          image=$(jq -er .image "$RUNNER_TEMP/searxng-release.json")
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: searxng-release-candidate
+          path: ${{ runner.temp }}/searxng-release.json
+          if-no-files-found: error
+          retention-days: 30
+
+  verify-published:
+    name: Verify public SearXNG pull (${{ matrix.arch }})
+    needs: publish
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Pull the published index without registry credentials
+        env:
+          SEARXNG_TEST_IMAGE: ${{ needs.publish.outputs.image }}
+        run: |
+          if ! docker pull "$SEARXNG_TEST_IMAGE"; then
+            echo '::error::Anonymous GHCR pull failed. A newly created package defaults to private: make odysseus-searxng public, then rerun the failed verification jobs.'
+            exit 1
+          fi
+      - name: Validate the pulled runtime through Compose
+        env:
+          SEARXNG_TEST_IMAGE: ${{ needs.publish.outputs.image }}
+        run: bash .github/scripts/validate-searxng-image.sh "$SEARXNG_TEST_IMAGE"
+
+  release:
+    name: Record verified Compose image reference
+    needs: [publish, verify-published]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: searxng-release-candidate
+          path: ${{ runner.temp }}/searxng-release
+      - name: Record the verified image
+        env:
+          SEARXNG_VERIFIED_IMAGE: ${{ needs.publish.outputs.image }}
+        run: |
+          {
+            echo '## Verified SearXNG image'
+            echo
+            echo "\`$SEARXNG_VERIFIED_IMAGE\`"
+            echo
+            echo 'Both native architectures passed Compose startup, fourteen incident-chain/HTTPS checks and live search before and after publication.'
+            echo 'Pin this reference in the base and standalone GPU Compose files and commit the release descriptor.'
+          } >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: searxng-release
+          path: ${{ runner.temp }}/searxng-release/searxng-release.json
+          if-no-files-found: error
+          retention-days: 30