diff --git a/.github/scripts/validate-searxng-image.sh b/.github/scripts/validate-searxng-image.sh new file mode 100644 index 000000000..3edd5ba16 --- /dev/null +++ b/.github/scripts/validate-searxng-image.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Exercise an already-built image through the normal Compose startup path. +set -euo pipefail + +searxng_repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +searxng_image=${1:?usage: validate-searxng-image.sh IMAGE} +[[ "$searxng_image" =~ ^[a-zA-Z0-9_./:@+-]+$ ]] || { + echo "Invalid image reference" >&2 + exit 2 +} +searxng_checks=$(mktemp -d "${TMPDIR:-/tmp}/searxng-ci-XXXXXXXX") +searxng_project=$(basename "$searxng_checks" | tr '[:upper:]' '[:lower:]') +export SEARXNG_VALIDATION_IMAGE="$searxng_image" +export SEARXNG_VALIDATION_CHECKS="$searxng_checks" +export SEARXNG_VALIDATION_VOLUME="${searxng_project}-settings" +# Never forward an operator's existing secret or load their .env for a CI probe. +export SEARXNG_SECRET=searxng-ci-validation-only +searxng_volume_created=0 +searxng_started=0 +searxng_compose=(docker compose --env-file /dev/null --project-name "$searxng_project" + -f "$searxng_repo/docker-compose.yml" + -f "$searxng_repo/.github/searxng-validation.compose.yml") + +searxng_cleanup() { + local searxng_exit=$? + if [ "$searxng_started" = 1 ]; then + "${searxng_compose[@]}" down --remove-orphans >/dev/null 2>&1 || true + fi + if [ "$searxng_volume_created" = 1 ]; then + docker volume rm "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1 || true + fi + rm -rf -- "$searxng_checks" + return "$searxng_exit" +} +trap searxng_cleanup EXIT + +install -m 0644 "$searxng_repo/scripts/check_ca_trust.py" \ + "$searxng_repo/docker/searxng/validate_image.py" "$searxng_checks/" +install -m 0644 "$searxng_repo/tests/fixtures/ca-trust/incident.json" "$searxng_checks/incident.json" +install -m 0644 "$searxng_repo/config/searxng/settings.yml" "$searxng_checks/settings.yml" +install -m 0644 "$searxng_repo/scripts/migrate_searxng_settings.py" "$searxng_checks/migrate_searxng_settings.py" +if docker volume inspect "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1; then + echo "Refusing to reuse an existing validation volume" >&2 + exit 1 +fi +docker volume create --label "odysseus.searxng-validation=$searxng_project" \ + "$SEARXNG_VALIDATION_VOLUME" >/dev/null +searxng_volume_created=1 +searxng_started=1 +if ! "${searxng_compose[@]}" up --detach --no-deps --no-build --pull never \ + --wait --wait-timeout 150 searxng; then + "${searxng_compose[@]}" logs --no-color searxng >&2 || true + exit 1 +fi +"${searxng_compose[@]}" exec -T searxng /usr/local/searxng/.venv/bin/python \ + /opt/odysseus-checks/validate_image.py --running --live diff --git a/.github/searxng-validation.compose.yml b/.github/searxng-validation.compose.yml new file mode 100644 index 000000000..d1e1c575c --- /dev/null +++ b/.github/searxng-validation.compose.yml @@ -0,0 +1,21 @@ +# CI-only isolation; retain the service's entrypoint, healthcheck and capabilities. +services: + searxng: + image: ${SEARXNG_VALIDATION_IMAGE:?set the image being validated} + ports: !reset [] + volumes: !override + - searxng-validation-data:/etc/searxng + - ${SEARXNG_VALIDATION_CHECKS:?}/settings.yml:/tmp/searxng-settings.yml.template:ro + - ${SEARXNG_VALIDATION_CHECKS:?}/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro + - ${SEARXNG_VALIDATION_CHECKS:?}:/opt/odysseus-checks:ro + networks: !override + - default + restart: "no" + +networks: + default: {} + +volumes: + searxng-validation-data: + external: true + name: ${SEARXNG_VALIDATION_VOLUME:?} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42f0651f0..229a4d599 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: "20" + node-version: "22.23.4" - run: npm ci --ignore-scripts --no-audit --no-fund - name: Check frontend architecture run: node scripts/check_frontend.mjs @@ -155,20 +155,35 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' with: python-version: "3.11" + # Establish system trust before dependency installers or their hooks run. + # Keep one apt pass for FFmpeg and bubblewrap; containment is checked later. + - name: Install FFmpeg and bubblewrap and validate system CA trust + if: steps.docs-check.outputs.docs_only != 'true' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates + python scripts/check_ca_trust.py --system + command -v ffmpeg + ffmpeg -version | head -n 1 + # uv resolves the same versions as pip but installs ~5x faster, and its # cache keeps each shard from re-downloading the whole requirement set. - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 if: steps.docs-check.outputs.docs_only != 'true' with: + version: "0.13.5" enable-cache: true cache-dependency-glob: requirements.txt prune-cache: false - - run: uv pip install --system -r requirements.txt + - run: | + uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt + python scripts/check_ca_trust.py --python + uv pip install --system -r requirements.txt if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 if: steps.docs-check.outputs.docs_only != 'true' with: - node-version: "20" + node-version: "22.23.4" cache: npm - run: npm ci if: steps.docs-check.outputs.docs_only != 'true' @@ -184,15 +199,9 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' - run: mkdir -p data # sqlite DB lives at ./data/app.db if: steps.docs-check.outputs.docs_only != 'true' - # One apt pass for FFmpeg (media integration tests) and bubblewrap - # (containment, verified in the next step). - - name: Install FFmpeg and bubblewrap + - name: Validate Python and system CA trust if: steps.docs-check.outputs.docs_only != 'true' - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap - command -v ffmpeg - ffmpeg -version | head -n 1 + run: python scripts/check_ca_trust.py --system --python - name: Establish functional bubblewrap containment if: steps.docs-check.outputs.docs_only != 'true' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 0e4f9a1f1..e78bba158 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -70,5 +70,9 @@ jobs: - name: Run pip-audit on requirements run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install pip-audit==2.10.0 pip-audit -r requirements.txt -r requirements-optional.txt --strict diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index da41c7140..6865d5120 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -29,7 +29,6 @@ permissions: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} jobs: build: @@ -52,6 +51,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Normalize image name + shell: bash + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - name: Set up Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Log in to GHCR @@ -65,6 +67,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . + pull: true platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.arch }} @@ -93,6 +96,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Normalize image name + shell: bash + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - name: Read APP_VERSION + short sha id: ver run: | diff --git a/.github/workflows/searxng-image.yml b/.github/workflows/searxng-image.yml new file mode 100644 index 000000000..844617b10 --- /dev/null +++ b/.github/workflows/searxng-image.yml @@ -0,0 +1,79 @@ +name: ci / SearXNG trust + +on: + pull_request: + paths: + - 'docker/searxng/**' + - 'scripts/check_ca_trust.py' + - 'tests/fixtures/ca-trust/**' + - 'tests/test_searxng*' + - 'docker-compose*.yml' + - 'config/searxng/**' + - 'scripts/migrate_searxng_settings.py' + - '.github/scripts/*searxng*' + - '.github/searxng-validation.compose.yml' + - '.github/workflows/searxng-image.yml' + push: + branches: [dev, main] + paths: + - 'docker/searxng/**' + - 'scripts/check_ca_trust.py' + - 'tests/fixtures/ca-trust/**' + - 'docker-compose*.yml' + - 'config/searxng/**' + - 'scripts/migrate_searxng_settings.py' + - '.github/scripts/*searxng*' + - '.github/searxng-validation.compose.yml' + - '.github/workflows/searxng-image.yml' + workflow_dispatch: + +concurrency: + group: searxng-image-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + validate: + name: SearXNG validate (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + platform: linux/amd64 + runner: ubuntu-latest + - arch: arm64 + platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Select the immutable upstream image from Compose + id: image + run: | + docker compose --env-file /dev/null -f docker-compose.yml config --format json | + python3 -c 'import json, re, sys; image = json.load(sys.stdin)["services"]["searxng"]["image"]; re.fullmatch(r"docker\.io/searxng/searxng:[0-9]{4}\.[^@\s]+@sha256:[0-9a-f]{64}", image) or sys.exit("Invalid SearXNG pin"); print("image=" + image)' >> "$GITHUB_OUTPUT" + - name: Pull the upstream image without registry credentials + env: + SEARXNG_TEST_IMAGE: ${{ steps.image.outputs.image }} + run: docker pull "$SEARXNG_TEST_IMAGE" + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - name: Check unmodified upstream trust and actual entrypoint offline + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: docker/searxng/Dockerfile + target: validation + build-args: SEARXNG_IMAGE=${{ steps.image.outputs.image }} + network: none + platforms: ${{ matrix.platform }} + no-cache-filters: validation + - name: Validate the selected upstream runtime through Compose and live HTTPS + env: + SEARXNG_TEST_IMAGE: ${{ steps.image.outputs.image }} + run: bash .github/scripts/validate-searxng-image.sh "$SEARXNG_TEST_IMAGE" diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index b00cd03a4..18a197eb0 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -76,5 +76,9 @@ jobs: - name: Run zizmor run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install zizmor==1.25.2 zizmor --offline --min-severity=low .github/workflows/ diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 38e184764..294289b7f 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -48,8 +48,8 @@ just composed. | Service | Image | Purpose | License | |---|---|---|---| | [SearXNG](https://github.com/searxng/searxng) | `searxng/searxng:2026.9.25-12f8b6515` (pinned tag; see compose) | Default metasearch backend | AGPL-3.0 | -| [ChromaDB](https://github.com/chroma-core/chroma) | `chromadb/chroma:latest` | Vector store for memory / RAG | Apache-2.0 | -| [ntfy](https://github.com/binwiederhier/ntfy) | `binwiederhier/ntfy` | Push notifications (self-hosted reminders) | Apache-2.0 / GPL-2.0 | +| [ChromaDB](https://github.com/chroma-core/chroma) | `chromadb/chroma:1.5.9` (pinned digest; see compose) | Vector store for memory / RAG | Apache-2.0 | +| [ntfy](https://github.com/binwiederhier/ntfy) | `binwiederhier/ntfy:v2.32.0` (pinned digest; see compose) | Push notifications (self-hosted reminders) | Apache-2.0 / GPL-2.0 | ## Bundled front-end libraries diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38586845f..8a7160245 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Thanks for helping. The project is moving quickly, so the best contributions are Odysseus has two branches: - **`dev`** — where all PRs land. Things can be in flux here; the merge button gets used freely. -- **`main`** — what users run. Curated and tested by the maintainer. Fast-forwarded to a stable `dev` commit at each release. +- **`main`** — what users run. Curated and tested by the release's [Release Maintainer](docs/maintainers/testing-and-releases.md#release-ownership). Fast-forwarded to a stable `dev` commit at each release. **Open your PR against `dev`, not `main`.** The GitHub "base" dropdown defaults to `dev`. If you opened a PR against `main` by accident, click "Edit" on the PR and change the base — no rebase needed. @@ -62,6 +62,10 @@ docker compose logs --tail=120 odysseus Mention what you ran in the pull request description. If you could not run a check, say so. +## Maintainer governance and review guidance + +For repository roles, product decisions, risk-based approvals, issue triage, independent security reviews, CodeQL findings, and release handoffs, see the [Odysseus maintainer handbook](docs/maintainers/README.md). This complements the contributor requirements below and does not change the branch model or current PR template. + ## Pull Requests Good pull requests usually include: diff --git a/Dockerfile b/Dockerfile index d6e7ca48f..936f35b90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,18 @@ # which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so # the final image / Cookbook never has to compile the broken sdists. See # docker/build-realesrgan-wheels.sh for the full rationale. -FROM python:3.14-slim AS realesrgan-wheels -RUN apt-get update && apt-get install -y --no-install-recommends curl \ +FROM python:3.14.9-slim AS realesrgan-wheels +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels -FROM python:3.14-slim +FROM python:3.14.9-slim # System deps. tmux is required by Cookbook for background downloads/serves. # openssh-client is required for Cookbook remote server tests, setup, probes, @@ -28,6 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cmake \ curl \ + ca-certificates \ git \ nodejs \ npm \ @@ -48,6 +54,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libmagic1 \ && rm -rf /var/lib/apt/lists/* +# Refresh both Python trust bundles before installing application dependencies. +# System roots also serve web fetches, webhooks, curl, and Node's OpenSSL mode. +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python +ENV NODE_OPTIONS=--use-openssl-ca \ + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt + +# Chromium has a separate root store. Distrust the compromised intermediate +# public keys explicitly, including cross-signs, without waiting for CRL updates. +COPY docker/chromium-emudhra.json /etc/chromium/policies/managed/odysseus-emudhra.json + # Private browser automation wrapper used by the native `private_browser` tool. # Chromium is installed above, so agent-browser can drive the existing browser # binary without paying `npx` startup/install overhead on each tool call. @@ -72,7 +92,8 @@ RUN npm install -g agent-browser@0.35.0 --omit=dev --loglevel=error # /var/run/docker.sock mount). The Debian `docker.io` package ships # dockerd but not the client binary on slim, so grab the static client # tarball from download.docker.com instead. -ARG DOCKER_CLI_VERSION=29.6.2 +# 29.9.2 includes Go 1.26.10's eMudhra intermediate-key distrust. +ARG DOCKER_CLI_VERSION=29.9.2 RUN ARCH="$(dpkg --print-architecture)" \ && case "$ARCH" in \ amd64) DARCH=x86_64 ;; \ @@ -108,6 +129,11 @@ COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/ RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \ && rm -rf /tmp/odysseus-wheels +# Check gRPC's effective defaults after all Python dependencies are installed. +# The application minimum fixes its bundled roots; the checked override also +# keeps the effective Docker selection consistent with system trust. +RUN python /usr/local/bin/check-ca-trust.py --system --python --grpc + # Copy app code COPY . . # Require the redistribution notices in the image build context. diff --git a/app.py b/app.py index 6c3974c65..ae859a579 100644 --- a/app.py +++ b/app.py @@ -792,7 +792,8 @@ app.include_router(setup_research_routes(research_handler, session_manager=sessi # History from routes.history.history_routes import setup_history_routes -app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler)) +history_router = setup_history_routes(session_manager, upload_handler=upload_handler) +app.include_router(history_router) # Search from routes.search.search_routes import setup_search_routes @@ -956,6 +957,7 @@ app.include_router(setup_codex_routes( memory_router=memory_router, calendar_router=calendar_router, document_router=document_router, + history_router=history_router, )) app.include_router(setup_claude_routes()) @@ -1033,12 +1035,10 @@ async def serve_login(request: Request): @app.get("/api/version") async def get_version(): from core.constants import APP_BUILD_VERSION, APP_SOURCE_COMMIT, APP_VERSION - from src import activity return { "version": APP_VERSION, "build": APP_BUILD_VERSION, "source_commit": APP_SOURCE_COMMIT, - "busy_image_ops": activity.heavy_image_ops_active(), } @@ -1046,7 +1046,10 @@ async def get_version(): async def get_busy(): """Report in-flight heavy work so a deploy can wait instead of dropping it.""" from src import activity - return activity.activity() + return { + "busy_image_ops": activity.heavy_image_ops_active(), + **activity.activity(), + } @app.get("/api/health") async def health_check() -> Dict[str, str]: diff --git a/build-windows-portable.ps1 b/build-windows-portable.ps1 index 7da912f73..634ae8068 100644 --- a/build-windows-portable.ps1 +++ b/build-windows-portable.ps1 @@ -47,7 +47,8 @@ if (-not $pyExe) { Write-Host ("Using Python: " + $pyExe) Write-Step "Installing build dependencies" -& $pyExe -m pip install --upgrade pip --quiet +& $pyExe -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS build dependency bootstrap failed." } & $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." } diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 183b31601..5c85014d6 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -127,7 +127,7 @@ services: - ${RENDER_GID:-render} chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -145,8 +145,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -196,7 +196,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 23404b2eb..5c628fcfb 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -130,7 +130,7 @@ services: capabilities: [gpu] chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -148,8 +148,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -199,7 +199,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker-compose.yml b/docker-compose.yml index a7ddf2f04..2b90227a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: restart: unless-stopped chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -126,8 +126,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -177,7 +177,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker/chromium-ca-policy.md b/docker/chromium-ca-policy.md new file mode 100644 index 000000000..b4852ac3a --- /dev/null +++ b/docker/chromium-ca-policy.md @@ -0,0 +1,20 @@ +# Chromium incident distrust policy + +`chromium-emudhra.json` is installed in the Docker image at `/etc/chromium/policies/managed/odysseus-emudhra.json`. Chromium 132 and later support `CADistrustedCertificates`: a list of base64-encoded DER X.509 certificates whose public keys must not be accepted for TLS server authentication. Matching is by SubjectPublicKeyInfo (SPKI), so one certificate per compromised key also blocks its cross-signs. This policy adds no trust, disables no certificate checks, and does not require a revocation-list download. Native browsers and other browser distributions require separately managed installation of the appropriate policy. + +Policy semantics: [Chromium's version-155 definition](https://github.com/chromium/chromium/blob/155.0.8059.79/components/policy/resources/templates/policy_definitions/CertificateManagement/CADistrustedCertificates.yaml). Linux installation paths: [Chromium policy documentation](https://www.chromium.org/administrators/configuring-policy-for-extensions/#linux). + +Public certificate source, retrieved 2026-10-21: [incident intermediate inventory](https://emudhra.we-have-a-problem.com/impacted-intermediates.json). Its downloaded JSON has SHA-256 `f67b3abe9b4879f4e91874ed041beccd71257f1db41c588c724ccf12aa90c59e`. Each selected certificate's DER fingerprint and public-key fingerprint were checked against that inventory before inclusion. The eight unique keys below are covered, including the code-signing intermediate keys listed in the incident. The policy contains only public certificates; no private key is included. + +| Intermediate | SHA-256 of DER SPKI | +| --- | --- | +| EM OV TLS CA - G3B | `43bffa19d6ef651451f6841e019931e96f6f2eaeff28ab4b6f1ce4576de2717b` | +| EM DV TLS CA - G3C | `49dd353dc2b919a3770645117d0e60c16115e7d1bc259eefe734b00ea434fe5f` | +| EM EV TLS CA - G3C | `5c0660a2b164910d8ff589c18e14552e9e0ec237dcefe2cb736d41a4811f81a0` | +| EM OV TLS CA - G3C | `63ecefee431d49523bc1136c4e72d0206a477086caf606cd5cf025686c109cf8` | +| EM OV CS CA - G3C | `92c063d67e8f14db71d288e11266ffdaf69cbf115f7bab14f3bc2f68d25e0e30` | +| EM EV TLS CA - G3B | `e0254c7fcdc265e2f0f8a428dabb75ab4ae039b4872d62f5ee57f0877d5a8a41` | +| EM DV TLS CA - G3B | `ebed6475c2c2d80fcb199f3fe8dd6e6b3ce60b77eb15d4e42c3dec0c6f5b5105` | +| EM OV CS CA - G3B | `ec4ffa1ad0ac88081314b7aaffef6a91f97cd57291bb5ba686c5415f5eb4f6b6` | + +`tests/test_chromium_ca_policy.py` verifies that the policy distrusts exactly these eight keys using valid intermediate CA certificates and adds no trust settings. Changes to the encoded material need a fresh source/fingerprint check and behavioral validation: all twelve published incident HTTPS chains must be rejected by the actual Chromium binary while ordinary HTTPS still succeeds. Confirm loading and any policy errors at `chrome://policy`. Do not treat a passing PEM-store check as a browser-policy check. diff --git a/docker/chromium-emudhra.json b/docker/chromium-emudhra.json new file mode 100644 index 000000000..104f82f60 --- /dev/null +++ b/docker/chromium-emudhra.json @@ -0,0 +1,12 @@ +{ + "CADistrustedCertificates": [ + "MIIDZjCCAu2gAwIBAgIPbK2jZ/scEACIEg6iAtVoMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNjAwMTJaFw00MDA0MjgxODMwMDBaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJWMRswGQYDVQQDDBJFTSBPViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARziXqg6zz9yosSQC/68KvE4SErr43ud4rJ7i1tuD+sKAxeBWOK56Ks1xwyKo42HN659pxDArndjXPRzGyqsswK7k4hbOl53VGKSvS2JuVoTN1lgDftDIVOTMEvTPhkbQ2jggGWMIIBkjAfBgNVHSMEGDAWgBThOqshfPHjTVSu1HK3/YtZQiCQVDAdBgNVHQ4EFgQUm9hlUa/n5yw3rQjuaRnZvzeErgowHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMF0GA1UdIARWMFQwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTANBgsrBgEEAYOOIQECbjAIBgZngQwBAgIwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1PVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNnADBkAjBAk+5BizdJhrED3wb7F2Gm7bWI60dlNd075nghT2TrR4NWKZtJZcbc28xxIY/NONICMF6mPdsjosaqmvEMWqKcBh4vsIvXSKnQj5o8zrFB4oKS+accjYTe8Dz3bQjauvy9uw==", + "MIIDaDCCAu6gAwIBAgIPWIsuPCu4zaE+GhIJJUMhMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM1MDlaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOnp4YsjETTJN3WkDOmeCwO/u/kBV5ooSP6A3G6O9kZRRTuFypDPjYw8CvRiAN9nlFJ+bd9uUooHfmHsv6oiLdL3+j9pMEJnubKGVZv4+8OX2sJpVbMyLpQVh3xsqRB0Bo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFK5+EFw8KZbBqEcnBVnx4D3U5QrzMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMEN2HWMocbuWdy1d33pH6JpMvP/V/DIS5PtZbv1Q7TXEbxrcyxgyuddOJscYDJJlAIwB4sH1BpbScDzkdD4tJBSwrG/1Mx82vb8YaySehNuu1+1oXNdAEKwaJ0mVzScy2+6", + "MIIDaDCCAu6gAwIBAgIQAIAJkopkEHYBrwX6LxnxCTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTUzOTMxWhcNNDAwNDI4MTgzMDAwWjBAMQswCQYDVQQGEwJVUzEUMBIGA1UECgwLZU11ZGhyYSBJbmMxGzAZBgNVBAMMEkVNIEVWIFRMUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABC1W6Cmm+4LGSkaX2WCvlR6QZ66R+SbTQwbaSKiq63hptqZ+bSdCbgRCqgOzbhmI3/Y2UU7hzwf1uks/8qBmRa59MdWGatMao9JG6wsxd+6Zy3VmxRbOdICKZo+R8MR/yqOCAZUwggGRMB8GA1UdIwQYMBaAFOE6qyF88eNNVK7Ucrf9i1lCIJBUMB0GA1UdDgQWBBSmJV6AqU2385Z8G8eR35K87kvrXzAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwXAYDVR0gBFUwUzA5BgsrBgEEAYOOIQEAATAqMCgGCCsGAQUFBwIBFhxodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tMA0GCysGAQQBg44hAQJ4MAcGBWeBDAEBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRVZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwBy1wTnzH4lXDTy2PD91ysLjvnlVN3JbJmc1BIMEU8TOLkYVHibMc9HeyonikFSfmAjEA6uE3QRxlrZTz/1/lXPTEMUhoauAtycagtlNFp79kjhvNa2/fAal1GNxsOutLnr0Y", + "MIIDaDCCAu6gAwIBAgIPKWkmwiejoVJjkrGUgqUEMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM3NDFaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gT1YgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEE4JRzVKskpjByKrZdzId65fnCb0LTKCspV3IrVHc/J0ViaFZfjpRIOHv1jHGPhg1ysrimGmmLhYx6KKBgatWYW3EXZGKvAnzqw5jz3kqpNztc4e+N7ulaVSfHAKycO1no4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFDTq6gzryQra+dDF4u2beMmHnhtsMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAm4wCAYGZ4EMAQICMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMSZt2xsPeumRdeSMSQ3CbzIZeQ/q7GDCWfeE1Pq+mhgA6gRM44cNVF3xJ+wsxJk+wIwFJbaDMokD8Ei6eNOiA8yUMYvfzQyngmrhZQMdPCK1SewrR794rSb9vGqxZzelQth", + "MIIDWzCCAuGgAwIBAgIPVEt/kiwxriIKSpIpFvbmMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2Mzg0MFoXDTQwMDQyODE4MzAwMFowPzELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRowGAYDVQQDDBFFTSBPViBDUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABLnw4GNHAqacpqrdRDz9YsQbm6SyCSRpfXl052kaEm+5avs+NML9O8X0gwYfJl3CpYQ2NHMjUQQ2so8E84YjYwZ5gOJQPgJTLWVGI1pxZaJZoe9Jb3QbX97neHII5Lj676OCAYswggGHMB8GA1UdIwQYMBaAFGh0A/iFyo1UUFcdFuDI3kUNq80tMB0GA1UdDgQWBBTub3MpjW+/p7Mbw5U09hVWRHhmtzATBgNVHSUEDDAKBggrBgEFBQcDAzBeBgNVHSAEVzBVMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDgYMKwYBBAGDjiEBAoFIMAgGBmeBDAEEATB0BggrBgEFBQcBAQRoMGYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLWMuZW1zaWduLmNvbTA+BggrBgEFBQcwAoYyaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbS9jZXJ0cy9FTU9WQ1NDQUczQy5wN2MwNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290Y3NjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNoADBlAjEA8yPg8sntOhLwV2QXJ5LEaKSpo/fnE3uOgTadK73fDhyjiqr0Io9GVuQTYqBPbQ2TAjBgeuApmBS486+iei03lRrp139dPw6k28jjGNvWsAU7A1N86J1zPJVDEyVH+nMAWC0=", + "MIIDaDCCAu2gAwIBAgIQAMfEHv6S2CXwXq6EGkqCwTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTYwMzMxWhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRVYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEJduoWmTTl9oLEFLp6lRuSHvv6pydnWigxerc/u91ppu11vwB9Cv7ym60C81kwHZ+Nv/QXZeoE8WqDGyVMOec9fCGZGjxOAZCCqXIUYQLHiUlQOeQ5LZUw4Uu1krx6GV9o4IBlTCCAZEwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFD7cINmJRpiHe4c/2IFHslwV2EcGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBcBgNVHSAEVTBTMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAngwBwYFZ4EMAQEwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1FVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNpADBmAjEA9Jq999oaOUKhFwopw33iCmt0OHIn3gjT43GfuOHR/YckXdwoNdDAlIEhz0NgRO3/AjEAvY/a3OMvktIZLYjn+WLKmp0QhrIMV+e6EpS7j4GVeOKouNp6MN/P+ByGeiRMghzB", + "MIIDaDCCAu6gAwIBAgIQAKSu5njyM2K0wORgt3LyFDAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTU1NDA4WhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE557ZwEqXc8RzXlERHahawr1eZKs3S4DhLLwgOOweTw47gS/JhZ4eZ6+wTi+LyX90hbplqpEe6KTkW2LVzAPVa+Z4s8OMtNyTfKJGDiOEU8iSk10p1E+R4VYGtjzO2rRoo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFOIeL+ccGTkOVUtiwSAkVzUkiiUUMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQi5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwLmsGXi6oFMJ2klkcEMXMug6gvxQWZWkewiDWVzQyl2L6z6DuQfPOIbh1grvdULDCAjEAxZvLlxUavG69J6wnSjC8XBbqNIQsJ6CE2QdNOe84Ptxb6VG8gt2TlfJbHii+KNvK", + "MIIDWzCCAuCgAwIBAgIPeBiTP4+FtGGaKzX7qD0KMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2MDgyN1oXDTQwMDQyODE4MzAwMFowPjELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGjAYBgNVBAMMEUVNIE9WIENTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEqrsPsGZqz88B+tUr/P1MK/LLPDSX3nOVDlgZth/mlQmHQcjEkUObNKBDHK/+W/m0wcjorgAB+fTcitgsWfNjnpbNbD1NsM7+wmw9YlF7N3YTcnQ/jordd7EAZPKr5eBQo4IBizCCAYcwHwYDVR0jBBgwFoAUaHQD+IXKjVRQVx0W4MjeRQ2rzS0wHQYDVR0OBBYEFHn66OzP2w11jaRY1p+27dx8Sd7mMBMGA1UdJQQMMAoGCCsGAQUFBwMDMF4GA1UdIARXMFUwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTAOBgwrBgEEAYOOIQECgUgwCAYGZ4EMAQQBMHQGCCsGAQUFBwEBBGgwZjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD4GCCsGAQUFBzAChjJodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZDU0NBRzNCLnA3YzA2BgNVHR8ELzAtMCugKaAnhiVodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3Rjc2NhZzMuY3JsMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMAoGCCqGSM49BAMDA2kAMGYCMQC/zr03YEELsNKqzUtH2dVFi4BaKFdyrr/6G1dIWp8KAVgsw2/u4lyyboUDaC9jDvICMQC9HTxku84tNX9dmHOtH7aC2Nu/JM0jTf6KjZAvovCJzzv1IFWSZpVLZibynMLXOrM=" + ] +} diff --git a/docker/searxng/Dockerfile b/docker/searxng/Dockerfile new file mode 100644 index 000000000..0b487e66c --- /dev/null +++ b/docker/searxng/Dockerfile @@ -0,0 +1,9 @@ +# Validation only. Deploy the unmodified upstream image selected by Compose. +ARG SEARXNG_IMAGE +FROM ${SEARXNG_IMAGE} AS validation +COPY scripts/check_ca_trust.py docker/searxng/validate_image.py /opt/odysseus-trust/ +COPY tests/fixtures/ca-trust/incident.json /opt/odysseus-trust/incident.json +RUN /usr/local/searxng/.venv/bin/python /opt/odysseus-trust/validate_image.py + +FROM validation AS live-validation +RUN /usr/local/searxng/.venv/bin/python /opt/odysseus-trust/validate_image.py --live diff --git a/docker/searxng/Dockerfile.dockerignore b/docker/searxng/Dockerfile.dockerignore new file mode 100644 index 000000000..3e7e75e3f --- /dev/null +++ b/docker/searxng/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +** +!scripts/ +!scripts/check_ca_trust.py +!docker/ +!docker/searxng/ +!docker/searxng/** +!tests/ +!tests/fixtures/ +!tests/fixtures/ca-trust/ +!tests/fixtures/ca-trust/incident.json diff --git a/docker/searxng/validate_image.py b/docker/searxng/validate_image.py new file mode 100644 index 000000000..c768cf010 --- /dev/null +++ b/docker/searxng/validate_image.py @@ -0,0 +1,162 @@ +"""Verify incident distrust and startup in the unmodified upstream image.""" + +import hashlib +import asyncio +import json +import os +from pathlib import Path +import re +import signal +import ssl +import subprocess +import sys +import tempfile +import time +import urllib.request + +import certifi + +from check_ca_trust import check_bundle + + +def check_chains(): + incident = json.loads(Path(__file__).with_name("incident.json").read_text()) + system = Path(os.environ["SSL_CERT_FILE"]) + capath = os.environ["SSL_CERT_DIR"] + stores = [["-CAfile", str(system), "-no-CApath"], + ["-CAfile", certifi.where(), "-no-CApath"], + ["-no-CAfile", "-CApath", capath]] + check_bundle(system) + check_bundle(Path(certifi.where())) + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + for fingerprint, pem in incident["certificates"].items(): + if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != fingerprint: + raise ValueError(f"incident fixture fingerprint mismatch: {fingerprint}") + (directory / fingerprint).write_text(pem) + positive = directory / "affected.pem" + positive.write_text("\n".join(incident["certificates"][fp] for fp in incident["roots"])) + for hostname, chain in incident["chains"].items(): + untrusted = directory / "untrusted.pem" + untrusted.write_text("\n".join(incident["certificates"][fp] for fp in chain[1:])) + base = ["openssl", "verify", "-no-CAstore", "-attime", str(incident["verification_time"]), + "-purpose", "sslserver", "-verify_hostname", hostname, + "-x509_strict", "-partial_chain", "-trusted_first", "-untrusted", str(untrusted)] + expectations = [(selection, False) for selection in stores] + for selection, expected in expectations + [ + (["-CAfile", str(positive), "-no-CApath"], True)]: + result = subprocess.run(base + selection + [str(directory / chain[0])], + capture_output=True, text=True) + if (result.returncode == 0) != expected: + raise RuntimeError(f"unexpected chain result: {hostname}: {selection}: {result.stderr}") + # Accept issuer lookup, self-signed and explicitly untrusted + # certificate errors, rather than tool or fixture failures. + if not expected and not re.search(r"error (?:2|18|19|20|21|27) at \d+ depth lookup", result.stderr): + raise RuntimeError(f"not a certificate trust rejection: {hostname}: {result.stderr}") + print(f"{len(incident['chains'])}/{len(incident['chains'])} incident chains rejected by system PEM, hashed directory and certifi; affected controls accepted") + + +def check_boot(live=False): + with tempfile.TemporaryDirectory() as directory: + settings = Path(directory) / "settings.yml" + settings.write_text("use_default_settings: true\nserver:\n secret_key: offline-build-validation-only\n limiter: false\nsearch:\n formats: [html, json]\n") + cache = Path(directory) / "cache" + cache.mkdir() + env = dict(os.environ, SEARXNG_SETTINGS_PATH=str(settings), + __SEARXNG_CONFIG_PATH=directory, __SEARXNG_DATA_PATH=str(cache), + GRANIAN_HOST="127.0.0.1", GRANIAN_PORT="18888") + # Do not expose a port or issue an outbound search during offline builds. + with tempfile.TemporaryFile() as log: + process = subprocess.Popen(["/usr/local/searxng/entrypoint.sh"], + cwd="/usr/local/searxng", env=env, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + healthy = False + for _ in range(60): + if process.poll() is not None: + break + try: + with opener.open("http://127.0.0.1:18888/healthz", timeout=1) as response: + healthy = response.status == 200 + except OSError: + pass + if healthy: + break + time.sleep(0.5) + if not healthy: + log.seek(0) + raise RuntimeError("SearXNG failed boot smoke: " + log.read().decode(errors="replace")[-6000:]) + print("SearXNG loopback healthcheck passed") + # Its root entrypoint regenerates system trust. Repeat the + # behavioral checks after that real startup path completes. + check_running(live=live, port=18888) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def check_running(live=False, port=None): + """Verify an existing service without repairing it or starting another one.""" + port = int(os.environ.get("GRANIAN_PORT", "8080")) if port is None else port + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(f"http://127.0.0.1:{port}/healthz", timeout=5) as response: + if response.status != 200: + raise RuntimeError("SearXNG running-service healthcheck failed") + check_chains() + for path in Path(os.environ["SSL_CERT_DIR"]).iterdir(): + if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name): + check_bundle(path) + if live: + asyncio.run(check_https()) + with opener.open(f"http://127.0.0.1:{port}/search?q=python+documentation&format=json", timeout=90) as search: + payload = json.load(search) + results = payload.get("results", []) + if not results: + raise RuntimeError("live search returned no results") + print(f"SearXNG live JSON search passed: {len(results)} results") + + +async def check_https(): + # Exercise the actual pinned SearXNG client factory with its default CA + # selection, rather than a different HTTP library's trust configuration. + sys.path.insert(0, "/usr/local/searxng") + from curl_cffi.requests.exceptions import RequestException + from searx.network.client import new_client + + client = new_client(enable_http=False, verify=True, enable_http2=True, + enable_http3=False, max_connections=2, proxies={}, + local_address=None, max_redirects=3) + try: + response = await client.get("https://pypi.org/simple/certifi/", timeout=30) + if response.status_code != 200: + raise RuntimeError(f"ordinary HTTPS failed: {response.status_code}") + print("SearXNG actual curl client ordinary HTTPS passed with verification enabled") + incident = json.loads(Path(__file__).with_name("incident.json").read_text()) + for hostname in incident["chains"]: + try: + await client.get(f"https://{hostname}/", timeout=30, allow_redirects=False) + except RequestException as exc: + # libcurl code 60 is peer certificate verification failure. + # DNS failures, timeouts and handshake EOF do not prove distrust. + if exc.code != 60: + raise RuntimeError(f"inconclusive incident TLS failure: {hostname}: {exc}") from exc + else: + raise RuntimeError(f"incident HTTPS unexpectedly accepted: {hostname}") + print(f"SearXNG actual curl client rejects {len(incident['chains'])}/{len(incident['chains'])} incident hosts with certificate errors after startup") + finally: + await client.aclose() + + +if __name__ == "__main__": + live = "--live" in sys.argv + if "--running" in sys.argv: + check_running(live=live) + else: + check_chains() + check_boot(live=live) diff --git a/docs/maintainers/README.md b/docs/maintainers/README.md new file mode 100644 index 000000000..bd5e93f24 --- /dev/null +++ b/docs/maintainers/README.md @@ -0,0 +1,38 @@ +# Odysseus maintainer handbook + +Odysseus maintainers are trusted to act, not to wait for permission. The default is individual responsibility; additional review is required when the **risk** warrants it. + +This handbook records the project's maintainer governance and gives reviewers a reusable process. It complements [CONTRIBUTING.md](../../CONTRIBUTING.md) and [SECURITY.md](../../SECURITY.md); it does not replace the PR template, repository protections, or product decisions. + +## At a glance + +| Area | Working rule | +|---|---| +| Product direction | Pewds is the Product Owner | +| Ordinary, reversible maintainer PR | Self-merge allowed after appropriate validation and applicable repository checks | +| External contribution | Maintainer review required | +| High-risk change | At least one **independent, qualified** maintainer review | +| Access | Need-to-have, least privilege, separately granted per system | +| Release | A designated Release Maintainer owns preparation and publish | +| Inactivity | Access review after 2 months without activity, notice, or response | +| Documentation | Repository Markdown is canonical | + +These are governance rules, not guarantees that a GitHub permission or branch-protection setting permits a particular action. Respect the stronger applicable automated gate. + +## Start here + +- [Roles and access](roles-and-access.md): responsibility, permissions, and continuity. +- [Product and feature proposals](product-and-features.md): product approval versus technical ownership. +- [Changes and review](changes-and-review.md): review thresholds, self-merge, contributor corrections. +- [Practical review playbook](review-playbook.md): risk, provenance, CI, CodeQL, independent audit, and handoff evidence. +- [Triage](triage.md): turning incoming reports into actionable work. +- [Testing and releases](testing-and-releases.md): evidence, platform validation, promotion to main. +- [Documentation](documentation.md): canonical sources, updates, and traceability. + +## Accountability + +Maintainers own the changes they take on, including validation, regression prevention, follow-up, and rollback when needed. Roles describe responsibilities, not a hierarchy of technical approval. + +Important product-direction decisions go through the Product Owner. Maintainers retain responsibility for feasibility, implementation, maintainability, and security. + +AI coding or review agents can help collect evidence, test hypotheses, and prepare changes, but they are not the qualified human reviewer required for high-risk changes. A maintainer must inspect and accept the evidence. diff --git a/docs/maintainers/changes-and-review.md b/docs/maintainers/changes-and-review.md new file mode 100644 index 000000000..664b2596a --- /dev/null +++ b/docs/maintainers/changes-and-review.md @@ -0,0 +1,62 @@ +# Changes and review + +Review is **risk-based**. Trusted maintainers should be able to ship ordinary work without creating a queue of ceremonial approvals. + +## Required review + +| Change | Minimum review rule | +|---|---| +| Ordinary, reversible maintainer change | Independent approval optional; self-merge permitted after appropriate validation | +| External contribution | Maintainer review required | +| High-risk change | At least one independent, qualified maintainer review required | + +Repository branch protection and required checks also apply. Never treat this document as permission to bypass a live protection rule. + +## Ordinary maintainer changes + +1. Open a PR against `dev`, scoped to one coherent change. +2. Explain intent, expected behavior, validation, and rollback if relevant. +3. Run suitable tests and inspect the actual result. +4. Ensure applicable required checks pass, address feedback, and merge when ready. + +Self-merge is allowed for ordinary reversible work. The author remains responsible for follow-up, correction, and rollback. + +## External contributions and small corrections + +External contributions receive maintainer review. + +A maintainer may directly fix a **small, obvious** problem in a contributor PR when the intended behavior is clear and the repair is faster than a separate round trip. Document what changed, preserve contributor attribution and history, avoid force-pushing, and re-run affected checks. + +Substantive or ambiguous corrections require discussion with the contributor or a separately scoped correction. A maintainer must not silently turn a contributor's PR into unrelated work. Any corrective commit changes the SHA being reviewed; update validation and audit evidence accordingly. + +## High-risk changes + +High-risk work requires **at least one independent qualified review**. The author cannot be the only approver. Examples: + +- Authentication, authorization, tenancy, sandboxing, agent tool permissions, or secrets. +- Outbound network and SSRF defenses, sensitive integrations, and deployment access. +- Destructive operations, deletion, filesystem ownership, and data migrations. +- Persistence and concurrency invariants with possible data loss. +- Major architectural contracts, release infrastructure, or changes hard to roll back. + +A high-risk review should verify the threat or failure model, exact final commit, reproducer or regression evidence, failure and rollback behavior, and the relevant CI/security findings. Use the [review playbook](review-playbook.md). + +## Cross-area changes and CODEOWNERS + +Touching multiple areas does **not** automatically require multiple approvals. Consult domain expertise when needed. `.github/CODEOWNERS` currently contains no ownership rules; the per-area map is under discussion in [issue #593](https://github.com/odysseus-dev/odysseus/issues/593). Once populated, it helps identify reviewers and accountable contacts; it is not a blanket approval requirement by itself. + +## Validation and UI evidence + +Automated tests count as runtime evidence when they truly exercise the changed behavior. Use manual validation when coverage is insufficient. + +For any change to visible UI behavior, follow the current [contributor UI checklist](../../CONTRIBUTING.md#style-and-visual-changes) and the repository PR template. Screenshots or clips are expected when they materially demonstrate the rendered result; screenshots alone do not replace behavioral tests. + +## Disagreement and escalation + +The maintainer doing the work owns ordinary implementation decisions within approved product direction and existing technical contracts. For substantive disagreements: + +1. Involve a qualified maintainer with relevant expertise. +2. Escalate unresolved cross-cutting technical issues to trusted project-wide maintainers. +3. Refer product-direction disagreements to the Product Owner. + +Security findings should be handled via [SECURITY.md](../../SECURITY.md), without disclosing exploit details in public issues. Repository protection rules should support these risk-based policies and should not accidentally force independent approval for every ordinary maintainer PR. diff --git a/docs/maintainers/documentation.md b/docs/maintainers/documentation.md new file mode 100644 index 000000000..36c88d81d --- /dev/null +++ b/docs/maintainers/documentation.md @@ -0,0 +1,32 @@ +# Documentation + +Repository Markdown is the canonical source for project documentation. A public docs site may be generated from it; avoid maintaining an independent wiki copy unless there is a deliberate future decision to do so. + +## Where information belongs + +| Location | Purpose | +|---|---| +| `/design` | Product intent, proposals, and accepted decisions (when those paths are used) | +| `/specs` | Technical contracts and invariants | +| `/website` | User, setup, configuration, and deployment guidance (published as the docs site) | +| `/docs` | Developer, architecture, security, and maintainer guidance | +| Root Markdown | Discoverable entry points and policies | + +Keep root documents easy to find: + +- `README.md` +- `CONTRIBUTING.md` +- `SECURITY.md` +- `THREAT_MODEL.md` + +Not every named directory exists today; do not invent a parallel documentation store just to match this map. + +## Updating documentation + +When behavior, architecture, a workflow, or its verification requirements change, update the canonical documentation in the same PR when appropriate. + +Separate **policy** from **examples**: examples may evolve with tooling; changes to role authority, review requirements, or product ownership require an explicit governance decision. Do not introduce a new mandatory approval rule through a checklist alone. + +Documentation PRs should link the relevant issue, record what was reviewed (links, commands, or affected features), and avoid publishing private channels, account names, local paths, secrets, or confidential incident details. + +For practical examples of review evidence, see the [review playbook](review-playbook.md). For contributor setup and PR requirements, use [CONTRIBUTING.md](../../CONTRIBUTING.md). diff --git a/docs/maintainers/product-and-features.md b/docs/maintainers/product-and-features.md new file mode 100644 index 000000000..d41055a7f --- /dev/null +++ b/docs/maintainers/product-and-features.md @@ -0,0 +1,48 @@ +# Product and feature proposals + +Pewds ([@pewdiepie-archdaemon](https://github.com/pewdiepie-archdaemon)) is the Product Owner for Odysseus and has final authority over product direction. Maintainers own technical feasibility, implementation, and safety. + +## Which changes require product approval? + +**Product Owner approval before implementation:** substantial new features, changes to product direction, and product changes that are hard to reverse. + +**No separate product approval by default:** bug fixes, security fixes, maintenance, tests, CI, documentation, internal refactors, and small, reversible product changes within existing direction. + +Maintainers may make those smaller product decisions. Pewds can ask for changes or a rollback; the maintainer owns safe rollback or recovery. + +### Urgent security and reliability fixes + +Urgent fixes or containment may proceed without prior product approval, even if behavior temporarily changes. Keep the change to what stops the immediate problem, notify Pewds promptly, and record the reason and rollback or recovery plan. Agree on any lasting product-direction change with Pewds afterwards. Normal high-risk review and repository checks still apply. + +## Product Owner initiated + +```text +Product Owner proposal or implementation + -> technical feasibility + -> plan and implementation + -> risk-based review and validation +``` + +A Product Owner proposal is already product-approved; it is not automatically technically approved or exempt from high-risk review. + +## Community or maintainer initiated + +Community proposals start as a Feature Request issue. Check [ROADMAP.md](../../ROADMAP.md) and existing issues first; take early ideas to [Discussions](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas). + +```text +Feature proposal + -> Product Owner accepts / rejects, where approval is required + -> technical feasibility + -> plan and implementation + -> risk-based review and validation +``` + +Record the decision in an issue, proposal, work item, or PR so future maintainers can discover it. Do not rely on a chat-only decision. + +## Decision boundaries + +The Product Owner decides **what belongs in Odysseus**. Maintainers decide **how approved work is built safely and maintainably**. + +A rejected or out-of-scope feature can be maintained independently in a contributor's fork. + +If the Product Owner is unavailable, work requiring prior product approval waits unless authority was explicitly delegated. Smaller decisions and urgent fixes follow the rules above. diff --git a/docs/maintainers/review-playbook.md b/docs/maintainers/review-playbook.md new file mode 100644 index 000000000..e9f0e6f99 --- /dev/null +++ b/docs/maintainers/review-playbook.md @@ -0,0 +1,129 @@ +# Practical PR review playbook + +Use this as a **repeatable review method**, not as an extra organizational approval ladder. The binding minimums remain in [changes and review](changes-and-review.md): ordinary maintainer changes may be self-merged, external PRs require maintainer review, and high-risk work requires at least one independent qualified maintainer review. + +No single checklist replaces engineering judgment. Increase depth with risk. + +## 1. Understand the PR and establish its review gates + +- Establish the problem, what the PR fundamentally proposes, and why the approach makes sense. +- Check product direction, architecture fit, and scope before detailed review. +- Write down the questions the review must answer, the evidence needed, and what would block further review or approval. +- Confirm repository, source fork, target branch (`dev`), exact **HEAD SHA**, and current base. +- Review the full effective diff, commit history, issue, prior feedback, and related upstream changes. +- Confirm the branch is mergeable against the **current** `dev`, not an older local snapshot. +- Identify the user-facing contract, affected data and users, security boundaries, and rollback plan. +- Separate pre-existing upstream problems from regressions introduced or made reachable by the PR. + +Example read-only commands: + +```bash +REPO="odysseus-dev/odysseus" +PR=1234 +gh pr view "$PR" --repo "$REPO" --json title,state,isDraft,headRefOid,baseRefName,mergeable +gh pr diff "$PR" --repo "$REPO" +gh pr checks "$PR" --repo "$REPO" +``` + +A green check on an old commit does not verify the revised branch. + +## 2. Review the failure and threat model + +For the relevant change, ask what happens with malformed input, missing auth state, wrong owner, concurrent requests, stale caches, partial failure, retries, cancelled jobs, unavailable dependencies, and restart/recovery. + +Prioritize boundaries that can lose data or authority: identity, delegated API tokens, filesystem confinement, email deletion, persistence versioning, network/SSRF destinations, secrets, and agent tool execution. + +For security claims, demonstrate the source-to-sink path and the *actual* guard. Tests must represent production middleware or transport where a unit mock could conceal a bypass. + +## 3. Correct without obscuring contributor work + +For small, unambiguous issues, maintainers may add focused corrections directly to a contributor PR under [the contribution review policy](changes-and-review.md#external-contributions-and-small-corrections). Preserve provenance, explain the change, prefer normal commits and fast-forward pushes, and avoid unrelated rewrites. + +For substantive or ambiguous work, ask the contributor or prepare a clearly separated patch. Any code change means validating **the new SHA**, not approving based solely on the previous audit. + +## 4. Build an evidence record + +Run contributor code under the [runtime isolation rules](testing-and-releases.md#runtime-isolation). + +Useful evidence includes: + +- Pre-fix reproduction, expected outcome, and actual outcome. +- Focused regression tests for the fix and broad suites relevant to cross-cutting effects. +- Actual runtime checks (browser/UI, backend, Docker, supported OS) where necessary. +- Exact executed commands, result counts, skipped cases, and environmental limitations. +- GitHub required checks and conflict-free state on the exact final SHA. +- Relevant operational consequences (data migration, rollback, compatibility). + +Keep audits reproducible. A screenshot alone is not a functional test, and a passing mocked test is not proof that a production integration path is safe. + +## 5. Independent review for high-risk changes + +A separate **qualified maintainer** must inspect high-risk changes; the implementing author cannot be the sole reviewer. Independent coding agents may be useful for adversarial tests and cross-checks, but **an AI agent is not itself a GitHub approver** and does not replace human accountability. + +To avoid confirmation bias, give the independent reviewer the current SHA, changed files, acceptance criteria, known reproduction, and explicit permission to challenge the proposed solution. Preserve unresolved findings and require a new audit when meaningful fixes change the diff. + +## CodeQL and security scanning + +A successful CodeQL workflow means analysis ran, not that the security dashboard is empty. Likewise, resolving a PR discussion does not necessarily resolve its code-scanning alert. + +For each relevant alert: + +1. Record the alert number, rule, severity, file/line, state, and source-to-sink trace if applicable. +2. Compare with the current `dev` baseline; decide whether the PR introduced, exposed, or did not affect it. +3. Reproduce or reason through the actual production behavior, including validation and authorization. +4. **Fix** credible vulnerabilities, preferably with a regression test. Do not hide them by broad suppressions. +5. **Recommend dismissal** only for a demonstrable false positive or a stale/invalid finding, with a precise technical explanation; only authorized maintainers should dismiss alerts. +6. Document genuine but intentionally accepted risk separately. A true positive is not a false positive merely because it is deliberate. +7. Rerun or inspect updated analysis on the final commit and confirm no blocking issues remain. + +Never publish exploit payloads, secrets, or credentials in public review comments. See [SECURITY.md](../../SECURITY.md). + +## 6. Decide and integrate + +Before approval or self-merge, verify: + +- Current branch and reviewed SHA are still the ones to be merged. +- All applicable required checks are complete and successful. +- High-risk independent review is present, where required. +- No unresolved, credible P0/P1/P2 security or data-integrity defects remain in the change. Resolve or explicitly document P3 observations with a tracked owner and priority. +- Review discussions, compatibility risks, and operational handoffs have been addressed. +- The correct target is `dev`; `main` is curated for releases. + +This P0-P3 classification is [triage guidance](triage.md#practical-priority-language), not a replacement for branch protection or an excuse to self-approve high-risk work. Maintainers may apply a stricter bar where warranted. + +For a final manual merge, GitHub CLI supports an exact-head guard: + +```bash +gh pr merge 1234 --repo odysseus-dev/odysseus --squash \ + --match-head-commit +``` + +Use `--delete-branch` only when removing the source branch is appropriate and does not disrupt another workstream. Do not force a merge past review or security requirements. + +## 7. Post-merge follow-up + +Verify the PR actually merged and reference its integration commit. Record remaining operational work, such as schema migrations, backup/restore verification, service restarts, or compatibility documentation. A code change to a container mount does **not** by itself migrate existing runtime data. + +Do not silently mark a deferred data migration complete; track its owner, evidence, rollback, and execution separately. + +## Reusable reviewer handoff + +```text +PR / target: +Reviewed source HEAD SHA: +Current target branch SHA: +Scope and changed files: +Impact and risk classification: +Security / correctness invariants challenged: +Pre-fix reproducer and post-fix evidence: +Tests run, skipped, and environment: +CodeQL / scanning alert dispositions: +Required GitHub checks (exact SHA): +Mergeability and remaining conflicts: +Unresolved P0 / P1 / P2 / P3 findings: +Rollback / operational follow-ups: +Independent qualified review required? Completed? +Verdict: APPROVED / CHANGES REQUIRED / BLOCKED +``` + +State what was actually verified. Do not write "all checks green" if only a subset of workflows was inspected. diff --git a/docs/maintainers/roles-and-access.md b/docs/maintainers/roles-and-access.md new file mode 100644 index 000000000..a98fd3258 --- /dev/null +++ b/docs/maintainers/roles-and-access.md @@ -0,0 +1,40 @@ +# Roles and access + +Roles clarify who is accountable. They do not create a management chain or automatically grant privileges. + +## Responsibility and typical access + +| Responsibility | Typical GitHub access | Purpose | +|---|---|---| +| Product Owner | No automatic repository access implied | Product direction | +| Organization Owner | Organization Owner | Organization administration and continuity | +| Repository Admin | Admin | Repository settings, permissions, and protection rules | +| Maintainer | Write by default | Implement, review, merge, and maintain | +| Triage | Triage | Organize intake, de-duplicate, and prepare work | +| Contributor / Platform Tester | Public / Read as appropriate | Contribute code or provide validation | + +Maintain access may be granted when repository-management duties require it. + +## Maintainer focus + +- **Core Maintainer:** project-wide judgment, difficult technical decisions, governance support. +- **Technical Maintainer:** implementation, architecture, validation, technical health. +- **Project Maintainer:** review, contributor coordination, documentation, releases, organization. + +These are responsibility labels. No label automatically changes GitHub permissions or overrides the [review rules](changes-and-review.md). + +## Functional responsibilities + +Maintainers may coordinate Release, Security, Infrastructure and CI, Community, or Test and Validation. These are duties, not ranks. For each release, designate one Release Maintainer and record the handoff if ownership changes. + +## Least privilege and continuity + +Grant access because it is necessary for current work, not because of a title. GitHub, Matrix, Plane, infrastructure, SSH, deployment systems, secrets, and external services are separate access surfaces. + +For elevated or sensitive access, document a responsible owner and recovery route in an appropriate **private** administrative system. Do not publish tokens, SSH details, personal addresses, or credential locations in this handbook. Remove access promptly when it is no longer needed. + +## Inactivity + +Tell an admin when you expect to be unavailable; a communicated absence is fine. + +If there is no meaningful maintainer or triage activity for **two months**, no prior notice, and no response to reasonable follow-up, elevated access may be reviewed and removed. This is access hygiene, not punishment. Returning members may regain access appropriate to their active responsibilities. diff --git a/docs/maintainers/testing-and-releases.md b/docs/maintainers/testing-and-releases.md new file mode 100644 index 000000000..23ab33853 --- /dev/null +++ b/docs/maintainers/testing-and-releases.md @@ -0,0 +1,52 @@ +# Testing and releases + +Validation must match the actual change, and evidence must state what was **run**, **observed**, and **not tested**. + +`dev` is the integration and stabilization branch. `main` is the curated release destination. Changes land in `dev`; a release promotes a verified `dev` revision to `main`. + +## Runtime isolation + +Run contributor-controlled tests, builds, installs, and repros in an isolated, secretless environment. Use disposable containers or microVMs where appropriate, with no credentials, sensitive host files, production data, or host/container-management sockets exposed. If safe isolation is unavailable, record the check as not run. + +Agents may use a skill for the maintainer's preferred environment and handling; the same isolation requirements apply. + +## Change validation + +- Run focused tests that cover the modified behavior and failure cases. +- Run broader regressions where risk crosses module boundaries. +- For authentication, data loss, or similar high-risk work, include adversarial and rollback tests. +- Reproduce the issue on the pre-fix version when safe and practical, then prove the correction. +- For visual changes, run the application and attach the screenshots or clips required by [CONTRIBUTING.md](../../CONTRIBUTING.md). +- Record exact commands, environment, result, skipped tests, and blockers. Never claim execution from a command you did not run. + +GitHub Actions success means a workflow finished successfully; it does **not** mean CodeQL found no alerts. Use [review playbook](review-playbook.md#codeql-and-security-scanning) to assess findings separately. A review must tie required CI and the final diff to the **same commit SHA**. + +## Supported-environment evidence + +Platform Testers may validate without maintainer permissions. Useful coverage includes: + +- Windows native and WSL. +- macOS Apple Silicon. +- macOS Intel while supported. +- Linux native on the reference distribution. +- Docker on supported hosts. + +Record the environment and configuration for each result. Do not imply that a platform was tested when it was not. + +## Release ownership + +A designated **Release Maintainer** owns preparation and the publish decision once the release criteria are met. Other maintainers and testers supply the evidence needed for the change set. If the Release Maintainer becomes unavailable, another maintainer explicitly takes over. + +## Release checklist + +Before promoting `dev` to `main`: + +1. Identify and freeze the release candidate SHA; check the actual content that will ship. +2. Verify required checks and relevant supported-environment validation. +3. Confirm high-risk and data-migration findings are resolved or addressed through an explicit approved plan. +4. Document skipped coverage, known issues, compatibility and upgrade implications. +5. Prepare the version, tag, artifacts, and release notes. +6. Confirm rollback/recovery, including any manual persistence or data migration steps. +7. Publish and record what was promoted, the verification evidence, and operational follow-ups. + +Publishing does not require ceremonial Core or domain sign-off when the criteria are met. Exceptional or disputed releases follow the normal high-risk escalation path. diff --git a/docs/maintainers/triage.md b/docs/maintainers/triage.md new file mode 100644 index 000000000..d2f47f8ab --- /dev/null +++ b/docs/maintainers/triage.md @@ -0,0 +1,48 @@ +# Triage + +Triage turns reports into evidence-backed work that a maintainer can pick up without guessing. + +## Intake flow + +```text +Report + -> reproduce / clarify + -> identify duplicates and affected versions + -> classify impact, urgency, and risk + -> prepare actionable work item + -> maintainer takes ownership + -> PR, verification, and follow-up +``` + +## A useful work item + +Record what is known and distinguish it from what remains unverified: + +- Original report and reproducible steps; installation method, OS, version/commit, and relevant environment. +- Expected versus actual behavior, including user-visible impact. +- Evidence: sanitized logs, screenshots, failing tests, or controlled reproduction. +- Related issues, PRs, duplicates, or a known prior regression. +- Scope, acceptance criteria, and non-goals. +- Priority, ownership, and relevant domain expertise. +- Remaining unknowns, risks, and safe recovery or workaround when available. + +Small and obvious fixes may go straight to a focused PR, consistent with [CONTRIBUTING.md](../../CONTRIBUTING.md) and [changes and review](changes-and-review.md). + +## Practical priority language + +Use P0-P3 as **triage shorthand**, not as a substitute for verified exploitability or the project's actual GitHub labels: + +| Priority | Typical impact | Expected response | +|---|---|---| +| P0 | Active exploitation, broad irreversible loss, catastrophic outage | Immediate containment and escalation | +| P1 | Credible severe security or data-integrity vulnerability | Urgent owner assignment and validated fix | +| P2 | Significant correctness, security, or operational regression | Prioritize and resolve before approving affected high-risk changes | +| P3 | Limited impact, maintainability, or low-risk polish | Fix or explicitly track with rationale | + +Do not call a concern harmless merely because an automated check is green. Escalate privately when reproduction would disclose a vulnerability. + +## Where decisions live + +Matrix is for coordination; durable decisions belong in GitHub, Plane, a specification, or the relevant PR. Keep confidential security and access details in appropriate private channels. Do not leave important project decisions only in chat history. + +See the [review playbook](review-playbook.md) for converting triage into review evidence and handling contested or uncertain findings. diff --git a/docs/searxng-trust-image.md b/docs/searxng-trust-image.md new file mode 100644 index 000000000..e0f406e82 --- /dev/null +++ b/docs/searxng-trust-image.md @@ -0,0 +1,39 @@ +# Upstream SearXNG trust validation + +SearXNG owns independent system and Python certificate stores. Updating Odysseus's Python environment does not update this sidecar. The base and both standalone GPU Compose files now select the official `docker.io/searxng/searxng:2026.10.24-ef0ec03da` image at immutable multiarch digest `sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8`. Its system and certifi stores remove the affected eMudhra trust. The [upstream certifi update](https://github.com/searxng/searxng/pull/6881) is included in this release. + +Deploy the upstream image directly. The validation Dockerfile adds public fixtures and diagnostics only; no repair layer, custom runtime image or project registry publisher is required. `scripts/check_ca_trust.py` owns the known-certificate inventory for both the application and SearXNG checks. + +## Validate a pin update + +Run from the repository root on a disposable Docker host. The helper uses a fresh configuration volume, omits published ports and does not load the operator's `.env`. It retains the base Compose settings wrapper, healthcheck and capabilities. Outbound network access and working search engines are required for the live checks. + +```sh +image=$(docker compose --env-file /dev/null -f docker-compose.yml config --format json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["services"]["searxng"]["image"])') + +# Offline build-step checks, including actual entrypoint and loopback health. +# Pulling the selected base still requires registry access. +docker build --network=none --target validation \ + --build-arg "SEARXNG_IMAGE=$image" -f docker/searxng/Dockerfile . + +# Exercise the exact unmodified deployment image after an anonymous pull. +docker pull "$image" +bash .github/scripts/validate-searxng-image.sh "$image" +``` + +The offline checks require all fourteen incident chains to fail certificate verification through the system PEM bundle, hashed CA directory and certifi. The affected positive-control roots must accept those same chains. Each fixture's DER fingerprint is verified first, and a fixed verification timestamp prevents expiry from hiding a broken mitigation. Checks run again after the real upstream root entrypoint regenerates trust. + +The live checks use SearXNG's actual curl client factory. Ordinary verified HTTPS must succeed, all fourteen incident hosts must fail with curl certificate error 60, and a JSON search must return results. DNS failures, timeouts and generic handshake failures are inconclusive and fail validation. The checks inspect the already-running service without changing its trust stores. + +The `ci / SearXNG trust` workflow runs these offline and Compose/live checks on native amd64 and arm64 runners. It selects the same immutable image as base Compose, pulls without registry credentials, and has read-only repository permissions. It neither publishes a replacement image nor requires package-write credentials. Update all three deployment pins together; the alignment regression guards the standalone GPU files. + +## Rollout and scope + +Changing a Compose pin does not replace running containers. After updating the checkout, run `docker compose pull searxng` and `docker compose up -d --no-deps searxng`, then inspect health, retained settings and actual Odysseus search integration. Use the appropriate standalone GPU Compose file when that is the deployment being managed. + +Operator-mounted trust, custom engines, persisted packages and independently managed clients need separate effective-client checks. An amd64 result does not establish arm64 behavior. The fixture-based check detects known certificate fingerprints; it does not model every native revocation mechanism or arbitrary reissued certificates sharing a compromised key. See [outbound TLS trust](tls-trust.md) for the application, bootstrap, Chromium and other runtime boundaries. + +## Public fixture provenance + +The public CA inventory comes from the incident's [roots](https://emudhra.we-have-a-problem.com/roots.json) and [intermediates](https://emudhra.we-have-a-problem.com/impacted-intermediates.json). The additional cross-signed issuer and fourteen chain fixtures come from [chains.zip](https://emudhra.we-have-a-problem.com/chains.zip), SHA-256 `7b949956955fc501a30f4d98d2f513d34c0bbbe92e29a968194c96e097ce2345`. Fixtures contain public certificates only. See [the fixture notes](../tests/fixtures/ca-trust/README.md). diff --git a/docs/tls-trust.md b/docs/tls-trust.md new file mode 100644 index 000000000..71f0fe76e --- /dev/null +++ b/docs/tls-trust.md @@ -0,0 +1,75 @@ +# Outbound HTTPS trust + +Odysseus uses both Python package CA bundles and system trust. HTTPX uses certifi by default; the guarded web/PDF fetcher and webhook delivery explicitly use Python's system SSL context. A configured `LLM_CA_BUNDLE` adds certificates to system trust for model calls. Upgrading certifi alone cannot repair every outbound HTTPS path. + +For the October 2026 eMudhra incident, use certifi 2026.10.23 or newer, pip 26.2.2 or newer, and uv 0.13.5 or newer wherever uv is used. Pip has its own vendored certifi and also loads system roots. Update the operating system's certificate package separately. See the [Python security announcement](https://discuss.python.org/t/109576) and [incident details](https://emudhra.we-have-a-problem.com/). + +Before other pip installs, bootstrap the selected environment with the integrity-pinned wheels: + +```sh +python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +python -m pip install -r requirements.txt +``` + +The pinned bootstrap protects package integrity even when the old installer's certificate bundle is affected. It is not an operating-system trust-store update. Native Windows/macOS trust and command-line OpenSSL PEM stores may differ; follow the platform vendor's remediation for every store in use. Restart Odysseus and its subprocesses after updating trust. + +On Linux/OpenSSL deployments, inspect both the effective CA file and hashed CA directory, plus the two Python bundles: + +```sh +python scripts/check_ca_trust.py --system --python --overrides --verbose +``` + +Explicit PEM bundles can also be passed as positional arguments. The check rejects the four published root fingerprints, all sixteen published intermediate certificates (including their cross-signs), the cross-signed G3 issuer in the published test chains, and malformed/empty bundles. Checking intermediates matters because adding one as a custom trust anchor can restore an affected chain even after its roots have been removed. + +`--overrides` checks every nonempty `LLM_CA_BUNDLE`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `PIP_CERT`, `NODE_EXTRA_CA_CERTS` and `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` setting in the checker's process environment, including configured fallback settings. Requests/pip hashed-directory overrides are inspected too; `--system` covers `SSL_CERT_DIR`. Run with the application's environment and filesystem access. The script does not load `.env` or application configuration files; pass any CA paths configured only there explicitly, for example `python scripts/check_ca_trust.py --system --python /path/to/private-ca.pem`. Missing or invalid configured files fail the check. Selecting only `--overrides` with no configured paths also fails instead of reporting that stores were validated. + +`--verbose` reports the Python executable/prefix, user home, effective UID/GID where available, and resolved CA paths, including bundles selected from user-installed packages. Compare these with the running application's identity and interpreter. It does not dump environment variables or credentials. + +This is an offline inventory check of known certificate fingerprints. It does not modify stores, reject requests at runtime, model native OS revocation, or detect arbitrary reissued certificates sharing the compromised keys. Chromium's policy separately matches public keys. Check mounted or replaced CA files again after deployment, remove affected trust through the owning platform/client configuration, and restart clients that cache SSL contexts. A passing build cannot validate custom files introduced later. + +Both Docker build stages use Python 3.14.9, refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if known affected CA certificates remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of known affected CA certificates. [grpcio 1.84.1](https://github.com/grpc/grpc/releases/tag/v1.84.1) includes the upstream root-bundle correction and is the application minimum. Retain the checked override as a consistent image trust selection; older persisted Python environments and explicit client roots still need independent checks. + +The image also bundles Docker CLI [29.9.2](https://github.com/moby/moby/releases/tag/docker-v29.9.2), built with Go 1.26.10's distrust of the compromised intermediate keys. Cookbook uses this client for optional Docker operations. Default Compose does not mount the host Docker socket; local daemon access requires the explicit [host-Docker overlay](../docker/host-docker.yml), whose usual Unix-socket connection does not use TLS. The client update is maintenance of a shipped tool, not evidence that these local calls were exposed. It does not update the host daemon or other containers' clients and trust stores; assess those separately. + +Chromium uses a separate trust store. The Docker image installs a managed `CADistrustedCertificates` policy for the eight compromised intermediate public keys, covering their cross-signs without depending on background revocation updates. It adds no trusted certificate and does not disable certificate verification. The policy and public-certificate provenance are documented in [the browser policy note](../docker/chromium-ca-policy.md). Native browser installations and independently managed profiles require their own platform policy or vendor update; this image policy does not configure host browsers. + +To replace only Odysseus in an already running Compose stack, build from the corrected checkout with a distinct local image tag. Use the same Compose files and project name as the existing deployment for every command below: + +```sh +export ODYSSEUS_IMAGE="odysseus-local:ca-$(git rev-parse --short=12 HEAD)" +docker compose build --pull --no-cache odysseus +docker compose up -d --no-deps --pull never --no-build odysseus +docker image inspect --format '{{.Id}}' "$ODYSSEUS_IMAGE" +docker inspect --format '{{.Image}}' "$(docker compose ps -q odysseus)" +``` + +The two image IDs must match. Keep the selected `ODYSSEUS_IMAGE` in the deployment configuration for subsequent recreations. Compose can prefer the registry image when both `image` and `build` are present; the explicit pull/build flags keep this replacement on the local build. `--no-deps` leaves the existing sidecars running and is intended for an existing stack. For registry deployments, select a verified corrected immutable tag or digest. A merge to `dev` publishes `:dev`; it does not update `:latest`, which is published from `main`. + +Check the standard image as its configured app account, using the image's Python interpreter and the same user-bin PATH prefix as the entrypoint: + +```sh +docker compose exec -T --user 0 odysseus sh -ec ' + gosu_bin=$(command -v gosu) + app_user=$(getent passwd "${PUID:-1000}" | cut -d: -f1) + test -n "$app_user" + exec "$gosu_bin" "$app_user" env PATH="/app/.local/bin:$PATH" \ + /usr/local/bin/python /app/scripts/check_ca_trust.py \ + --system --python --grpc --overrides --verbose +' +``` + +`gosu` selects the account's HOME and drops privileges before Python runs, so the check includes its persisted user packages. The entrypoint itself is not rerun. Plain `docker compose exec ... python` uses the image's default root user and can miss `/app/.local` packages. A custom service user, entrypoint, virtual environment or launch command requires matching that deployment's actual user, HOME and Python interpreter instead of copying the standard command unchanged. Docker exec inherits container-creation environment, not variables exported later by the app or entrypoint: reproduce any additional Python/TLS settings with exec's `--env` options, and pass CA paths configured only in application files explicitly. Confirm the verbose identity and selected paths before accepting the result. + +For containers started non-root or without the SETUID/SETGID capabilities needed by `gosu`, use `docker compose exec --user UID:GID --env HOME=/actual/app/home` with the deployment's actual numeric IDs, home, PATH and interpreter instead. Do not add capabilities merely to run this check. + +SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. + +For behavioral validation, all fourteen [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs and two chains without Certificate Transparency entries, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. In Chromium, confirm the managed policy is loaded at `chrome://policy` and test the incident endpoints through the actual browser path with certificate verification enabled. The PEM checker cannot inspect Chromium's policy or its active verifier. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. + +## Follow-up releases + +The maintained bootstrap selects certifi 2026.10.23 with its verified wheel hash; pip remains 26.2.2. The certifi update removes the C1/C3 roots in addition to the earlier incident changes. CI uses uv 0.13.5, which adds verifier-level rejection of the compromised intermediate keys and updated managed CPython builds. Existing virtual environments do not update automatically. See the [certifi change](https://github.com/certifi/python-certifi/commit/b60466c9f2882d9cf088943c650b0faac06c080c), [uv release](https://github.com/astral-sh/uv/releases/tag/0.13.5), and [Python security releases](https://blog.python.org/2026/10/python-3151-3149-31317/). The pinned bootstrap still replaces the selected versions exactly; preserving newer installed versions and private-mirror support remain separate maintenance. + +SearXNG remains an independent trust owner. Use the [upstream-image validation and rollout instructions](searxng-trust-image.md); updating these application packages does not repair an existing sidecar. Host daemons, native browser policy, optional compiled clients and independently managed integrations require their own effective-client checks. + +CPython's [intermediate-key verifier block](https://github.com/python/cpython/pull/160042) and its [3.14 backport](https://github.com/python/cpython/pull/160569) merged on October 24. They are distinct from Python 3.14.9's bundled-pip update. No released 3.14 interpreter containing that verifier change was identified when this image pin was selected; retain the corrected effective trust stores and track the subsequent security release. hf-xet's [trust fix](https://github.com/huggingface/xet-core/pull/1038) is also merged but not included in the latest published 1.7.0 release at that check. Agent-browser 0.39.2 and primp 2.0.1 are not established incident fixes for their independent native trust paths. A version bump alone does not establish that the active client uses corrected trust. diff --git a/frontend/src/features/notes/ui/panel.js b/frontend/src/features/notes/ui/panel.js index 0a6031233..2265897c0 100644 --- a/frontend/src/features/notes/ui/panel.js +++ b/frontend/src/features/notes/ui/panel.js @@ -190,7 +190,7 @@ export function createPanel({ // which left it as a side panel squeezing the chat). if (window.innerWidth <= 768) { pane.style.position = 'fixed'; - pane.style.inset = '0'; + pane.style.inset = 'var(--sheet-top) 0 0'; pane.style.width = '100%'; pane.style.maxWidth = '100%'; pane.style.zIndex = '170'; diff --git a/integrations/claude/skills/odysseus/SKILL.md b/integrations/claude/skills/odysseus/SKILL.md index 31b40ee01..c8bd94db3 100644 --- a/integrations/claude/skills/odysseus/SKILL.md +++ b/integrations/claude/skills/odysseus/SKILL.md @@ -100,6 +100,14 @@ python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory - `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. - `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. +## Chat history + +- `GET /api/codex/chat/{session_id}?offset=0&limit=100` — read a paginated conversation transcript owned by the token owner. Requires the existing `chat` scope. + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py GET /api/codex/chat/SESSION_ID +``` + ## Email draft + send - Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. diff --git a/integrations/claude/skills/odysseus/scripts/odysseus_api.py b/integrations/claude/skills/odysseus/scripts/odysseus_api.py index 8a22eb494..54df59fa1 100755 --- a/integrations/claude/skills/odysseus/scripts/odysseus_api.py +++ b/integrations/claude/skills/odysseus/scripts/odysseus_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Small Odysseus scoped API helper for Codex terminal sessions.""" +"""Small Odysseus scoped API helper for Claude Code terminal sessions.""" from __future__ import annotations @@ -44,7 +44,7 @@ def _config() -> tuple[str, str] | None: if not token: missing.append("ODYSSEUS_API_TOKEN") if missing: - print(f"missing {', '.join(missing)}; create a Codex Agent token in Odysseus Settings", file=sys.stderr) + print(f"missing {', '.join(missing)}; create a Claude Agent token in Odysseus Settings", file=sys.stderr) return None return base_url, token diff --git a/integrations/codex/skills/odysseus/SKILL.md b/integrations/codex/skills/odysseus/SKILL.md index d4cbdf726..a939cac33 100644 --- a/integrations/codex/skills/odysseus/SKILL.md +++ b/integrations/codex/skills/odysseus/SKILL.md @@ -100,6 +100,14 @@ python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"tex - `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. - `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. +## Chat history + +- `GET /api/codex/chat/{session_id}?offset=0&limit=100` — read a paginated conversation transcript owned by the token owner. Requires the existing `chat` scope. + +```bash +python3 integrations/codex/scripts/odysseus_api.py GET /api/codex/chat/SESSION_ID +``` + ## Email draft + send - Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. diff --git a/launch-windows.ps1 b/launch-windows.ps1 index ab0e3542b..31c45a5fb 100644 --- a/launch-windows.ps1 +++ b/launch-windows.ps1 @@ -131,7 +131,8 @@ if (-not (Test-Path $venvPy)) { # 3. Install / update dependencies Write-Step "Installing dependencies (first run can take a few minutes)" -& $venvPy -m pip install --upgrade pip --quiet +& $venvPy -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS dependency bootstrap failed." } & $venvPy -m pip install -r requirements.txt if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed. Scroll up for the pip error." } diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt new file mode 100644 index 000000000..da1d62220 --- /dev/null +++ b/requirements-bootstrap.txt @@ -0,0 +1,8 @@ +# Integrity-pinned bootstrap for the October 2026 eMudhra CA incident. +# Install with --require-hashes --only-binary=:all: --no-deps before other pip work. +# pip vendors its own certifi; updating the application bundle cannot update it. +# Direct universal wheels avoid index resolution through the old trust store. +pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl \ + --hash=sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70 +certifi @ https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 diff --git a/requirements.txt b/requirements.txt index 7fabaae93..415b4b503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,13 +3,16 @@ uvicorn python-multipart python-dotenv httpx +# HTTPS clients using certifi must distrust the compromised eMudhra CA paths. +certifi>=2026.10.23 httpcore>=1.0,<2.0 pydantic>=2.13.4 pydantic-settings>=2.14.1 SQLAlchemy pypdf pypdfium2 -Pillow +# Pillow 13 fixes known image-processing advisories; keep local embeddings compatible. +Pillow>=13.0.0 faster-whisper pdfplumber beautifulsoup4 @@ -21,7 +24,9 @@ numpy # chromadb-client is the lightweight HTTP client (talks to a standalone # ChromaDB service); fastembed runs local ONNX embeddings. chromadb-client -fastembed +# gRPC vendors independent roots; 1.84.1 removes the incident trust anchors. +grpcio>=1.84.1 +fastembed>=0.9.1 youtube-transcript-api yt-dlp # Markdown rendering for research reports (src/visual_report.py). diff --git a/routes/codex_routes.py b/routes/codex_routes.py index f42c6b632..542e44d49 100644 --- a/routes/codex_routes.py +++ b/routes/codex_routes.py @@ -35,6 +35,7 @@ CALENDAR_READ_SCOPES = {"calendar:read", "calendar:write"} CALENDAR_WRITE_SCOPES = {"calendar:write"} DOCS_READ_SCOPES = {"documents:read", "documents:write"} DOCS_WRITE_SCOPES = {"documents:write"} +CHAT_READ_SCOPES = {"chat"} WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"} @@ -161,6 +162,7 @@ def setup_codex_routes( memory_router: APIRouter | None = None, calendar_router: APIRouter | None = None, document_router: APIRouter | None = None, + history_router: APIRouter | None = None, ) -> APIRouter: router = APIRouter(prefix="/api/codex", tags=["codex"]) email_list_endpoint = _find_endpoint(email_router, "GET", "/api/email/list") @@ -174,6 +176,7 @@ def setup_codex_routes( documents_library_endpoint = _find_endpoint(document_router, "GET", "/api/documents/library") documents_get_endpoint = _find_endpoint(document_router, "GET", "/api/document/{doc_id}") documents_create_endpoint = _find_endpoint(document_router, "POST", "/api/document") + chat_history_endpoint = _find_endpoint(history_router, "GET", "/api/history/{session_id}") @router.get("/capabilities") def capabilities(request: Request): @@ -219,6 +222,11 @@ def setup_codex_routes( "launch": scoped(COOKBOOK_LAUNCH_SCOPES), "actions": ["tasks", "servers", "output", "serve", "stop"], }, + "chat": { + "read": scoped(CHAT_READ_SCOPES), + "actions": ["read"], + "available": chat_history_endpoint is not None, + }, }, "safety": { "email_send_requires_confirmation": True, @@ -226,6 +234,33 @@ def setup_codex_routes( }, } + @router.get("/chat/{session_id}") + async def read_chat( + request: Request, + session_id: str, + offset: int = 0, + limit: int = 100, + ): + _scope_owner(request, CHAT_READ_SCOPES) + if chat_history_endpoint is None: + raise HTTPException(503, "chat history unavailable") + offset, limit = _clamp_pagination( + offset, + limit, + default_limit=100, + max_limit=100, + ) + result = await chat_history_endpoint(request, session_id, limit, offset) + return { + "session_id": session_id, + "name": result["name"], + "model": result["model"], + "messages": result["history"], + "total": result["total"], + "offset": result["offset"], + "limit": result["limit"], + } + @router.get("/plugin.zip") def plugin_zip(request: Request): require_authenticated_request(request) diff --git a/routes/editor_agent_routes.py b/routes/editor_agent_routes.py index 3fe3b7901..410bab501 100644 --- a/routes/editor_agent_routes.py +++ b/routes/editor_agent_routes.py @@ -1732,6 +1732,7 @@ def setup_editor_agent_routes() -> APIRouter: from core.database import SessionLocal, ModelEndpoint from src.auth_helpers import owner_filter from src.endpoint_resolver import _endpoint_enabled_models + from src.image_model_ids import looks_like_image_generation_model db = SessionLocal() try: @@ -1741,10 +1742,16 @@ def setup_editor_agent_routes() -> APIRouter: owner, ) for ep in query.all(): - if getattr(ep, "model_type", None) != "image": - continue + # Image-only endpoints (the local Qwen server) list their + # models verbatim. Gateway/chat endpoints (OpenRouter, …) + # mix chat and image models, so expose only ids that are + # recognisably image-capable — the same test the request + # router uses. + image_endpoint = getattr(ep, "model_type", None) == "image" provider = getattr(ep, "name", None) or getattr(ep, "base_url", "") or "image" for name in _endpoint_enabled_models(ep): + if not image_endpoint and not looks_like_image_generation_model(name): + continue key = (provider, name) if key in seen: continue diff --git a/routes/email/email_helpers.py b/routes/email/email_helpers.py index 1808af13b..d697b2fe7 100644 --- a/routes/email/email_helpers.py +++ b/routes/email/email_helpers.py @@ -17,6 +17,8 @@ import base64 import time import imaplib import smtplib +import socket +import ssl import email as email_mod import email.header import email.utils @@ -36,6 +38,7 @@ from typing import Optional, List from src.auth_helpers import _auth_disabled, get_current_user from src.secret_storage import decrypt as _decrypt +from src.url_safety import OutboundAddressBlocked, connect_outbound_tcp logger = logging.getLogger(__name__) @@ -160,6 +163,94 @@ def _smtp_security_mode(cfg: dict) -> str: return "ssl" +# Raised when a mail host resolves into a denied range (see _mail_private_blocked). +MailServerAddressBlocked = OutboundAddressBlocked + + +def _mail_private_blocked(owner: str | None) -> bool: + """Whether private/loopback/shared mail destinations are denied for *owner*. + + Link-local (cloud metadata), multicast, reserved and unspecified addresses + are always denied (src/url_safety.py). Private ranges are where a mail + tester or saved account becomes an internal-network probe, so they are + only allowed for an admin or in single-user mode (AUTH_ENABLED=false), the + same principals trusted with host-level tools. Operators of multi-user + deployments with a LAN mail server opt in with EMAIL_ALLOW_PRIVATE_IPS=true; + EMAIL_BLOCK_PRIVATE_IPS=true denies private ranges for everyone. + """ + if os.getenv("EMAIL_BLOCK_PRIVATE_IPS", "").strip().lower() == "true": + return True + if os.getenv("EMAIL_ALLOW_PRIVATE_IPS", "").strip().lower() == "true": + return False + from src.tool_security import owner_is_admin_or_single_user + + return not owner_is_admin_or_single_user((owner or "").strip() or None) + + +def _mail_socket_timeout(timeout): + return socket._GLOBAL_DEFAULT_TIMEOUT if timeout is None else timeout + + +class _PolicyIMAP4(imaplib.IMAP4): + """IMAP4 whose socket comes from connect_outbound_tcp. + + The mail host is resolved once and connected to only at the addresses the + address policy approved, so DNS rebinding between check and connect is + impossible. STARTTLS still verifies against ``self.host``. + """ + + def __init__(self, host, port, *, block_private: bool, timeout=None): + self._block_private = block_private + super().__init__(host, port, timeout=timeout) + + def _create_socket(self, timeout): + if timeout is not None and not timeout: + raise ValueError("Non-blocking socket (timeout=0) is not supported") + return connect_outbound_tcp( + self.host, self.port, timeout=_mail_socket_timeout(timeout), block_private=self._block_private, + ) + + +class _PolicyIMAP4_SSL(imaplib.IMAP4_SSL): + """IMAP4_SSL over a policy-checked socket; TLS SNI/verification use the hostname.""" + + def __init__(self, host, port, *, block_private: bool, timeout=None, ssl_context=None): + self._block_private = block_private + super().__init__(host, port, ssl_context=ssl_context, timeout=timeout) + + def _create_socket(self, timeout): + sock = _PolicyIMAP4._create_socket(self, timeout) + return self.ssl_context.wrap_socket(sock, server_hostname=self.host) + + +class _PolicySMTP(smtplib.SMTP): + """SMTP whose socket comes from connect_outbound_tcp (see _PolicyIMAP4).""" + + def __init__(self, host="", port=0, *, block_private: bool, **kwargs): + self._block_private = block_private + super().__init__(host, port, **kwargs) + + def _get_socket(self, host, port, timeout): + if timeout is not None and not timeout: + raise ValueError("Non-blocking socket (timeout=0) is not supported") + return connect_outbound_tcp( + host, port, timeout=timeout, block_private=self._block_private, + source_address=self.source_address, + ) + + +class _PolicySMTP_SSL(smtplib.SMTP_SSL): + """SMTP_SSL over a policy-checked socket; TLS SNI/verification use the hostname.""" + + def __init__(self, host="", port=0, *, block_private: bool, **kwargs): + self._block_private = block_private + super().__init__(host, port, **kwargs) + + def _get_socket(self, host, port, timeout): + sock = _PolicySMTP._get_socket(self, host, port, timeout) + return self.context.wrap_socket(sock, server_hostname=self._host) + + def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message: str | bytes, timeout: int = 30) -> None: """Send through SMTP using the configured transport security mode.""" host = cfg["smtp_host"] @@ -178,14 +269,15 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message smtp.login(user, password) security = _smtp_security_mode(cfg) + block_private = _mail_private_blocked(cfg.get("owner")) if security == "ssl": - with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp: + with _PolicySMTP_SSL(host, port, timeout=timeout, block_private=block_private) as smtp: _auth_smtp(smtp) smtp.sendmail(from_addr, recipients, message) return - with smtplib.SMTP(host, port, timeout=timeout) as smtp: + with _PolicySMTP(host, port, timeout=timeout, block_private=block_private) as smtp: if security == "starttls": smtp.starttls() _auth_smtp(smtp) @@ -224,6 +316,39 @@ def _friendly_email_auth_error(protocol: str, host: str, error: object) -> str: return raw[:200] +# Bound at import: callers (and tests) may swap out imaplib.IMAP4 itself. +_IMAP_ABORT = imaplib.IMAP4.abort + + +def _mail_connection_test_error(protocol: str, host: str, error: BaseException) -> str: + """User-facing result for a failed IMAP/SMTP connection test. + + Transport failures are reported by category, never with the peer's own + bytes: a non-mail service answering on the chosen host/port would + otherwise have its banner echoed back (an internal-service fingerprinting + oracle). Errors from a server that does speak the protocol (login/auth + rejections) keep their text via _friendly_email_auth_error, since that is + what the user needs to fix their settings. + """ + if isinstance(error, MailServerAddressBlocked): + return f"{protocol} server address is not allowed by this server's network policy" + if isinstance(error, (_IMAP_ABORT, smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected)): + return f"{protocol} server did not respond like an {protocol} server" + if isinstance(error, ssl.SSLCertVerificationError): + return f"{protocol} TLS certificate verification failed" + if isinstance(error, ssl.SSLError): + return f"{protocol} TLS handshake failed; check the port and security setting" + if isinstance(error, (socket.timeout, TimeoutError)): + return f"{protocol} connection timed out" + if isinstance(error, ConnectionRefusedError): + return f"{protocol} connection refused" + if isinstance(error, socket.gaierror): + return f"{protocol} server name could not be resolved" + if isinstance(error, OSError) and not isinstance(error, smtplib.SMTPException): + return f"{protocol} connection failed" + return _friendly_email_auth_error(protocol, host, error) + + def _strip_think(text: str) -> str: """Email-flavored think strip — thin wrapper over the central helper. @@ -685,18 +810,21 @@ def _ensure_sender_signatures_table(conn): _lg.getLogger(__name__).warning(f"sender_signatures owner-migration skipped: {_mig_e}") -def attachment_extract_dir(folder: str, uid: str) -> Path: - """Containment-safe extraction directory for an attachment. +def attachment_extract_dir(folder: str, uid: str, *, owner: str, account_id: str | None) -> Path: + """Containment-safe extraction directory for one message's attachments. - `folder` and `uid` are user-controlled (query/path params). Flatten them to - a single safe path segment so a value like folder='../../tmp' can't escape - ATTACHMENTS_DIR, then assert containment as belt-and-suspenders.""" - key = re.sub(r"[^A-Za-z0-9._-]", "_", f"{folder}_{uid}") or "_" - target = (ATTACHMENTS_DIR / key).resolve() - base = ATTACHMENTS_DIR.resolve() - if target != base and base not in target.parents: + IMAP UIDs are small per-mailbox counters, so `folder_uid` alone put every + user's and every account's "INBOX 42" in one directory, where extractions + overwrote each other by filename (a path handed to one user's agent could + then hold another user's attachment). See src.mail_attachment_paths: the + directory is keyed by (owner, account, folder, uid) and contained in + ATTACHMENTS_DIR whatever `folder`/`uid` hold.""" + from src.mail_attachment_paths import attachment_scope_dir + + try: + return attachment_scope_dir(ATTACHMENTS_DIR, folder, uid, owner=owner, account_id=account_id) + except ValueError: raise HTTPException(400, "Invalid attachment location") - return target def _init_scheduled_db(): @@ -1077,6 +1205,8 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict: cfg = { "account_id": row.id, "account_name": row.name, + # Principal for the outbound mail address policy. + "owner": row.owner or owner or "", "smtp_host": row.smtp_host or "", "smtp_port": int(row.smtp_port or 465), "smtp_security": _smtp_security_mode({"smtp_security": getattr(row, "smtp_security", ""), "smtp_port": row.smtp_port}), @@ -1110,6 +1240,7 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict: cfg = { "account_id": resolved_id, "account_name": "legacy", + "owner": owner or "", "smtp_host": settings.get("smtp_host", os.environ.get("SMTP_HOST", "")), "smtp_port": int(settings.get("smtp_port", os.environ.get("SMTP_PORT", "465")) or 465), "smtp_security": _smtp_security_mode({ @@ -1173,11 +1304,16 @@ def _open_imap_connection( starttls: bool, timeout: int = _IMAP_TIMEOUT_SECONDS, ssl_context=None, + owner: str | None = None, ): - """Open an IMAP connection using the configured security mode.""" + """Open an IMAP connection using the configured security mode. + + `owner` is the principal the destination is judged for + (see _mail_private_blocked).""" port = int(port or 993) + block_private = _mail_private_blocked(owner) if starttls: - conn = imaplib.IMAP4(host, port, timeout=timeout) + conn = _PolicyIMAP4(host, port, timeout=timeout, block_private=block_private) try: if ssl_context: conn.starttls(ssl_context=ssl_context) @@ -1193,9 +1329,9 @@ def _open_imap_connection( raise elif port == 993: kwargs = {"ssl_context": ssl_context} if ssl_context else {} - conn = imaplib.IMAP4_SSL(host, port, timeout=timeout, **kwargs) + conn = _PolicyIMAP4_SSL(host, port, timeout=timeout, block_private=block_private, **kwargs) else: - conn = imaplib.IMAP4(host, port, timeout=timeout) + conn = _PolicyIMAP4(host, port, timeout=timeout, block_private=block_private) try: conn.sock.settimeout(timeout) except Exception: @@ -1235,6 +1371,7 @@ def _imap_connect(account_id: str | None = None, owner: str = "", cfg["imap_port"], starttls=bool(cfg.get("imap_starttls")), timeout=timeout, + owner=cfg.get("owner"), ) try: if cfg.get("oauth_provider") == "google": diff --git a/routes/email/email_routes.py b/routes/email/email_routes.py index 12d2dd82a..294585e79 100644 --- a/routes/email/email_routes.py +++ b/routes/email/email_routes.py @@ -25,17 +25,21 @@ import json import re import html import io +import tempfile import zipfile -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse from html.parser import HTMLParser as _HTMLParser import logging import uuid -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path +from email.message import EmailMessage +from email import policy as email_policy from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +from starlette.concurrency import run_in_threadpool from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request from fastapi.responses import FileResponse, StreamingResponse from src.constants import DATA_DIR @@ -61,6 +65,7 @@ from .email_helpers import ( _fetch_sender_thread_context, _pre_retrieve_context, _EMAIL_REPLY_SYS_PROMPT_BASE, _POOL_HOOKS, _friendly_email_auth_error, _email_summary_failure_log_detail, + _mail_connection_test_error, _mail_private_blocked, _PolicySMTP, _PolicySMTP_SSL, _generate_email_summary, EMAIL_SUMMARY_ERROR_CODE, EMAIL_SUMMARY_ERROR_MESSAGE, SendEmailRequest, ExtractStyleRequest, ATTACHMENTS_DIR, COMPOSE_UPLOADS_DIR, SCHEDULED_DB, @@ -83,6 +88,17 @@ _SERVER_OWNED_OAUTH_FIELDS = { _CACHE_HOOKS: dict = {"invalidate_list": None} +def _check_mail_host(host, owner: str, label: str) -> str | None: + """Reject unsafe or unresolved mail hosts before saving or testing them.""" + from src.url_safety import check_outbound_host + ok, reason = check_outbound_host(host, block_private=_mail_private_blocked(owner)) + if ok: + return None + logger.warning("Rejected %s mail host for owner=%r: %s", label, owner, reason) + # Resolver and exception details belong in server logs, never the response. + return f"Rejected {label} host: server address is not allowed or does not resolve" + + def _normalized_mail_host(value) -> str: """Normalize a mail hostname for exact provider-bound comparisons.""" return str(value or "").strip().lower().rstrip(".") @@ -635,10 +651,15 @@ def _message_id_chain(*values: str) -> list[str]: def _uid_from_fetch_meta(meta_b: bytes) -> str: - m = re.search(rb"\bUID\s+(\d+)\b", meta_b) + m = re.search(rb"\bUID\s+(\d+)\b", meta_b, re.IGNORECASE) return m.group(1).decode() if m else "" +def _size_from_fetch_meta(meta_b: bytes) -> int | None: + m = re.search(rb"\bRFC822\.SIZE\s+(\d+)\b", meta_b, re.IGNORECASE) + return int(m.group(1)) if m else None + + def _parse_list_unsubscribe_header(value: str | None) -> list[dict]: """Parse RFC List-Unsubscribe entries into safe reviewable actions. @@ -1542,6 +1563,44 @@ def _envelope_recipients(*fields: str) -> list: return out +_MD_LINK_URL_RE = re.compile(r"\((https?://)[^)\s]*") + + +def _md_links_to_html(s: str) -> str: + """Linear ``re.sub(r"\\[([^\\]]+)\\]\\((https?://[^)\\s]+)\\)", '\\1', s)``. + + Every `[` before the next `]` shares that `]`, so only the first can match; + and when a URL runs into whitespace/end without `)`, every later `[` whose + `]` also lies before that stop fails the same way. Skipping those spans + keeps a `[[[` or `[a](http://x[a](http://x` flood O(n) instead of the + regex's O(n^2). + """ + out: list[str] = [] + emitted = scan = 0 + while True: + open_at = s.find("[", scan) + if open_at < 0: + break + close_at = s.find("]", open_at + 1) + if close_at < 0: + break + scan = close_at + 1 + if close_at == open_at + 1: + continue + url = _MD_LINK_URL_RE.match(s, close_at + 1) + if url is None: + continue + end = url.end() + if end < len(s) and s[end] == ")" and end > url.end(1): + out.append(s[emitted:open_at]) + out.append(f'{s[open_at + 1:close_at]}') + emitted = scan = end + 1 + elif end > url.end(1): + scan = max(scan, s.rfind("]", scan, end) + 1) + out.append(s[emitted:]) + return "".join(out) + + def _md_to_email_html(text: str) -> str: """Render the compose markdown body to a SAFE HTML fragment for the email's text/html part. Everything is HTML-escaped FIRST (so a pasted ', + '' + ].join('\n'); + } else if (uid === 'long_msg') { + body_html = `

${'A'.repeat(5000)}

`; + } else { + body_html = `

Standard message body for ${uid}

`; + } + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + uid: uid, + subject: `Subject ${uid}`, + from_name: 'Sender', + from_address: 'sender@example.com', + to: 'user@example.com', + date: '2026-10-20', + date_display: 'Oct 20, 2026', + body: `Plain text for ${uid}`, + body_html: body_html, + })); + return; + } + + let filePath = path.join(ROOT, url.pathname); + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + const ext = path.extname(filePath); + const mime = ext === '.js' ? 'application/javascript' : ext === '.html' ? 'text/html' : ext === '.css' ? 'text/css' : 'text/plain'; + res.writeHead(200, { 'Content-Type': mime }); + res.end(fs.readFileSync(filePath)); + return; + } + res.writeHead(404); + res.end('Not found'); + }); + + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const port = server.address().port; + const origin = `http://127.0.0.1:${port}`; + + const launchOpts = { headless: true }; + const chromePath = findChromiumPath(); + if (chromePath) launchOpts.executablePath = chromePath; + + let browser; + try { + browser = await chromium.launch(launchOpts); + } catch (launchErr) { + throw new Error( + `Failed to launch Chromium browser: ${launchErr.message}\n` + + 'Missing or unexecutable Chromium binary.\n' + + 'To install Playwright Chromium, run:\n' + + ' npx playwright install chromium\n' + + 'Or set PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH to a valid Chromium executable.' + ); + } + const context = await browser.newContext(); + const page = await context.newPage(); + + const results = {}; + + try { + await page.goto(`${origin}/static/index.html`); + await page.evaluate(() => { + if (!document.getElementById('email-lib-bulk-actions')) { + const d = document.createElement('div'); d.id = 'email-lib-bulk-actions'; document.body.appendChild(d); + } + if (!document.getElementById('email-lib-selected-count')) { + const d = document.createElement('div'); d.id = 'email-lib-selected-count'; document.body.appendChild(d); + } + }); + + // TEST 1: Full Print Image Sanitization and Replacement + const printImageResult = await page.evaluate(async (base) => { + window.API_BASE = base; + const mod = await import(`${base}/static/js/emailLibrary/index.js`); + const stateMod = await import(`${base}/static/js/emailLibrary/state.js`); + + stateMod.state._selectedUids = new Set(['img_test']); + stateMod.state._libEmails = [{ uid: 'img_test', subject: 'Image Test', folder: 'INBOX' }]; + + let iframePrintInvoked = false; + let printDocHtml = ''; + let afterPrintTriggered = false; + let hasPlaceholderElement = false; + + const observer = new MutationObserver(() => { + const frame = document.getElementById('email-browser-print-frame'); + if (frame && frame.contentWindow && !frame._patched) { + frame._patched = true; + frame.contentWindow.print = function() { + iframePrintInvoked = true; + printDocHtml = frame.contentDocument ? frame.contentDocument.documentElement.outerHTML : ''; + const body = frame.contentDocument ? frame.contentDocument.body : null; + hasPlaceholderElement = !!body?.querySelector('.email-inline-image-placeholder'); + frame.contentWindow.dispatchEvent(new Event('afterprint')); + afterPrintTriggered = true; + }; + } + }); + observer.observe(document.body, { childList: true }); + + await mod._exportSelectedEmailPdf(); + observer.disconnect(); + + const frameAfter = document.getElementById('email-browser-print-frame'); + + return { + iframePrintInvoked, + afterPrintTriggered, + frameRemovedAfterPrint: frameAfter === null, + hasLogoNotice: printDocHtml.includes('[Image: Company Brand Logo]'), + hasTrackerNotice: printDocHtml.includes('[Image: Tracking Pixel]'), + hasNoAltNotice: printDocHtml.includes('[Image omitted]'), + hasPlaceholderElement, + hasRemoteImageBlockedText: printDocHtml.includes('Remote image blocked'), + hasDownloadButton: printDocHtml.includes('email-inline-image-download-btn'), + hasInteractiveActionBtn: printDocHtml.includes('btn-action'), + hasXSS: printDocHtml.includes('