diff --git a/.github/scripts/validate-searxng-image.sh b/.github/scripts/validate-searxng-image.sh new file mode 100644 index 000000000..3edd5ba16 --- /dev/null +++ b/.github/scripts/validate-searxng-image.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Exercise an already-built image through the normal Compose startup path. +set -euo pipefail + +searxng_repo=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +searxng_image=${1:?usage: validate-searxng-image.sh IMAGE} +[[ "$searxng_image" =~ ^[a-zA-Z0-9_./:@+-]+$ ]] || { + echo "Invalid image reference" >&2 + exit 2 +} +searxng_checks=$(mktemp -d "${TMPDIR:-/tmp}/searxng-ci-XXXXXXXX") +searxng_project=$(basename "$searxng_checks" | tr '[:upper:]' '[:lower:]') +export SEARXNG_VALIDATION_IMAGE="$searxng_image" +export SEARXNG_VALIDATION_CHECKS="$searxng_checks" +export SEARXNG_VALIDATION_VOLUME="${searxng_project}-settings" +# Never forward an operator's existing secret or load their .env for a CI probe. +export SEARXNG_SECRET=searxng-ci-validation-only +searxng_volume_created=0 +searxng_started=0 +searxng_compose=(docker compose --env-file /dev/null --project-name "$searxng_project" + -f "$searxng_repo/docker-compose.yml" + -f "$searxng_repo/.github/searxng-validation.compose.yml") + +searxng_cleanup() { + local searxng_exit=$? + if [ "$searxng_started" = 1 ]; then + "${searxng_compose[@]}" down --remove-orphans >/dev/null 2>&1 || true + fi + if [ "$searxng_volume_created" = 1 ]; then + docker volume rm "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1 || true + fi + rm -rf -- "$searxng_checks" + return "$searxng_exit" +} +trap searxng_cleanup EXIT + +install -m 0644 "$searxng_repo/scripts/check_ca_trust.py" \ + "$searxng_repo/docker/searxng/validate_image.py" "$searxng_checks/" +install -m 0644 "$searxng_repo/tests/fixtures/ca-trust/incident.json" "$searxng_checks/incident.json" +install -m 0644 "$searxng_repo/config/searxng/settings.yml" "$searxng_checks/settings.yml" +install -m 0644 "$searxng_repo/scripts/migrate_searxng_settings.py" "$searxng_checks/migrate_searxng_settings.py" +if docker volume inspect "$SEARXNG_VALIDATION_VOLUME" >/dev/null 2>&1; then + echo "Refusing to reuse an existing validation volume" >&2 + exit 1 +fi +docker volume create --label "odysseus.searxng-validation=$searxng_project" \ + "$SEARXNG_VALIDATION_VOLUME" >/dev/null +searxng_volume_created=1 +searxng_started=1 +if ! "${searxng_compose[@]}" up --detach --no-deps --no-build --pull never \ + --wait --wait-timeout 150 searxng; then + "${searxng_compose[@]}" logs --no-color searxng >&2 || true + exit 1 +fi +"${searxng_compose[@]}" exec -T searxng /usr/local/searxng/.venv/bin/python \ + /opt/odysseus-checks/validate_image.py --running --live diff --git a/.github/searxng-validation.compose.yml b/.github/searxng-validation.compose.yml new file mode 100644 index 000000000..d1e1c575c --- /dev/null +++ b/.github/searxng-validation.compose.yml @@ -0,0 +1,21 @@ +# CI-only isolation; retain the service's entrypoint, healthcheck and capabilities. +services: + searxng: + image: ${SEARXNG_VALIDATION_IMAGE:?set the image being validated} + ports: !reset [] + volumes: !override + - searxng-validation-data:/etc/searxng + - ${SEARXNG_VALIDATION_CHECKS:?}/settings.yml:/tmp/searxng-settings.yml.template:ro + - ${SEARXNG_VALIDATION_CHECKS:?}/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro + - ${SEARXNG_VALIDATION_CHECKS:?}:/opt/odysseus-checks:ro + networks: !override + - default + restart: "no" + +networks: + default: {} + +volumes: + searxng-validation-data: + external: true + name: ${SEARXNG_VALIDATION_VOLUME:?} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42f0651f0..229a4d599 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: "20" + node-version: "22.23.4" - run: npm ci --ignore-scripts --no-audit --no-fund - name: Check frontend architecture run: node scripts/check_frontend.mjs @@ -155,20 +155,35 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' with: python-version: "3.11" + # Establish system trust before dependency installers or their hooks run. + # Keep one apt pass for FFmpeg and bubblewrap; containment is checked later. + - name: Install FFmpeg and bubblewrap and validate system CA trust + if: steps.docs-check.outputs.docs_only != 'true' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap ca-certificates + python scripts/check_ca_trust.py --system + command -v ffmpeg + ffmpeg -version | head -n 1 + # uv resolves the same versions as pip but installs ~5x faster, and its # cache keeps each shard from re-downloading the whole requirement set. - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 if: steps.docs-check.outputs.docs_only != 'true' with: + version: "0.13.5" enable-cache: true cache-dependency-glob: requirements.txt prune-cache: false - - run: uv pip install --system -r requirements.txt + - run: | + uv pip install --system --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt + python scripts/check_ca_trust.py --python + uv pip install --system -r requirements.txt if: steps.docs-check.outputs.docs_only != 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 if: steps.docs-check.outputs.docs_only != 'true' with: - node-version: "20" + node-version: "22.23.4" cache: npm - run: npm ci if: steps.docs-check.outputs.docs_only != 'true' @@ -184,15 +199,9 @@ jobs: if: steps.docs-check.outputs.docs_only != 'true' - run: mkdir -p data # sqlite DB lives at ./data/app.db if: steps.docs-check.outputs.docs_only != 'true' - # One apt pass for FFmpeg (media integration tests) and bubblewrap - # (containment, verified in the next step). - - name: Install FFmpeg and bubblewrap + - name: Validate Python and system CA trust if: steps.docs-check.outputs.docs_only != 'true' - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends ffmpeg bubblewrap - command -v ffmpeg - ffmpeg -version | head -n 1 + run: python scripts/check_ca_trust.py --system --python - name: Establish functional bubblewrap containment if: steps.docs-check.outputs.docs_only != 'true' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 0e4f9a1f1..e78bba158 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -70,5 +70,9 @@ jobs: - name: Run pip-audit on requirements run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install pip-audit==2.10.0 pip-audit -r requirements.txt -r requirements-optional.txt --strict diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index da41c7140..6865d5120 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -29,7 +29,6 @@ permissions: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} jobs: build: @@ -52,6 +51,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Normalize image name + shell: bash + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - name: Set up Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Log in to GHCR @@ -65,6 +67,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . + pull: true platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.arch }} @@ -93,6 +96,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Normalize image name + shell: bash + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - name: Read APP_VERSION + short sha id: ver run: | diff --git a/.github/workflows/searxng-image.yml b/.github/workflows/searxng-image.yml new file mode 100644 index 000000000..844617b10 --- /dev/null +++ b/.github/workflows/searxng-image.yml @@ -0,0 +1,79 @@ +name: ci / SearXNG trust + +on: + pull_request: + paths: + - 'docker/searxng/**' + - 'scripts/check_ca_trust.py' + - 'tests/fixtures/ca-trust/**' + - 'tests/test_searxng*' + - 'docker-compose*.yml' + - 'config/searxng/**' + - 'scripts/migrate_searxng_settings.py' + - '.github/scripts/*searxng*' + - '.github/searxng-validation.compose.yml' + - '.github/workflows/searxng-image.yml' + push: + branches: [dev, main] + paths: + - 'docker/searxng/**' + - 'scripts/check_ca_trust.py' + - 'tests/fixtures/ca-trust/**' + - 'docker-compose*.yml' + - 'config/searxng/**' + - 'scripts/migrate_searxng_settings.py' + - '.github/scripts/*searxng*' + - '.github/searxng-validation.compose.yml' + - '.github/workflows/searxng-image.yml' + workflow_dispatch: + +concurrency: + group: searxng-image-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + validate: + name: SearXNG validate (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + platform: linux/amd64 + runner: ubuntu-latest + - arch: arm64 + platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Select the immutable upstream image from Compose + id: image + run: | + docker compose --env-file /dev/null -f docker-compose.yml config --format json | + python3 -c 'import json, re, sys; image = json.load(sys.stdin)["services"]["searxng"]["image"]; re.fullmatch(r"docker\.io/searxng/searxng:[0-9]{4}\.[^@\s]+@sha256:[0-9a-f]{64}", image) or sys.exit("Invalid SearXNG pin"); print("image=" + image)' >> "$GITHUB_OUTPUT" + - name: Pull the upstream image without registry credentials + env: + SEARXNG_TEST_IMAGE: ${{ steps.image.outputs.image }} + run: docker pull "$SEARXNG_TEST_IMAGE" + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - name: Check unmodified upstream trust and actual entrypoint offline + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: docker/searxng/Dockerfile + target: validation + build-args: SEARXNG_IMAGE=${{ steps.image.outputs.image }} + network: none + platforms: ${{ matrix.platform }} + no-cache-filters: validation + - name: Validate the selected upstream runtime through Compose and live HTTPS + env: + SEARXNG_TEST_IMAGE: ${{ steps.image.outputs.image }} + run: bash .github/scripts/validate-searxng-image.sh "$SEARXNG_TEST_IMAGE" diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index b00cd03a4..18a197eb0 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -76,5 +76,9 @@ jobs: - name: Run zizmor run: | set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ca-certificates + python scripts/check_ca_trust.py --system + python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt pip install zizmor==1.25.2 zizmor --offline --min-severity=low .github/workflows/ diff --git a/.gitignore b/.gitignore index 1ae2878b5..d51361f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ services/data/ # Test capture artifacts (browser session dumps may contain personal content) .playwright-mcp/ +test-results/ +playwright-report/ # OS .DS_Store diff --git a/.gitleaks.toml b/.gitleaks.toml index 0eec3f1ae..8543c9666 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -35,3 +35,15 @@ regexTarget = "secret" regexes = [ '''^7abc5b334a6c3c52dd3eba15d8bc516da728a67f14fb7f56d0bb5992d89a16ca$''', ] + +[[allowlists]] +description = "Synthetic dummy API key in trace artifact sanitizer test fixture" +targetRules = ["generic-api-key"] +condition = "AND" +paths = [ + '''(?:^|/)tests/test_trace_artifact_sanitizer\.py$''', +] +regexTarget = "secret" +regexes = [ + '''^sk-123456789$''', +] diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 38e184764..294289b7f 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -48,8 +48,8 @@ just composed. | Service | Image | Purpose | License | |---|---|---|---| | [SearXNG](https://github.com/searxng/searxng) | `searxng/searxng:2026.9.25-12f8b6515` (pinned tag; see compose) | Default metasearch backend | AGPL-3.0 | -| [ChromaDB](https://github.com/chroma-core/chroma) | `chromadb/chroma:latest` | Vector store for memory / RAG | Apache-2.0 | -| [ntfy](https://github.com/binwiederhier/ntfy) | `binwiederhier/ntfy` | Push notifications (self-hosted reminders) | Apache-2.0 / GPL-2.0 | +| [ChromaDB](https://github.com/chroma-core/chroma) | `chromadb/chroma:1.5.9` (pinned digest; see compose) | Vector store for memory / RAG | Apache-2.0 | +| [ntfy](https://github.com/binwiederhier/ntfy) | `binwiederhier/ntfy:v2.32.0` (pinned digest; see compose) | Push notifications (self-hosted reminders) | Apache-2.0 / GPL-2.0 | ## Bundled front-end libraries diff --git a/Dockerfile b/Dockerfile index d6e7ca48f..fe6a6792f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,33 @@ +# Both stages need the fixed native decoder, including curl in the builder and +# distro Node in the runtime. A cached Python base may predate the Debian fix. +FROM python:3.14.9-slim AS brotli-base +COPY scripts/check_brotli.py /usr/local/bin/check-brotli.py +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +# Check inherited trust before the first HTTPS package request, retaining Debian's +# signed archive verification. Refresh only the explicit native dependency. +RUN python /usr/local/bin/check-ca-trust.py --system \ + && sed -i 's|http://deb.debian.org/|https://deb.debian.org/|g' /etc/apt/sources.list.d/debian.sources \ + && apt-get update && apt-get install -y --no-install-recommends libbrotli1 \ + && python /usr/local/bin/check-brotli.py \ + && rm -rf /var/lib/apt/lists/* + # ---- builder: patch + build wheels for Real-ESRGAN's broken-on-3.14 deps ---- # basicsr/gfpgan/facexlib read their version via exec()+locals()['__version__'], # which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so # the final image / Cookbook never has to compile the broken sdists. See # docker/build-realesrgan-wheels.sh for the full rationale. -FROM python:3.14-slim AS realesrgan-wheels -RUN apt-get update && apt-get install -y --no-install-recommends curl \ +FROM brotli-base AS realesrgan-wheels +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels -FROM python:3.14-slim +FROM brotli-base # System deps. tmux is required by Cookbook for background downloads/serves. # openssh-client is required for Cookbook remote server tests, setup, probes, @@ -28,6 +46,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ cmake \ curl \ + ca-certificates \ git \ nodejs \ npm \ @@ -46,8 +65,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libglib2.0-0t64 \ libxcb1 \ libmagic1 \ + && python /usr/local/bin/check-brotli.py --node /usr/bin/node \ && rm -rf /var/lib/apt/lists/* +# Refresh both Python trust bundles before installing application dependencies. +# System roots also serve web fetches, webhooks, curl, and Node's OpenSSL mode. +COPY scripts/check_ca_trust.py /usr/local/bin/check-ca-trust.py +RUN python /usr/local/bin/check-ca-trust.py --system +COPY requirements-bootstrap.txt /tmp/requirements-bootstrap.txt +RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r /tmp/requirements-bootstrap.txt \ + && python /usr/local/bin/check-ca-trust.py --python +ENV NODE_OPTIONS=--use-openssl-ca \ + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/etc/ssl/certs/ca-certificates.crt + +# Chromium has a separate root store. Distrust the compromised intermediate +# public keys explicitly, including cross-signs, without waiting for CRL updates. +COPY docker/chromium-emudhra.json /etc/chromium/policies/managed/odysseus-emudhra.json + # Private browser automation wrapper used by the native `private_browser` tool. # Chromium is installed above, so agent-browser can drive the existing browser # binary without paying `npx` startup/install overhead on each tool call. @@ -72,7 +106,8 @@ RUN npm install -g agent-browser@0.35.0 --omit=dev --loglevel=error # /var/run/docker.sock mount). The Debian `docker.io` package ships # dockerd but not the client binary on slim, so grab the static client # tarball from download.docker.com instead. -ARG DOCKER_CLI_VERSION=29.6.2 +# 29.9.2 includes Go 1.26.10's eMudhra intermediate-key distrust. +ARG DOCKER_CLI_VERSION=29.9.2 RUN ARCH="$(dpkg --print-architecture)" \ && case "$ARCH" in \ amd64) DARCH=x86_64 ;; \ @@ -108,6 +143,12 @@ COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/ RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \ && rm -rf /tmp/odysseus-wheels +# Check gRPC's effective defaults after all Python dependencies are installed. +# The application minimum fixes its bundled roots; the checked override also +# keeps the effective Docker selection consistent with system trust. +RUN python /usr/local/bin/check-ca-trust.py --system --python --grpc \ + && python /usr/local/bin/check-brotli.py --node /usr/bin/node + # Copy app code COPY . . # Require the redistribution notices in the image build context. diff --git a/app.py b/app.py index 60441baf1..ae859a579 100644 --- a/app.py +++ b/app.py @@ -178,6 +178,14 @@ app.add_middleware(SecurityHeadersMiddleware) import asyncio as _asyncio from starlette.middleware.base import BaseHTTPMiddleware as _BaseHTTPMiddleware from starlette.responses import JSONResponse as _JSONResponse +import anyio as _anyio +from starlette.requests import ClientDisconnect as _ClientDisconnect + +# SSE endpoints (e.g. /api/editor/bridge/events) end/abort mid-stream when the +# browser disconnects; Starlette's BaseHTTPMiddleware surfaces that as +# anyio.EndOfStream (and sometimes ClientDisconnect). Those are client-side +# events, not server errors, so we map them to a 499 instead of a 500 traceback. +_STREAM_CLIENT_EXC = (_anyio.EndOfStream, _ClientDisconnect) REQUEST_HARD_TIMEOUT = float(os.getenv("REQUEST_HARD_TIMEOUT", "45")) _TIMEOUT_EXEMPT_PREFIXES = ( @@ -190,6 +198,7 @@ _TIMEOUT_EXEMPT_PREFIXES = ( "/api/cookbook/setup", # remote pacman/apt installs "/api/upload", # large files "/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout + "/api/editor", # editor agent/bridge: generative steps can take 30-110s (SAM + local diffusion) "/api/memory/audit", # retains own 120s LLM inactivity timeout ) @@ -225,15 +234,43 @@ class _InteractiveActivityMiddleware(_BaseHTTPMiddleware): return await call_next(request) +# Heavy image ops that can run 30-150 s; a deploy restart must not cut these off. +_HEAVY_IMAGE_OP_PREFIXES = ( + "/api/image/inpaint", + "/api/image/replace-background", + "/api/image/outpaint", + "/api/image/extend", + "/api/image/remove-bg", + "/api/image/mask", + "/api/image/upscale-local", + "/api/image/denoise", + "/api/image/enhance-face", + "/api/gallery/ai-upscale", + "/api/gallery/style-transfer", + "/api/editor/agent", +) + + class _SlowRequestLogMiddleware(_BaseHTTPMiddleware): async def dispatch(self, request, call_next): + from src import activity + path = request.url.path or "" + heavy = any(path.startswith(p) for p in _HEAVY_IMAGE_OP_PREFIXES) + if heavy: + activity.begin_image_op(path) start = time.perf_counter() status = 500 try: response = await call_next(request) status = getattr(response, "status_code", 0) or 0 return response + except _STREAM_CLIENT_EXC: + # Client disconnected / SSE stream ended — not a server error. + status = 499 + return _JSONResponse({"detail": "client disconnected"}, status_code=499) finally: + if heavy: + activity.end_image_op(path) elapsed = time.perf_counter() - start try: threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75") @@ -755,7 +792,8 @@ app.include_router(setup_research_routes(research_handler, session_manager=sessi # History from routes.history.history_routes import setup_history_routes -app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler)) +history_router = setup_history_routes(session_manager, upload_handler=upload_handler) +app.include_router(history_router) # Search from routes.search.search_routes import setup_search_routes @@ -821,6 +859,10 @@ app.include_router(setup_gallery_routes()) from routes.editor_draft_routes import setup_editor_draft_routes app.include_router(setup_editor_draft_routes()) +# Natural-language editor planning (planner wired to the configured chat model) +from routes.editor_agent_routes import setup_editor_agent_routes +app.include_router(setup_editor_agent_routes()) + # Scheduled tasks + event bus from src.task_scheduler import TaskScheduler task_scheduler = TaskScheduler(session_manager) @@ -915,6 +957,7 @@ app.include_router(setup_codex_routes( memory_router=memory_router, calendar_router=calendar_router, document_router=document_router, + history_router=history_router, )) app.include_router(setup_claude_routes()) @@ -998,6 +1041,16 @@ async def get_version(): "source_commit": APP_SOURCE_COMMIT, } + +@app.get("/api/busy") +async def get_busy(): + """Report in-flight heavy work so a deploy can wait instead of dropping it.""" + from src import activity + return { + "busy_image_ops": activity.heavy_image_ops_active(), + **activity.activity(), + } + @app.get("/api/health") async def health_check() -> Dict[str, str]: return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()} @@ -1436,4 +1489,35 @@ if __name__ == "__main__": bind_host = os.getenv("APP_BIND", "127.0.0.1") bind_port = int(os.getenv("APP_PORT", "7011")) - uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") + # Give in-flight requests time to finish on SIGTERM instead of being cut + # off mid-flight. Diffusion edits run 60-120 s; outpaint/extend can run up + # to ~330 s (provider cap 900 s), so default the graceful window to 900 s. + # systemd's TimeoutStopSec must exceed this (see the graceful-stop drop-in). + # + # We hook uvicorn's signal handler so a SIGTERM also flips src.shutdown: + # SSE / streaming generators check it and end promptly, so an open browser + # stream cannot stall a restart for the whole window. The long window stays + # a safety net for genuine in-flight image ops. + from src import shutdown as _shutdown + + config = uvicorn.Config( + app, + host=bind_host, + port=bind_port, + log_level="info", + timeout_graceful_shutdown=int(os.getenv("ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT", "900") or 900), + ) + server = uvicorn.Server(config) + + # uvicorn's capture_signals() wires SIGINT/SIGTERM to self.handle_exit, so + # wrapping that method lets us set the shutdown flag before the normal exit + # path runs (this uvicorn version has no install_signal_handlers kwarg). + _uvicorn_handle_exit = server.handle_exit + + def _handle_shutdown_signal(signum, frame): + _shutdown.request_shutdown() + _uvicorn_handle_exit(signum, frame) + + server.handle_exit = _handle_shutdown_signal + + server.run() diff --git a/build-windows-portable.ps1 b/build-windows-portable.ps1 index 7da912f73..634ae8068 100644 --- a/build-windows-portable.ps1 +++ b/build-windows-portable.ps1 @@ -47,7 +47,8 @@ if (-not $pyExe) { Write-Host ("Using Python: " + $pyExe) Write-Step "Installing build dependencies" -& $pyExe -m pip install --upgrade pip --quiet +& $pyExe -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS build dependency bootstrap failed." } & $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." } diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 183b31601..5c85014d6 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -127,7 +127,7 @@ services: - ${RENDER_GID:-render} chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -145,8 +145,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -196,7 +196,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 23404b2eb..5c628fcfb 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -130,7 +130,7 @@ services: capabilities: [gpu] chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -148,8 +148,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -199,7 +199,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker-compose.yml b/docker-compose.yml index a7ddf2f04..2b90227a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: restart: unless-stopped chromadb: - image: docker.io/chromadb/chroma:latest + image: docker.io/chromadb/chroma:1.5.9@sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6 ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -126,8 +126,8 @@ services: # (depends_on: condition: service_healthy), so a broken upstream `latest` # tag blocks the whole app from starting. 2026.6.2 crashes on boot with # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). - # Bump this deliberately after verifying a newer tag boots clean. - image: docker.io/searxng/searxng:2026.9.25-12f8b6515@sha256:5286edb35782454ab8a102c5eff6b54bff745853191b46aeead95f225aa6dfb6 + # Update all default/GPU pins together after native startup and trust checks. + image: docker.io/searxng/searxng:2026.10.24-ef0ec03da@sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8 entrypoint: - /bin/sh - -c @@ -177,7 +177,7 @@ services: restart: unless-stopped ntfy: - image: docker.io/binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy:v2.32.0@sha256:59250e1d8be2ee2186c4560322f625faef22ace380331f05b63f8842dbd62602 command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker/chromium-ca-policy.md b/docker/chromium-ca-policy.md new file mode 100644 index 000000000..b4852ac3a --- /dev/null +++ b/docker/chromium-ca-policy.md @@ -0,0 +1,20 @@ +# Chromium incident distrust policy + +`chromium-emudhra.json` is installed in the Docker image at `/etc/chromium/policies/managed/odysseus-emudhra.json`. Chromium 132 and later support `CADistrustedCertificates`: a list of base64-encoded DER X.509 certificates whose public keys must not be accepted for TLS server authentication. Matching is by SubjectPublicKeyInfo (SPKI), so one certificate per compromised key also blocks its cross-signs. This policy adds no trust, disables no certificate checks, and does not require a revocation-list download. Native browsers and other browser distributions require separately managed installation of the appropriate policy. + +Policy semantics: [Chromium's version-155 definition](https://github.com/chromium/chromium/blob/155.0.8059.79/components/policy/resources/templates/policy_definitions/CertificateManagement/CADistrustedCertificates.yaml). Linux installation paths: [Chromium policy documentation](https://www.chromium.org/administrators/configuring-policy-for-extensions/#linux). + +Public certificate source, retrieved 2026-10-21: [incident intermediate inventory](https://emudhra.we-have-a-problem.com/impacted-intermediates.json). Its downloaded JSON has SHA-256 `f67b3abe9b4879f4e91874ed041beccd71257f1db41c588c724ccf12aa90c59e`. Each selected certificate's DER fingerprint and public-key fingerprint were checked against that inventory before inclusion. The eight unique keys below are covered, including the code-signing intermediate keys listed in the incident. The policy contains only public certificates; no private key is included. + +| Intermediate | SHA-256 of DER SPKI | +| --- | --- | +| EM OV TLS CA - G3B | `43bffa19d6ef651451f6841e019931e96f6f2eaeff28ab4b6f1ce4576de2717b` | +| EM DV TLS CA - G3C | `49dd353dc2b919a3770645117d0e60c16115e7d1bc259eefe734b00ea434fe5f` | +| EM EV TLS CA - G3C | `5c0660a2b164910d8ff589c18e14552e9e0ec237dcefe2cb736d41a4811f81a0` | +| EM OV TLS CA - G3C | `63ecefee431d49523bc1136c4e72d0206a477086caf606cd5cf025686c109cf8` | +| EM OV CS CA - G3C | `92c063d67e8f14db71d288e11266ffdaf69cbf115f7bab14f3bc2f68d25e0e30` | +| EM EV TLS CA - G3B | `e0254c7fcdc265e2f0f8a428dabb75ab4ae039b4872d62f5ee57f0877d5a8a41` | +| EM DV TLS CA - G3B | `ebed6475c2c2d80fcb199f3fe8dd6e6b3ce60b77eb15d4e42c3dec0c6f5b5105` | +| EM OV CS CA - G3B | `ec4ffa1ad0ac88081314b7aaffef6a91f97cd57291bb5ba686c5415f5eb4f6b6` | + +`tests/test_chromium_ca_policy.py` verifies that the policy distrusts exactly these eight keys using valid intermediate CA certificates and adds no trust settings. Changes to the encoded material need a fresh source/fingerprint check and behavioral validation: all twelve published incident HTTPS chains must be rejected by the actual Chromium binary while ordinary HTTPS still succeeds. Confirm loading and any policy errors at `chrome://policy`. Do not treat a passing PEM-store check as a browser-policy check. diff --git a/docker/chromium-emudhra.json b/docker/chromium-emudhra.json new file mode 100644 index 000000000..104f82f60 --- /dev/null +++ b/docker/chromium-emudhra.json @@ -0,0 +1,12 @@ +{ + "CADistrustedCertificates": [ + "MIIDZjCCAu2gAwIBAgIPbK2jZ/scEACIEg6iAtVoMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNjAwMTJaFw00MDA0MjgxODMwMDBaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJWMRswGQYDVQQDDBJFTSBPViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARziXqg6zz9yosSQC/68KvE4SErr43ud4rJ7i1tuD+sKAxeBWOK56Ks1xwyKo42HN659pxDArndjXPRzGyqsswK7k4hbOl53VGKSvS2JuVoTN1lgDftDIVOTMEvTPhkbQ2jggGWMIIBkjAfBgNVHSMEGDAWgBThOqshfPHjTVSu1HK3/YtZQiCQVDAdBgNVHQ4EFgQUm9hlUa/n5yw3rQjuaRnZvzeErgowHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMF0GA1UdIARWMFQwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTANBgsrBgEEAYOOIQECbjAIBgZngQwBAgIwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1PVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNnADBkAjBAk+5BizdJhrED3wb7F2Gm7bWI60dlNd075nghT2TrR4NWKZtJZcbc28xxIY/NONICMF6mPdsjosaqmvEMWqKcBh4vsIvXSKnQj5o8zrFB4oKS+accjYTe8Dz3bQjauvy9uw==", + "MIIDaDCCAu6gAwIBAgIPWIsuPCu4zaE+GhIJJUMhMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM1MDlaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOnp4YsjETTJN3WkDOmeCwO/u/kBV5ooSP6A3G6O9kZRRTuFypDPjYw8CvRiAN9nlFJ+bd9uUooHfmHsv6oiLdL3+j9pMEJnubKGVZv4+8OX2sJpVbMyLpQVh3xsqRB0Bo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFK5+EFw8KZbBqEcnBVnx4D3U5QrzMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMEN2HWMocbuWdy1d33pH6JpMvP/V/DIS5PtZbv1Q7TXEbxrcyxgyuddOJscYDJJlAIwB4sH1BpbScDzkdD4tJBSwrG/1Mx82vb8YaySehNuu1+1oXNdAEKwaJ0mVzScy2+6", + "MIIDaDCCAu6gAwIBAgIQAIAJkopkEHYBrwX6LxnxCTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTUzOTMxWhcNNDAwNDI4MTgzMDAwWjBAMQswCQYDVQQGEwJVUzEUMBIGA1UECgwLZU11ZGhyYSBJbmMxGzAZBgNVBAMMEkVNIEVWIFRMUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABC1W6Cmm+4LGSkaX2WCvlR6QZ66R+SbTQwbaSKiq63hptqZ+bSdCbgRCqgOzbhmI3/Y2UU7hzwf1uks/8qBmRa59MdWGatMao9JG6wsxd+6Zy3VmxRbOdICKZo+R8MR/yqOCAZUwggGRMB8GA1UdIwQYMBaAFOE6qyF88eNNVK7Ucrf9i1lCIJBUMB0GA1UdDgQWBBSmJV6AqU2385Z8G8eR35K87kvrXzAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwXAYDVR0gBFUwUzA5BgsrBgEEAYOOIQEAATAqMCgGCCsGAQUFBwIBFhxodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tMA0GCysGAQQBg44hAQJ4MAcGBWeBDAEBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRVZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwBy1wTnzH4lXDTy2PD91ysLjvnlVN3JbJmc1BIMEU8TOLkYVHibMc9HeyonikFSfmAjEA6uE3QRxlrZTz/1/lXPTEMUhoauAtycagtlNFp79kjhvNa2/fAal1GNxsOutLnr0Y", + "MIIDaDCCAu6gAwIBAgIPKWkmwiejoVJjkrGUgqUEMAoGCCqGSM49BAMDMFYxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMSAwHgYDVQQDDBdlbVNpZ24gUm9vdCBUTFMgQ0EgLSBHMzAeFw0yNTA0MzAxNTM3NDFaFw00MDA0MjgxODMwMDBaMEAxCzAJBgNVBAYTAlVTMRQwEgYDVQQKDAtlTXVkaHJhIEluYzEbMBkGA1UEAwwSRU0gT1YgVExTIENBIC0gRzNDMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEE4JRzVKskpjByKrZdzId65fnCb0LTKCspV3IrVHc/J0ViaFZfjpRIOHv1jHGPhg1ysrimGmmLhYx6KKBgatWYW3EXZGKvAnzqw5jz3kqpNztc4e+N7ulaVSfHAKycO1no4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFDTq6gzryQra+dDF4u2beMmHnhtsMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAm4wCAYGZ4EMAQICMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYy5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZUTFNDQUczQy5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIxAMSZt2xsPeumRdeSMSQ3CbzIZeQ/q7GDCWfeE1Pq+mhgA6gRM44cNVF3xJ+wsxJk+wIwFJbaDMokD8Ei6eNOiA8yUMYvfzQyngmrhZQMdPCK1SewrR794rSb9vGqxZzelQth", + "MIIDWzCCAuGgAwIBAgIPVEt/kiwxriIKSpIpFvbmMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2Mzg0MFoXDTQwMDQyODE4MzAwMFowPzELMAkGA1UEBhMCVVMxFDASBgNVBAoMC2VNdWRocmEgSW5jMRowGAYDVQQDDBFFTSBPViBDUyBDQSAtIEczQzB2MBAGByqGSM49AgEGBSuBBAAiA2IABLnw4GNHAqacpqrdRDz9YsQbm6SyCSRpfXl052kaEm+5avs+NML9O8X0gwYfJl3CpYQ2NHMjUQQ2so8E84YjYwZ5gOJQPgJTLWVGI1pxZaJZoe9Jb3QbX97neHII5Lj676OCAYswggGHMB8GA1UdIwQYMBaAFGh0A/iFyo1UUFcdFuDI3kUNq80tMB0GA1UdDgQWBBTub3MpjW+/p7Mbw5U09hVWRHhmtzATBgNVHSUEDDAKBggrBgEFBQcDAzBeBgNVHSAEVzBVMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDgYMKwYBBAGDjiEBAoFIMAgGBmeBDAEEATB0BggrBgEFBQcBAQRoMGYwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLWMuZW1zaWduLmNvbTA+BggrBgEFBQcwAoYyaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbS9jZXJ0cy9FTU9WQ1NDQUczQy5wN2MwNgYDVR0fBC8wLTAroCmgJ4YlaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290Y3NjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNoADBlAjEA8yPg8sntOhLwV2QXJ5LEaKSpo/fnE3uOgTadK73fDhyjiqr0Io9GVuQTYqBPbQ2TAjBgeuApmBS486+iei03lRrp139dPw6k28jjGNvWsAU7A1N86J1zPJVDEyVH+nMAWC0=", + "MIIDaDCCAu2gAwIBAgIQAMfEHv6S2CXwXq6EGkqCwTAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTYwMzMxWhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRVYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEJduoWmTTl9oLEFLp6lRuSHvv6pydnWigxerc/u91ppu11vwB9Cv7ym60C81kwHZ+Nv/QXZeoE8WqDGyVMOec9fCGZGjxOAZCCqXIUYQLHiUlQOeQ5LZUw4Uu1krx6GV9o4IBlTCCAZEwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFD7cINmJRpiHe4c/2IFHslwV2EcGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBcBgNVHSAEVTBTMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAngwBwYFZ4EMAQEwdQYIKwYBBQUHAQEEaTBnMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC1iLmVtc2lnbi5jb20wPwYIKwYBBQUHMAKGM2h0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20vY2VydHMvRU1FVlRMU0NBRzNCLnA3YzA3BgNVHR8EMDAuMCygKqAohiZodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3R0bHNjYWczLmNybDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAKBggqhkjOPQQDAwNpADBmAjEA9Jq999oaOUKhFwopw33iCmt0OHIn3gjT43GfuOHR/YckXdwoNdDAlIEhz0NgRO3/AjEAvY/a3OMvktIZLYjn+WLKmp0QhrIMV+e6EpS7j4GVeOKouNp6MN/P+ByGeiRMghzB", + "MIIDaDCCAu6gAwIBAgIQAKSu5njyM2K0wORgt3LyFDAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAwwXZW1TaWduIFJvb3QgVExTIENBIC0gRzMwHhcNMjUwNDMwMTU1NDA4WhcNNDAwNDI4MTgzMDAwWjA/MQswCQYDVQQGEwJOTDETMBEGA1UECgwKZU11ZGhyYSBCVjEbMBkGA1UEAwwSRU0gRFYgVExTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE557ZwEqXc8RzXlERHahawr1eZKs3S4DhLLwgOOweTw47gS/JhZ4eZ6+wTi+LyX90hbplqpEe6KTkW2LVzAPVa+Z4s8OMtNyTfKJGDiOEU8iSk10p1E+R4VYGtjzO2rRoo4IBljCCAZIwHwYDVR0jBBgwFoAU4TqrIXzx401UrtRyt/2LWUIgkFQwHQYDVR0OBBYEFOIeL+ccGTkOVUtiwSAkVzUkiiUUMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBdBgNVHSAEVjBUMDkGCysGAQQBg44hAQABMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9yZXBvc2l0b3J5LmVtc2lnbi5jb20wDQYLKwYBBAGDjiEBAmQwCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD8GCCsGAQUFBzAChjNodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNRFZUTFNDQUczQi5wN2MwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5lbXNpZ24uY29tLz9yb290dGxzY2FnMy5jcmwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwCgYIKoZIzj0EAwMDaAAwZQIwLmsGXi6oFMJ2klkcEMXMug6gvxQWZWkewiDWVzQyl2L6z6DuQfPOIbh1grvdULDCAjEAxZvLlxUavG69J6wnSjC8XBbqNIQsJ6CE2QdNOe84Ptxb6VG8gt2TlfJbHii+KNvK", + "MIIDWzCCAuCgAwIBAgIPeBiTP4+FtGGaKzX7qD0KMAoGCCqGSM49BAMDMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxlTXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9vdCBDUyBDQSAtIEczMB4XDTI1MDQzMDE2MDgyN1oXDTQwMDQyODE4MzAwMFowPjELMAkGA1UEBhMCTkwxEzARBgNVBAoMCmVNdWRocmEgQlYxGjAYBgNVBAMMEUVNIE9WIENTIENBIC0gRzNCMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEqrsPsGZqz88B+tUr/P1MK/LLPDSX3nOVDlgZth/mlQmHQcjEkUObNKBDHK/+W/m0wcjorgAB+fTcitgsWfNjnpbNbD1NsM7+wmw9YlF7N3YTcnQ/jordd7EAZPKr5eBQo4IBizCCAYcwHwYDVR0jBBgwFoAUaHQD+IXKjVRQVx0W4MjeRQ2rzS0wHQYDVR0OBBYEFHn66OzP2w11jaRY1p+27dx8Sd7mMBMGA1UdJQQMMAoGCCsGAQUFBwMDMF4GA1UdIARXMFUwOQYLKwYBBAGDjiEBAAEwKjAoBggrBgEFBQcCARYcaHR0cDovL3JlcG9zaXRvcnkuZW1zaWduLmNvbTAOBgwrBgEEAYOOIQECgUgwCAYGZ4EMAQQBMHQGCCsGAQUFBwEBBGgwZjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AtYi5lbXNpZ24uY29tMD4GCCsGAQUFBzAChjJodHRwOi8vcmVwb3NpdG9yeS5lbXNpZ24uY29tL2NlcnRzL0VNT1ZDU0NBRzNCLnA3YzA2BgNVHR8ELzAtMCugKaAnhiVodHRwOi8vY3JsLmVtc2lnbi5jb20vP3Jvb3Rjc2NhZzMuY3JsMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMAoGCCqGSM49BAMDA2kAMGYCMQC/zr03YEELsNKqzUtH2dVFi4BaKFdyrr/6G1dIWp8KAVgsw2/u4lyyboUDaC9jDvICMQC9HTxku84tNX9dmHOtH7aC2Nu/JM0jTf6KjZAvovCJzzv1IFWSZpVLZibynMLXOrM=" + ] +} diff --git a/docker/searxng/Dockerfile b/docker/searxng/Dockerfile new file mode 100644 index 000000000..0b487e66c --- /dev/null +++ b/docker/searxng/Dockerfile @@ -0,0 +1,9 @@ +# Validation only. Deploy the unmodified upstream image selected by Compose. +ARG SEARXNG_IMAGE +FROM ${SEARXNG_IMAGE} AS validation +COPY scripts/check_ca_trust.py docker/searxng/validate_image.py /opt/odysseus-trust/ +COPY tests/fixtures/ca-trust/incident.json /opt/odysseus-trust/incident.json +RUN /usr/local/searxng/.venv/bin/python /opt/odysseus-trust/validate_image.py + +FROM validation AS live-validation +RUN /usr/local/searxng/.venv/bin/python /opt/odysseus-trust/validate_image.py --live diff --git a/docker/searxng/Dockerfile.dockerignore b/docker/searxng/Dockerfile.dockerignore new file mode 100644 index 000000000..3e7e75e3f --- /dev/null +++ b/docker/searxng/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +** +!scripts/ +!scripts/check_ca_trust.py +!docker/ +!docker/searxng/ +!docker/searxng/** +!tests/ +!tests/fixtures/ +!tests/fixtures/ca-trust/ +!tests/fixtures/ca-trust/incident.json diff --git a/docker/searxng/validate_image.py b/docker/searxng/validate_image.py new file mode 100644 index 000000000..c768cf010 --- /dev/null +++ b/docker/searxng/validate_image.py @@ -0,0 +1,162 @@ +"""Verify incident distrust and startup in the unmodified upstream image.""" + +import hashlib +import asyncio +import json +import os +from pathlib import Path +import re +import signal +import ssl +import subprocess +import sys +import tempfile +import time +import urllib.request + +import certifi + +from check_ca_trust import check_bundle + + +def check_chains(): + incident = json.loads(Path(__file__).with_name("incident.json").read_text()) + system = Path(os.environ["SSL_CERT_FILE"]) + capath = os.environ["SSL_CERT_DIR"] + stores = [["-CAfile", str(system), "-no-CApath"], + ["-CAfile", certifi.where(), "-no-CApath"], + ["-no-CAfile", "-CApath", capath]] + check_bundle(system) + check_bundle(Path(certifi.where())) + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + for fingerprint, pem in incident["certificates"].items(): + if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != fingerprint: + raise ValueError(f"incident fixture fingerprint mismatch: {fingerprint}") + (directory / fingerprint).write_text(pem) + positive = directory / "affected.pem" + positive.write_text("\n".join(incident["certificates"][fp] for fp in incident["roots"])) + for hostname, chain in incident["chains"].items(): + untrusted = directory / "untrusted.pem" + untrusted.write_text("\n".join(incident["certificates"][fp] for fp in chain[1:])) + base = ["openssl", "verify", "-no-CAstore", "-attime", str(incident["verification_time"]), + "-purpose", "sslserver", "-verify_hostname", hostname, + "-x509_strict", "-partial_chain", "-trusted_first", "-untrusted", str(untrusted)] + expectations = [(selection, False) for selection in stores] + for selection, expected in expectations + [ + (["-CAfile", str(positive), "-no-CApath"], True)]: + result = subprocess.run(base + selection + [str(directory / chain[0])], + capture_output=True, text=True) + if (result.returncode == 0) != expected: + raise RuntimeError(f"unexpected chain result: {hostname}: {selection}: {result.stderr}") + # Accept issuer lookup, self-signed and explicitly untrusted + # certificate errors, rather than tool or fixture failures. + if not expected and not re.search(r"error (?:2|18|19|20|21|27) at \d+ depth lookup", result.stderr): + raise RuntimeError(f"not a certificate trust rejection: {hostname}: {result.stderr}") + print(f"{len(incident['chains'])}/{len(incident['chains'])} incident chains rejected by system PEM, hashed directory and certifi; affected controls accepted") + + +def check_boot(live=False): + with tempfile.TemporaryDirectory() as directory: + settings = Path(directory) / "settings.yml" + settings.write_text("use_default_settings: true\nserver:\n secret_key: offline-build-validation-only\n limiter: false\nsearch:\n formats: [html, json]\n") + cache = Path(directory) / "cache" + cache.mkdir() + env = dict(os.environ, SEARXNG_SETTINGS_PATH=str(settings), + __SEARXNG_CONFIG_PATH=directory, __SEARXNG_DATA_PATH=str(cache), + GRANIAN_HOST="127.0.0.1", GRANIAN_PORT="18888") + # Do not expose a port or issue an outbound search during offline builds. + with tempfile.TemporaryFile() as log: + process = subprocess.Popen(["/usr/local/searxng/entrypoint.sh"], + cwd="/usr/local/searxng", env=env, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True) + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + healthy = False + for _ in range(60): + if process.poll() is not None: + break + try: + with opener.open("http://127.0.0.1:18888/healthz", timeout=1) as response: + healthy = response.status == 200 + except OSError: + pass + if healthy: + break + time.sleep(0.5) + if not healthy: + log.seek(0) + raise RuntimeError("SearXNG failed boot smoke: " + log.read().decode(errors="replace")[-6000:]) + print("SearXNG loopback healthcheck passed") + # Its root entrypoint regenerates system trust. Repeat the + # behavioral checks after that real startup path completes. + check_running(live=live, port=18888) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def check_running(live=False, port=None): + """Verify an existing service without repairing it or starting another one.""" + port = int(os.environ.get("GRANIAN_PORT", "8080")) if port is None else port + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(f"http://127.0.0.1:{port}/healthz", timeout=5) as response: + if response.status != 200: + raise RuntimeError("SearXNG running-service healthcheck failed") + check_chains() + for path in Path(os.environ["SSL_CERT_DIR"]).iterdir(): + if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name): + check_bundle(path) + if live: + asyncio.run(check_https()) + with opener.open(f"http://127.0.0.1:{port}/search?q=python+documentation&format=json", timeout=90) as search: + payload = json.load(search) + results = payload.get("results", []) + if not results: + raise RuntimeError("live search returned no results") + print(f"SearXNG live JSON search passed: {len(results)} results") + + +async def check_https(): + # Exercise the actual pinned SearXNG client factory with its default CA + # selection, rather than a different HTTP library's trust configuration. + sys.path.insert(0, "/usr/local/searxng") + from curl_cffi.requests.exceptions import RequestException + from searx.network.client import new_client + + client = new_client(enable_http=False, verify=True, enable_http2=True, + enable_http3=False, max_connections=2, proxies={}, + local_address=None, max_redirects=3) + try: + response = await client.get("https://pypi.org/simple/certifi/", timeout=30) + if response.status_code != 200: + raise RuntimeError(f"ordinary HTTPS failed: {response.status_code}") + print("SearXNG actual curl client ordinary HTTPS passed with verification enabled") + incident = json.loads(Path(__file__).with_name("incident.json").read_text()) + for hostname in incident["chains"]: + try: + await client.get(f"https://{hostname}/", timeout=30, allow_redirects=False) + except RequestException as exc: + # libcurl code 60 is peer certificate verification failure. + # DNS failures, timeouts and handshake EOF do not prove distrust. + if exc.code != 60: + raise RuntimeError(f"inconclusive incident TLS failure: {hostname}: {exc}") from exc + else: + raise RuntimeError(f"incident HTTPS unexpectedly accepted: {hostname}") + print(f"SearXNG actual curl client rejects {len(incident['chains'])}/{len(incident['chains'])} incident hosts with certificate errors after startup") + finally: + await client.aclose() + + +if __name__ == "__main__": + live = "--live" in sys.argv + if "--running" in sys.argv: + check_running(live=live) + else: + check_chains() + check_boot(live=live) diff --git a/docs/EDITOR_AGENT_API.md b/docs/EDITOR_AGENT_API.md new file mode 100644 index 000000000..b53873785 --- /dev/null +++ b/docs/EDITOR_AGENT_API.md @@ -0,0 +1,208 @@ +# Editor Agent API + +How an agent (LLM or script) drives the PhotoCraft editor. There are three layers: + +1. **Control API** — `window.geEditorControl`, a stable in-page command surface. +2. **NL planner** — `window.geEditorAI`, turns a prompt into control-API steps (LLM or basic mode). +3. **Bridge** — server-mediated SSE channel so a *server-side* agent can drive an **open** editor. + +All three are owner-scoped by the app session; nothing is exposed to unauthenticated users. + +--- + +## 1. Control API (`window.geEditorControl`) + +Installed when the editor opens. Version `EDITOR_CONTROL_VERSION`. + +```js +await window.geEditorControl.invoke(op, params) // { ok, op, result } | { ok:false, op, error } +await window.geEditorControl.run([{op, params}]) // sequential, stops on first failure +window.geEditorControl.list() // [{op, description}] +``` + +`invoke` is async. Every op that mutates the document pushes **one** undo entry via +`_saveState`, so a single op is one undo step. + +### Op catalog + +| Op | Params | Notes | +| --- | --- | --- | +| `document.info` | – | size, layers, mode | +| `document.importImage` | `{dataUrl,name,raster}` | load a raster image as a new layer | +| `layers.list` | – | top-most first | +| `layers.select` | `{id\|index\|name}` | index is top-most-first | +| `layers.addRaster` | – | new empty pixel layer | +| `layers.addAdjustment` | `{type,params}` | adjustment layer | +| `layers.duplicate` / `layers.delete` | – | on the active/selected layer(s) | +| `layers.setOpacity` | `{id?,opacity 0-1}` | | +| `layers.setBlend` | `{id?,blendMode\|mode}` | | +| `layers.group` / `layers.ungroup` | – | | +| `selection.selectAll` / `deselect` / `invert` | – | | +| `selection.rectangle` / `selection.ellipse` | `{x,y,w,h}` | document space | +| `selection.subject` | `{label}` | SAM text grounding (falls back to a centred rect) | +| `selection.face` | – | SAM "face" (falls back to an upper-centre ellipse) | +| `selection.sky` | – | SAM "sky" (falls back to a top-band colour range) | +| `selection.background` | `{fuzziness}` | border colour union | +| `selection.samText` | `{text}` | SAM text grounding only (throws without SAM) | +| `selection.colorRange` | `{color,fuzziness}` | | +| `selection.hasMask` | – | `{active, space}` | +| `adjustments.apply` | `{type,params}` | adjustment layer; camelCase types accepted | +| `adjustments.applyMasked` | `{brightness,contrast,saturation,warmth,invert}` | pixel pass, masked if a selection exists else whole layer | +| `filters.apply` | `{id}` | opens the slider dialog and applies | +| `filters.applyMasked` | `{id,radius,amount,invert}` | `id`: blur-gaussian, sharpen, unsharp-mask | +| `transform.start` / `transform.apply` / `transform.cancel` | – | | +| `text.add` | `{text,x,y,color,size,font,align}` | | +| `history.undo` / `history.redo` | – | | +| `history.mark` | – | returns current undo depth (for grouping) | +| `history.collapse` | `{mark,label}` | collapses entries above `mark` into one | +| `export.png` | – | `{dataUrl}` of the flattened document | +| `export.gallery` | – | saves a copy back to the gallery | +| `screenshot` | – | `{dataUrl,width,height}` of the composited canvas | + +Param conventions the planner must respect: + +- `adjustments.applyMasked`: `brightness`/`contrast`/`warmth` are **-100..100** (0 neutral); + `saturation` is a **multiplier** (1 neutral; values >3 are treated as percentages). +- `adjustments.apply` `type` aliases are normalised (`blackAndWhite` -> `black-white`). +- `color-balance` params are `{shadows:{r,g,b}, midtones:{r,g,b}, highlights:{r,g,b}}`; + a bare number is coerced to `{r:n,g:n,b:n}`. +- A masked op with **no** active selection degrades to a whole-layer edit (does not throw). + +--- + +## 2. NL planner (`window.geEditorAI`) + +```js +const res = await window.geEditorAI.execute('blur the background behind the person', { onStep }); +// exec: window.geEditorAI.execute('blur the background behind the person', { onStep }); +// res = { ok, prompt, reply, planner, basicMode, steps, trace, failed } +``` + +Flow (`static/js/editor/nl-edit.js`): + +1. Read document state + a **before** screenshot. +2. Ask the **server planner** (`POST /api/editor/plan`) with the prompt, state and + screenshot. The client passes its strong guidance via `state.planner_guidance`. + If the route is unavailable it falls back to `localPlan` (regex — labelled + **"basic mode"** in the UI). +3. `sanitizeSteps()`: drop no-op/export steps and, if a masked op has no preceding + `selection.*`, insert the region implied by the prompt (face/sky/background/subject). +4. `history.mark`, then run each step, screenshotting after each one. +5. **Self-check** (server plans only, max 3 rounds): send the after screenshot back in + `mode:"check"` with the original intent. If not satisfied, **revert** the previous + attempt (undo its history entries) and apply the model's **full corrected** plan. +6. `history.collapse` — the whole request becomes a single undoable step. + +UI: the Quick Edit box routes multi-clause / "and ..." prompts to this executor and +streams `Step i/n: ` into its status line; a `(basic mode)` suffix marks the offline planner. + +### Server route + +`POST /api/editor/plan` (HARNESS, `routes/editor_agent_routes.py`) + +``` +{ prompt, state, mode?: "plan"|"check", screenshot?: "data:image/..." } +-> { reply, planner, steps: [{op, params}], done? } +``` + +Owner-scoped, authenticated. Uses the configured default/utility model with vision. +If the model call is unavailable it returns a deterministic `planner:"fallback"` plan. + +--- + +## 3. Bridge (server agent drives an open editor) + +`static/js/editor/editor-bridge.js` subscribes the open editor to the server queue. +Enabled automatically when the editor opens (`window.__geEditorBridge`). + +- Subscribe: `GET /api/editor/bridge/events?session=` — SSE, `data: {id,op,params}` (keepalive every 15s). +- The page runs the op via the control API and replies: + `POST /api/editor/bridge/result {id,ok,result,error,screenshot,ms}`. +- Enqueue a command (server side): `POST /api/editor/bridge/command {session, op|steps, wait, timeout}`. + With `wait:true` and one step the call blocks and returns the editor's result. + Otherwise use `POST /api/editor/bridge/await {id, timeout}` for each queued id. + Timeouts are seconds, greater than zero and at most 900. Pending results expire + after the command timeout; an unknown or another owner's id returns 404. + +Round-trip example (authenticated; `` = `window.__geEditorBridge.session`): + +```bash +curl -s -X POST http://127.0.0.1:7011/api/editor/bridge/command \ + -H 'Content-Type: application/json' -b "odysseus_session=$TOKEN" \ + -d '{"session":"default","op":"layers.addRaster","wait":true,"timeout":8}' +# -> {"id":"...","ok":true,"result":{...},"screenshot":"data:image/png;base64,..."} +``` + +Session scoping uses the authenticated owner and session as separate values. A +bound editor-tool turn uses its server-owned session, not a model-selected one. +The bridge is a **browser-only** flow: it requires an editor page to be open. +For `/api/editor/agent`, clients must send `incognito: true` for private image +results; omitted or false requests use normal gallery persistence. These +owner-scoped routes do not accept delegated API tokens without a scope contract. + +--- + +## 4. SAM dependency + +`selection.subject` / `selection.face` / `selection.sky` / `selection.samText` use the +server SAM mask tool (`POST /api/image/mask`). When the SAM tools are **not installed** +the editor raises *"Object mask tools are not installed"* and the control op falls back +to a **coarse approximation** (centred rect / upper-centre ellipse / top-band colour range). +On a plain background the approximation is acceptable; on real photos it leaks, producing +visible rectangular/elliptical artifacts. Install the Cookbook SAM mask dependencies for +true subject/face/sky segmentation. + +`selection.colorRange` and `selection.background` never need SAM (pure colour matching), +but they are also coarse on busy scenes. + +--- + +## 5. Two ways for an agent to use this — pick one + +### Option A — built-in tool (in-process) + +Register an editor-control tool inside the app (same process as the model) that calls the +bridge/control API directly. Uses the app's normal login; nothing new is exposed. This is +the rejected stdio-MCP design's replacement: a server-side tool that talks to the bridge +needs either an internal token (auth-bypass) or to run in-process. **Decision pending (user).** + +### Option B — browser-only flow (works today) + +No new server tool. An external agent drives an open editor purely through the bridge +endpoints using the logged-in owner's session. Restricted to the editor the user has open. +This is the flow demonstrated above and requires **no** auth-bypass. + +A stdio `photo_editor` MCP server was prototyped and **rejected** because the built-in MCP +subprocess has no internal token; wiring it would require expanding an auth bypass. +The two viable options above are left for the user to choose. + +--- + +## 6. Known limitations / honest grading + +Live run against real 7011 with the server LLM planner (7 prompts): all 7 executed +(`planner:"llm"`). Graded by eye: + +- PASS: black & white portrait; warm cinematic; sharper + more vivid; text title. +- FAIL (region-masked, SAM-bound): "dramatic sky" (global over-saturation instead of a + dramatic, masked sky), "blur the background" (coarse mask leaks; visible rectangle), + "brighten the face" (coarse ellipse/rect artifact). + +Whole-image edits are reliable; subject/face/sky region edits need SAM. The self-check +loop works (revert-and-replan) but the region cases spend all 3 iterations. + +## 7. Theming (control kit) + +The editor's interactive controls (ranges, number fields, selects, checkboxes) share a +single themed kit: `static/js/editor/control-kit.js` adds `.ge-ctl-*` classes scoped to +the editor roots (`#gallery-editor-container`, `.ge-adj-popup`, `.ge-filter-overlay`, +`.ge-canvas-size-overlay`, `.ge-float-window`, `.ge-transform-popup`), and +`static/css/13-image-editor.css` (section "Editor control kit") styles them from the app +design tokens (`static/css/00-tokens.css`) to match the app sliders/selects. It keeps +range fills in sync and attaches a **scrub-on-label** affordance to numeric fields +(drag the field's label to change the value; arrow keys work, Shift = x10). + +**The PhotoCraft editor is dark-only.** `:root.light` flips the global tokens, but +`13-image-editor.css` has no `.light` handling and the editor relies on dark surfaces, +so light mode renders a garbled mix. Unlike the rest of Odysseus, the editor follows +Photoshop and is not offered in a light theme. diff --git a/docs/QUICKEDIT_COVERAGE.md b/docs/QUICKEDIT_COVERAGE.md new file mode 100644 index 000000000..895d84502 --- /dev/null +++ b/docs/QUICKEDIT_COVERAGE.md @@ -0,0 +1,255 @@ +# Quick Edit Coverage Audit + +Audit of how much of the PhotoCraft editor is reachable through **Quick Edit** +(the natural-language editor agent). Built against build **0.20.44** (HARNESS +bridge fan-out fix `4dd11826`) and client commit `e78b5a96`. + +Quick Edit executes in two stages: + +1. **Plan** — `static/js/editor/editor-agent.js` posts `{prompt, state, + screenshot}` to `/api/editor/agent`. HARNESS `routes/editor_agent_routes.py` + plans with the shared style guide (`src/editor_style_guide.py`) plus an LLM, + validates every step against its own `OP_CATALOG`, then executes. +2. **Execute** — only `ai_edit_image` / `generate_image` run server-side; every + other op is delivered to the page over the editor bridge (SSE) and run by the + client control surface (`window.geEditorControl.invoke`, catalog in + `static/js/galleryEditor.js`). A local fallback planner lives in + `static/js/editor/nl-edit.js`. + +There are **three** catalogs that must agree: + +| Catalog | File | Owner | +|---|---|---| +| Client control ops | `static/js/galleryEditor.js` | PHOTOSHOP | +| Server `OP_CATALOG` | `routes/editor_agent_routes.py` | HARNESS | +| Style guide / planner | `src/editor_style_guide.py` + `nl-edit.js` | HARNESS + PHOTOSHOP | + +Legend: **C** = in client catalog · **S** = in server `OP_CATALOG` (incl. +`NEW_OP_NAMES`) · **P** = the planner/style guide emits it for some phrasing. +A capability is **usable from Quick Edit** only when the planner can emit it +(**P**) and the op exists on the side that runs it. + +--- + +## 1. Menu bar (12 menus) + +| Capability | Trigger phrasing | Op | C | S | P | Status | +|---|---|---|---|---|---|---| +| File → Import | "open this image" | `document.importImage` | ✔ | ✔ | partial | OK | +| File → Load project | "open my project" | `document.openPsd` | ✔ | ✘ | ✘ | **Server gap** | +| File → Save / Save copy / Export PSD | "save as psd" | `document.savePsd` | ✔ | ✔ | ✔ | OK | +| File → Export | "export png" | `export.png` | ✔ | ✔ | ✔ | OK | +| File → Save project | — | (project store) | n/a | n/a | ✘ | Out of scope (not an edit op) | +| Edit → Undo/Redo/History | "undo" | `history.undo` / `redo` | ✔ | ✔ | ✔ | OK | +| Edit → Free Transform | "free transform" | `transform.start` | ✔ | ✔ | ✔ | OK | +| Edit → Transform → Rotate 90/180 | "rotate this layer 90" | `layer.rotate` | ✔ | ✔ | ✔ | OK (whole image vs layer routed by the fast path) | +| Edit → Transform → Flip H/V | "flip this layer" | `layer.flip` | ✔ | ✔ | ✔ | OK (whole image vs layer routed by the fast path) | +| Image → Fill | "fill the whole image with white" | — | ✘ | ✘ | ✘ | **No op** | +| Image → Canvas Size | "make the canvas wider" | `document.resizeCanvas` | ✔ | ✔ | partial | OK | +| Image → Image Size | "resize the image to 1200px" | `document.resizeCanvas` | ✔ | ✔ | partial | OK | +| Image → Rotate 90/180 (whole image) | "rotate the whole image 90" | `document.rotate`/all-layers | ✘ | ✘ | ✘ | **No control op** | +| Image → Flip H/V (whole image) | "flip the whole canvas" | `document.flip`/all-layers | ✘ | ✘ | ✘ | **No control op** | +| Image → Mode RGB/Grayscale | "make it black and white" | `adjustments.apply` black-white | ✔ | ✔ | ✔ | OK (mode via adjustment) | +| Image → Adjustments → Invert | "invert the colours" | `adjustments.apply` invert | ✔ | ✔ | ✔ | OK (fast path colour-inverts the active layer; "invert the selection" stays Select ▸ Inverse) | +| Image → Mode CMYK/Lab, Bits 16/32 | — | — | ✘ | ✘ | ✘ | Not implemented (disabled in UI) | +| Layer → New | "add a layer" | `layers.addRaster` | ✔ | ✔ | partial | OK | +| Layer → Duplicate | "duplicate the layer" | `layers.duplicate` | ✔ | ✔ | partial | OK | +| Layer → Group | "group the layers" | `layers.group` | ✔ | ✔ | ✘ | **Server OK, planner gap** | +| Layer → Rasterize | "rasterize the text" | `layers.rasterize` | ✔ | ✔ | ✔ | OK | +| Layer → Delete | "delete the layer" | `layers.delete` | ✔ | ✔ | partial | OK | +| Layer → Merge Down | "merge down" | `layers.mergeDown` | ✔ | ✘ | ✘ | **Server gap** | +| Layer → Merge All / Flatten | "flatten the image" | `layers.flatten` | ✔ | ✘ | ✘ | **Server gap** | +| Layer → Select All | "select all layers" | (multi-select) | ✔ | ✔ | ✔ | OK (fast path selects every layer) | +| Layer → Combine (unite/subtract/…) | "combine the shapes" | `_combineSelectedShapes` | ✔ | ✔ | ✔ | OK (fast path selects the shape layers and combines) | +| Layer → Fill | "fill the layer with blue" | `selection.fill` / flood | partial | ✘ | ✘ | **Gap** | +| Layer → Layer Mask (reveal/hide/apply/invert/feather) | "add a layer mask" | `layers.addMaskFromSelection` | ✔ | ✔ | ✔ | OK | +| Layer → Opacity | "set the layer opacity to 50%" | `layers.setOpacity` | ✔ | ✔ | partial | OK (fast path sets the active layer's opacity) | +| Layer → Blend Mode | "set the blend mode to multiply" | `layers.setBlend` | ✔ | ✔ | partial | OK (fast path sets the active layer's blend mode) | +| Type → Add | "add a title" | `text.add` | ✔ | ✔ | ✔ | OK | +| Type → Character | "make the text bigger" | `text.edit` | ✔ | ✘ | ✘ | **Server gap + planner gap** | +| Type → Paragraph | "centre the text" | `text.layout` | ✔ | ✔ | partial | OK | +| Type → Warp | "warp the text" | `_rerenderTextLayer` (warp) | ✔ | ✔ | ✔ | OK | +| Select → All/Deselect/Invert | "invert the selection" | `selection.*` | ✔ | ✔ | ✔ | OK | +| Select → Reselect | "reselect" | `_reselectSelection` | ✔ | ✔ | ✔ | OK | +| Select → Transform | "transform the selection" | `transform.start` | ✔ | ✔ | partial | OK | +| Select → Refine | "refine the edge" | `selection.feather/expand` | ✔ | ✘ | ✘ | **Server gap** | +| Select → Color Range | "select the blue sky" | `selection.colorRange` | ✔ | ✔ | partial | OK | +| Select → Select and Mask | "select the subject" | `selection.subject` | ✔ | ✔ | ✔ | OK | +| Select → Save | "save the selection" | `_saveNamedSelection` | ✔ | ✔ | ✔ | OK | +| Select → Load | "load the selection as sky" | `_loadNamedSelection` | ✔ | ✔ | ✔ | OK | +| Filter → every filter (30+) | "pixelate the image" | `filters.apply` | ✔ | ✔ | ✔ | OK (fast path covers pixelate/emboss/twirl/pinch/spherize/ripple/solarize/find-edges/crystallize/noise/despeckle/unsharp + blur/sharpen/vignette) | +| Filter → Blur → Motion / Zoom | "motion blur" / "zoom blur" | `filters.apply` (blur-motion / blur-zoom) | ✔ | ✔ | ✔ | OK (fast path maps motion blur and zoom blur to their Filter-menu variants) | +| Filter → Layer effects (drop shadow, stroke, colour overlay, outer glow) | "add a drop shadow" | `layers.addStyle` | ✔ | ✔ | ✔ | OK (fast path adds drop shadow / stroke / outer glow / colour & gradient overlay; colour parsed from the prompt) | +| View → Rulers/Grid/Snap/Clear guides | "show the grid" | `#ge-view-menu` | ✔ | ✔ | ✔ | OK | +| Window → panels | — | n/a | n/a | n/a | ✘ | UI-only | +| 3D / Plugins | — | — | ✘ | ✘ | ✘ | Not implemented (disabled in UI) | + +## 2. Tools + +Toolbar tools are interactive, not bridge ops, so coverage is about whether the +**effect** of a tool is reachable by prompt. + +| Tool | NL-reachable? | Notes | +|---|---|---| +| Move / Transform | `transform.*` | OK (transform.start/apply/cancel) | +| Hand / Zoom | — | View only | +| Crop | `document.crop` + format guard | OK | +| Text / Shape | `text.add`, `text.layout` | Shape has no add-shape op — **gap** | +| Brush / Eraser / Dodge / Burn / Smudge / Clone / Heal / Sharpen | — | Interactive painting; no NL op — **gap** (out of scope for planner) | +| Gradient | — | **No op** | +| **Paint Bucket** | `fill.flood` | **Server gap + planner gap** (op exists client-side only) | +| Eyedropper | — | UI only | +| Marquee / Lasso / Wand / Quick Select / Object Select / Pen / Path | `selection.rectangle/ellipse/polygon/subject/…` | OK for region/reference selects; lasso/pen paths have `selection.polygon` only | +| SAM / Inpaint / Rembg | `selection.subject`, `ai.removeObject`, `ai.removeBackground` | OK | +| **Pixel brush + grid** | — | UI state only; no op — **gap** (acceptable: it is a brush mode, not an edit op) | + +## 3. Right-click context menu + +| Item | Backing op | Reachable by prompt? | +|---|---|---| +| Brush size / preset / Pixel Brush | `state` UI | No (interactive) | +| Select All / Deselect / Inverse / Feather / Expand / Contract / Border / Smooth | `selection.*` | Yes | +| Transform Selection / Refine Edge | `transform.start`, `selection.feather` | Partial | +| Apply / Cancel Crop | `document.crop` | Yes | +| Commit / Cancel / Free Transform | `transform.apply/cancel/start` | Yes | +| Add Text Layer / Rasterize Text | `text.add`, `layers.rasterize` | Yes (rasterize OK) | +| Copy Color Hex / Swap Colors | `state.color` | OK (fast path swaps / resets the foreground & background swatches) | +| Fill Selection w/ Foreground | `selection.fill` | OK (fast path fills the selection/canvas with the foreground or background colour) | +| Cut / Copy / Paste | `selection.copy/paste` | **Gap** (no planner intent) | +| Rotate 90/180, Flip H/V | `layer.rotate/flip` | **Server+planner gap** (same as Edit menu) | +| Undo / Redo | `history.*` | Yes | + +## 4. Adjustments + +`adjustments.apply` / `adjustments.applyMasked` cover all adjustment types +(brightness-contrast, white-balance, shadows-highlights, vibrance, clarity, +vignette, black-white, …). The style guide emits several. **Status: OK** +(planner covers the common ones; the long tail is reachable only via explicit +wording). + +## 5. Filters + +`filters.apply` / `filters.applyMasked` run every filter the UI lists. The +planner only emits **sharpen**, **blur-gaussian**, and **vignette** via intents. +**Gap (planner):** no intent for noise, pixelate, stylize, distort, render, +unsharp, sharpen-edges, colour-overlay/drop-shadow/stroke. +**Gap (HARNESS):** `filters.apply`/`applyMasked` are in the server catalog, so +the fix is style-guide intents, not catalog. + +## 6. Layer styles + +`layers.addStyle {type, params}` exists in both catalogs. **Planner never emits +it**, so "add a drop shadow to the title", "add a stroke", "colour overlay" fail. +**Gap (planner).** + +## 7. Selection ops + +`selection.rectangle/ellipse/polygon/fill/delete/copy/paste/feather/expand/ +contract/selectAll/deselect/invert/colorRange/subject/samText/face/sky/ +background/hasMask/maskBounds/exportMask/regionLuminance`. **Status: mostly OK.** +Missing: reselect, save-selection. + +## 8. Document ops + +| Capability | Op | C | S | P | +|---|---|---|---|---| +| Crop | `document.crop` | ✔ | ✔ | ✔ | +| Resize canvas / image | `document.resizeCanvas` | ✔ | ✔ | partial | +| **Extend canvas** | `document.extend` | ✔ | ✘ | ✘ | +| Save PSD | `document.savePsd` | ✔ | ✔ | ✔ | +| Export layers | `document.exportLayers` | ✔ | ✘ | ✘ | +| Duplicate / Create / Switch / List | `document.*` | ✔ | ✘ | partial | + +## 9. AI edit & generate modes + +| Mode | Op | Notes | +|---|---|---| +| Generative fill | `ai.generativeFill` | OK | +| Remove object | `ai.removeObject` | OK | +| Harmonize | `ai.harmonize` | OK | +| Upscale | `ai.generativeUpscale` | OK | +| Generate image | `ai.generateImage` / server `generate_image` | OK | +| Expand | `ai.generativeExpand` | OK (content-bounds prefill, SW v760) | +| Remove background | `ai.removeBackground` | OK | +| Instruction edit / scene replace | server `ai_edit_image` | OK (HARNESS) | + +--- + +## 10. Gap list + +### Client-side (fix in this repo — PHOTOSHOP-owned) — **ALL FIXED** +1. ✅ **`fill.flood` not in the NL path** — added to `nl-edit.js OP_CATALOG` + + `localPlan` fill intent ("fill … with "; falls back to `selection.fill` + when a region is selected). +2. ✅ **`layer.flip` / `layer.rotate` not in the NL path** — added to + `nl-edit.js OP_CATALOG` + "this layer" intent (whole-image phrasing routes to + `document.flip`/`document.rotate`). +3. ✅ **`text.edit` not in the NL path** — added catalog entry + intent for font + size/colour ("make the text bigger/smaller", "make it red"). +4. ✅ **`document.extend` not in the NL path** — added catalog entry + intent for + "extend the canvas to 16:9" (derives target size from the document). +5. ✅ **`layers.align`** — added planner intent ("align … to the centre/left/…"). +6. ✅ **`layers.addStyle`** — added planner trigger ("drop shadow", "stroke", + "colour overlay"). +7. ✅ **Whole-image rotate/flip has no control op** — added `document.rotate` / + `document.flip` control ops (`galleryEditor.js`) and catalog/intent entries. +8. ✅ **`selection.copy/paste` intents** — added "copy the selection"/"paste". + +### Server-side (file for HARNESS in the passover) +1. `routes/editor_agent_routes.py OP_CATALOG` is missing ops the client runs: + `layer.flip`, `layer.rotate`, `text.edit`, `fill.flood`, `layers.align`, + `layers.rasterize`, `layers.mergeDown`, `layers.flatten`, `document.extend`, + `document.exportLayers`, `document.*` (list/duplicate/create/switch/openPsd), + `mask.*`, `selection.*` (polygon/feather/expand/contract/copy/paste/hasMask/ + face), `transform.*` are partly present. +2. `src/editor_style_guide.py` has no intents for: layer transform + (rotate/flip active layer), whole-document rotate/flip, text editing, align, + layer styles (drop shadow/stroke/overlay), flood fill, copy/paste, noise/ + pixelate/stylize/distort/render filters. + +## 11. Coverage summary + +Counting distinct user-facing capabilities in the toolbar + 12 menus + right-click +(~150 including the filter/effect long tail): + +- **Have a client op:** ~85% +- **Client op also in server OP_CATALOG:** ~75% +- **Planner can emit for natural phrasing:** ~72% (was ~55%; client-side + planner intents for fill, layer/whole-image transform, text edit, align, + layer styles, copy/paste and extend are now implemented) +- **Usable from Quick Edit end-to-end today:** **45/45 prompts (100%)** measured + live on build **0.20.45** with `tests/e2e/photo-editor/quickedit-coverage.js` + (`/tmp/opencode/qe-coverage45/quickedit-coverage.json`). HARNESS build 0.20.45 + (`4a6932a2`) added the 12 missing structural ops to the server `OP_CATALOG` + + style guide v5, so every audited prompt now plans to the exact op. + +The gap was dominated by the **planner**, not by missing client functionality; +that planner gap is now closed end-to-end (30/45 on 0.20.44 -> 45/45 on 0.20.45). + +## 12. Test prompts (~40) + +See `docs/QUICKEDIT_PROMPTS.md` and the runner +`tests/e2e/photo-editor/quickedit-coverage.js` for per-prompt pass/fail. + +### Live run (build 0.20.45) — 45/45 PASS + +All 45 prompts plan to the exact expected op. HARNESS 0.20.45 routes these +structural ops server-side: `fill.flood`, `text.edit`, `layers.align`, +`layer.rotate`/`layer.flip`, `document.rotate`/`document.flip`, +`selection.copy`/`selection.paste`, `layers.flatten`, `layers.mergeDown` (plus +the previously passing set). Results: +`/tmp/opencode/qe-coverage45/quickedit-coverage.json`. + +### Previous run (build 0.20.44) — 30/45 PASS + +Kept for provenance. Failures were all the server planner emitting a generic +`adjustments.apply`/`transform.apply` or a wrong op; ids 1, 2, 3, 4, 8, 9, 10, +11, 12, 13, 14, 19, 20, 40, 41. All now fixed in 0.20.45. + + +## Provenance + +Recon inventory captured from `static/js/editor/build/topbar.js`, +`static/js/editor/build/toolbar.js`, `static/js/editor/canvas-context-menu.js`, +`static/js/galleryEditor.js` (`installEditorControl`), `static/js/editor/nl-edit.js`, +`routes/editor_agent_routes.py`, and `src/editor_style_guide.py`. diff --git a/docs/searxng-trust-image.md b/docs/searxng-trust-image.md new file mode 100644 index 000000000..e0f406e82 --- /dev/null +++ b/docs/searxng-trust-image.md @@ -0,0 +1,39 @@ +# Upstream SearXNG trust validation + +SearXNG owns independent system and Python certificate stores. Updating Odysseus's Python environment does not update this sidecar. The base and both standalone GPU Compose files now select the official `docker.io/searxng/searxng:2026.10.24-ef0ec03da` image at immutable multiarch digest `sha256:7c7cbe509429e82755104b2a95f52085a98633c95d53f44e4e905fa3a59f08c8`. Its system and certifi stores remove the affected eMudhra trust. The [upstream certifi update](https://github.com/searxng/searxng/pull/6881) is included in this release. + +Deploy the upstream image directly. The validation Dockerfile adds public fixtures and diagnostics only; no repair layer, custom runtime image or project registry publisher is required. `scripts/check_ca_trust.py` owns the known-certificate inventory for both the application and SearXNG checks. + +## Validate a pin update + +Run from the repository root on a disposable Docker host. The helper uses a fresh configuration volume, omits published ports and does not load the operator's `.env`. It retains the base Compose settings wrapper, healthcheck and capabilities. Outbound network access and working search engines are required for the live checks. + +```sh +image=$(docker compose --env-file /dev/null -f docker-compose.yml config --format json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["services"]["searxng"]["image"])') + +# Offline build-step checks, including actual entrypoint and loopback health. +# Pulling the selected base still requires registry access. +docker build --network=none --target validation \ + --build-arg "SEARXNG_IMAGE=$image" -f docker/searxng/Dockerfile . + +# Exercise the exact unmodified deployment image after an anonymous pull. +docker pull "$image" +bash .github/scripts/validate-searxng-image.sh "$image" +``` + +The offline checks require all fourteen incident chains to fail certificate verification through the system PEM bundle, hashed CA directory and certifi. The affected positive-control roots must accept those same chains. Each fixture's DER fingerprint is verified first, and a fixed verification timestamp prevents expiry from hiding a broken mitigation. Checks run again after the real upstream root entrypoint regenerates trust. + +The live checks use SearXNG's actual curl client factory. Ordinary verified HTTPS must succeed, all fourteen incident hosts must fail with curl certificate error 60, and a JSON search must return results. DNS failures, timeouts and generic handshake failures are inconclusive and fail validation. The checks inspect the already-running service without changing its trust stores. + +The `ci / SearXNG trust` workflow runs these offline and Compose/live checks on native amd64 and arm64 runners. It selects the same immutable image as base Compose, pulls without registry credentials, and has read-only repository permissions. It neither publishes a replacement image nor requires package-write credentials. Update all three deployment pins together; the alignment regression guards the standalone GPU files. + +## Rollout and scope + +Changing a Compose pin does not replace running containers. After updating the checkout, run `docker compose pull searxng` and `docker compose up -d --no-deps searxng`, then inspect health, retained settings and actual Odysseus search integration. Use the appropriate standalone GPU Compose file when that is the deployment being managed. + +Operator-mounted trust, custom engines, persisted packages and independently managed clients need separate effective-client checks. An amd64 result does not establish arm64 behavior. The fixture-based check detects known certificate fingerprints; it does not model every native revocation mechanism or arbitrary reissued certificates sharing a compromised key. See [outbound TLS trust](tls-trust.md) for the application, bootstrap, Chromium and other runtime boundaries. + +## Public fixture provenance + +The public CA inventory comes from the incident's [roots](https://emudhra.we-have-a-problem.com/roots.json) and [intermediates](https://emudhra.we-have-a-problem.com/impacted-intermediates.json). The additional cross-signed issuer and fourteen chain fixtures come from [chains.zip](https://emudhra.we-have-a-problem.com/chains.zip), SHA-256 `7b949956955fc501a30f4d98d2f513d34c0bbbe92e29a968194c96e097ce2345`. Fixtures contain public certificates only. See [the fixture notes](../tests/fixtures/ca-trust/README.md). diff --git a/docs/tls-trust.md b/docs/tls-trust.md new file mode 100644 index 000000000..71f0fe76e --- /dev/null +++ b/docs/tls-trust.md @@ -0,0 +1,75 @@ +# Outbound HTTPS trust + +Odysseus uses both Python package CA bundles and system trust. HTTPX uses certifi by default; the guarded web/PDF fetcher and webhook delivery explicitly use Python's system SSL context. A configured `LLM_CA_BUNDLE` adds certificates to system trust for model calls. Upgrading certifi alone cannot repair every outbound HTTPS path. + +For the October 2026 eMudhra incident, use certifi 2026.10.23 or newer, pip 26.2.2 or newer, and uv 0.13.5 or newer wherever uv is used. Pip has its own vendored certifi and also loads system roots. Update the operating system's certificate package separately. See the [Python security announcement](https://discuss.python.org/t/109576) and [incident details](https://emudhra.we-have-a-problem.com/). + +Before other pip installs, bootstrap the selected environment with the integrity-pinned wheels: + +```sh +python -m pip install --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +python -m pip install -r requirements.txt +``` + +The pinned bootstrap protects package integrity even when the old installer's certificate bundle is affected. It is not an operating-system trust-store update. Native Windows/macOS trust and command-line OpenSSL PEM stores may differ; follow the platform vendor's remediation for every store in use. Restart Odysseus and its subprocesses after updating trust. + +On Linux/OpenSSL deployments, inspect both the effective CA file and hashed CA directory, plus the two Python bundles: + +```sh +python scripts/check_ca_trust.py --system --python --overrides --verbose +``` + +Explicit PEM bundles can also be passed as positional arguments. The check rejects the four published root fingerprints, all sixteen published intermediate certificates (including their cross-signs), the cross-signed G3 issuer in the published test chains, and malformed/empty bundles. Checking intermediates matters because adding one as a custom trust anchor can restore an affected chain even after its roots have been removed. + +`--overrides` checks every nonempty `LLM_CA_BUNDLE`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `PIP_CERT`, `NODE_EXTRA_CA_CERTS` and `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` setting in the checker's process environment, including configured fallback settings. Requests/pip hashed-directory overrides are inspected too; `--system` covers `SSL_CERT_DIR`. Run with the application's environment and filesystem access. The script does not load `.env` or application configuration files; pass any CA paths configured only there explicitly, for example `python scripts/check_ca_trust.py --system --python /path/to/private-ca.pem`. Missing or invalid configured files fail the check. Selecting only `--overrides` with no configured paths also fails instead of reporting that stores were validated. + +`--verbose` reports the Python executable/prefix, user home, effective UID/GID where available, and resolved CA paths, including bundles selected from user-installed packages. Compare these with the running application's identity and interpreter. It does not dump environment variables or credentials. + +This is an offline inventory check of known certificate fingerprints. It does not modify stores, reject requests at runtime, model native OS revocation, or detect arbitrary reissued certificates sharing the compromised keys. Chromium's policy separately matches public keys. Check mounted or replaced CA files again after deployment, remove affected trust through the owning platform/client configuration, and restart clients that cache SSL contexts. A passing build cannot validate custom files introduced later. + +Both Docker build stages use Python 3.14.9, refresh system CA certificates and bootstrap pip/certifi before dependency installation. The build fails if known affected CA certificates remain. Node uses the corrected system store through `--use-openssl-ca`. Python gRPC uses that same PEM file through `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH`; updating certifi alone does not replace gRPC's independent wheel bundle. After dependency installation, the image checks gRPC's selected file as well. An explicit gRPC override must be readable and free of known affected CA certificates. [grpcio 1.84.1](https://github.com/grpc/grpc/releases/tag/v1.84.1) includes the upstream root-bundle correction and is the application minimum. Retain the checked override as a consistent image trust selection; older persisted Python environments and explicit client roots still need independent checks. + +The image also bundles Docker CLI [29.9.2](https://github.com/moby/moby/releases/tag/docker-v29.9.2), built with Go 1.26.10's distrust of the compromised intermediate keys. Cookbook uses this client for optional Docker operations. Default Compose does not mount the host Docker socket; local daemon access requires the explicit [host-Docker overlay](../docker/host-docker.yml), whose usual Unix-socket connection does not use TLS. The client update is maintenance of a shipped tool, not evidence that these local calls were exposed. It does not update the host daemon or other containers' clients and trust stores; assess those separately. + +Chromium uses a separate trust store. The Docker image installs a managed `CADistrustedCertificates` policy for the eight compromised intermediate public keys, covering their cross-signs without depending on background revocation updates. It adds no trusted certificate and does not disable certificate verification. The policy and public-certificate provenance are documented in [the browser policy note](../docker/chromium-ca-policy.md). Native browser installations and independently managed profiles require their own platform policy or vendor update; this image policy does not configure host browsers. + +To replace only Odysseus in an already running Compose stack, build from the corrected checkout with a distinct local image tag. Use the same Compose files and project name as the existing deployment for every command below: + +```sh +export ODYSSEUS_IMAGE="odysseus-local:ca-$(git rev-parse --short=12 HEAD)" +docker compose build --pull --no-cache odysseus +docker compose up -d --no-deps --pull never --no-build odysseus +docker image inspect --format '{{.Id}}' "$ODYSSEUS_IMAGE" +docker inspect --format '{{.Image}}' "$(docker compose ps -q odysseus)" +``` + +The two image IDs must match. Keep the selected `ODYSSEUS_IMAGE` in the deployment configuration for subsequent recreations. Compose can prefer the registry image when both `image` and `build` are present; the explicit pull/build flags keep this replacement on the local build. `--no-deps` leaves the existing sidecars running and is intended for an existing stack. For registry deployments, select a verified corrected immutable tag or digest. A merge to `dev` publishes `:dev`; it does not update `:latest`, which is published from `main`. + +Check the standard image as its configured app account, using the image's Python interpreter and the same user-bin PATH prefix as the entrypoint: + +```sh +docker compose exec -T --user 0 odysseus sh -ec ' + gosu_bin=$(command -v gosu) + app_user=$(getent passwd "${PUID:-1000}" | cut -d: -f1) + test -n "$app_user" + exec "$gosu_bin" "$app_user" env PATH="/app/.local/bin:$PATH" \ + /usr/local/bin/python /app/scripts/check_ca_trust.py \ + --system --python --grpc --overrides --verbose +' +``` + +`gosu` selects the account's HOME and drops privileges before Python runs, so the check includes its persisted user packages. The entrypoint itself is not rerun. Plain `docker compose exec ... python` uses the image's default root user and can miss `/app/.local` packages. A custom service user, entrypoint, virtual environment or launch command requires matching that deployment's actual user, HOME and Python interpreter instead of copying the standard command unchanged. Docker exec inherits container-creation environment, not variables exported later by the app or entrypoint: reproduce any additional Python/TLS settings with exec's `--env` options, and pass CA paths configured only in application files explicitly. Confirm the verbose identity and selected paths before accepting the result. + +For containers started non-root or without the SETUID/SETGID capabilities needed by `gosu`, use `docker compose exec --user UID:GID --env HOME=/actual/app/home` with the deployment's actual numeric IDs, home, PATH and interpreter instead. Do not add capabilities merely to run this check. + +SearXNG is a separate HTTPS client and image; updating the Odysseus image does not patch it. Inspect its effective CA path and update or rebuild that image independently. Also inspect persisted `/app/.local` packages, remote Cookbook environments, and externally configured MCP runtimes. Image replacement does not necessarily replace those environments. + +For behavioral validation, all fourteen [published test chains](https://emudhra.we-have-a-problem.com/chains.zip), including the AAA cross-signs and two chains without Certificate Transparency entries, must fail certificate verification through the effective stores. Confirm that an ordinary known-good HTTPS request still succeeds. In Chromium, confirm the managed policy is loaded at `chrome://policy` and test the incident endpoints through the actual browser path with certificate verification enabled. The PEM checker cannot inspect Chromium's policy or its active verifier. Package versions and a successful image build alone do not prove that a running deployment selects the corrected store. + +## Follow-up releases + +The maintained bootstrap selects certifi 2026.10.23 with its verified wheel hash; pip remains 26.2.2. The certifi update removes the C1/C3 roots in addition to the earlier incident changes. CI uses uv 0.13.5, which adds verifier-level rejection of the compromised intermediate keys and updated managed CPython builds. Existing virtual environments do not update automatically. See the [certifi change](https://github.com/certifi/python-certifi/commit/b60466c9f2882d9cf088943c650b0faac06c080c), [uv release](https://github.com/astral-sh/uv/releases/tag/0.13.5), and [Python security releases](https://blog.python.org/2026/10/python-3151-3149-31317/). The pinned bootstrap still replaces the selected versions exactly; preserving newer installed versions and private-mirror support remain separate maintenance. + +SearXNG remains an independent trust owner. Use the [upstream-image validation and rollout instructions](searxng-trust-image.md); updating these application packages does not repair an existing sidecar. Host daemons, native browser policy, optional compiled clients and independently managed integrations require their own effective-client checks. + +CPython's [intermediate-key verifier block](https://github.com/python/cpython/pull/160042) and its [3.14 backport](https://github.com/python/cpython/pull/160569) merged on October 24. They are distinct from Python 3.14.9's bundled-pip update. No released 3.14 interpreter containing that verifier change was identified when this image pin was selected; retain the corrected effective trust stores and track the subsequent security release. hf-xet's [trust fix](https://github.com/huggingface/xet-core/pull/1038) is also merged but not included in the latest published 1.7.0 release at that check. Agent-browser 0.39.2 and primp 2.0.1 are not established incident fixes for their independent native trust paths. A version bump alone does not establish that the active client uses corrected trust. diff --git a/frontend/src/features/notes/ui/cardInteractions.js b/frontend/src/features/notes/ui/cardInteractions.js index 09e083c41..4160e5c1b 100644 --- a/frontend/src/features/notes/ui/cardInteractions.js +++ b/frontend/src/features/notes/ui/cardInteractions.js @@ -112,7 +112,7 @@ export function createCardInteractions({ if (state.selectMode) { body.querySelectorAll('.note-card').forEach(card => { card.addEventListener('click', (e) => { - if (e.target.closest('.note-card-cb')) return; // checkbox handles itself + if (e.target.closest('.note-card-cb, a')) return; // links open without toggling selection e.stopPropagation(); tapToEditOrSelect(card); }); diff --git a/frontend/src/features/notes/ui/editor.js b/frontend/src/features/notes/ui/editor.js index f5bdc7c9f..678b08804 100644 --- a/frontend/src/features/notes/ui/editor.js +++ b/frontend/src/features/notes/ui/editor.js @@ -51,7 +51,7 @@ export function createEditor({ - ${currentImageUrl && type !== 'draw' ? `
` : ''} + ${currentImageUrl && type !== 'draw' ? `
` : ''}
${type === 'note' ? `` @@ -297,7 +297,7 @@ export function createEditor({ form.querySelector('.note-form-image-wrap')?.remove(); const wrap = document.createElement('div'); wrap.className = 'note-form-image-wrap'; - wrap.innerHTML = ``; + wrap.innerHTML = ``; // Insert AFTER the whole header (a flex-row), not after the // title input itself : otherwise the image lands as a sibling // of the title inside the header and flex puts them side-by-side. diff --git a/frontend/src/features/notes/ui/text.js b/frontend/src/features/notes/ui/text.js index 75ecbaf78..bb71cb690 100644 --- a/frontend/src/features/notes/ui/text.js +++ b/frontend/src/features/notes/ui/text.js @@ -31,7 +31,7 @@ export function createText({ url = url.slice(0, -1); } const href = url.startsWith('www.') ? `https://${url}` : url; - return `${url}` + (url !== m ? m.slice(url.length) : ''); + return `${url}` + (url !== m ? m.slice(url.length) : ''); }); } diff --git a/integrations/claude/skills/odysseus/SKILL.md b/integrations/claude/skills/odysseus/SKILL.md index 31b40ee01..c8bd94db3 100644 --- a/integrations/claude/skills/odysseus/SKILL.md +++ b/integrations/claude/skills/odysseus/SKILL.md @@ -100,6 +100,14 @@ python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory - `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. - `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. +## Chat history + +- `GET /api/codex/chat/{session_id}?offset=0&limit=100` — read a paginated conversation transcript owned by the token owner. Requires the existing `chat` scope. + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py GET /api/codex/chat/SESSION_ID +``` + ## Email draft + send - Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. diff --git a/integrations/claude/skills/odysseus/scripts/odysseus_api.py b/integrations/claude/skills/odysseus/scripts/odysseus_api.py index 8a22eb494..54df59fa1 100755 --- a/integrations/claude/skills/odysseus/scripts/odysseus_api.py +++ b/integrations/claude/skills/odysseus/scripts/odysseus_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Small Odysseus scoped API helper for Codex terminal sessions.""" +"""Small Odysseus scoped API helper for Claude Code terminal sessions.""" from __future__ import annotations @@ -44,7 +44,7 @@ def _config() -> tuple[str, str] | None: if not token: missing.append("ODYSSEUS_API_TOKEN") if missing: - print(f"missing {', '.join(missing)}; create a Codex Agent token in Odysseus Settings", file=sys.stderr) + print(f"missing {', '.join(missing)}; create a Claude Agent token in Odysseus Settings", file=sys.stderr) return None return base_url, token diff --git a/integrations/codex/skills/odysseus/SKILL.md b/integrations/codex/skills/odysseus/SKILL.md index d4cbdf726..a939cac33 100644 --- a/integrations/codex/skills/odysseus/SKILL.md +++ b/integrations/codex/skills/odysseus/SKILL.md @@ -100,6 +100,14 @@ python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"tex - `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. - `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. +## Chat history + +- `GET /api/codex/chat/{session_id}?offset=0&limit=100` — read a paginated conversation transcript owned by the token owner. Requires the existing `chat` scope. + +```bash +python3 integrations/codex/scripts/odysseus_api.py GET /api/codex/chat/SESSION_ID +``` + ## Email draft + send - Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. diff --git a/launch-windows.ps1 b/launch-windows.ps1 index ab0e3542b..31c45a5fb 100644 --- a/launch-windows.ps1 +++ b/launch-windows.ps1 @@ -131,7 +131,8 @@ if (-not (Test-Path $venvPy)) { # 3. Install / update dependencies Write-Step "Installing dependencies (first run can take a few minutes)" -& $venvPy -m pip install --upgrade pip --quiet +& $venvPy -m pip install --quiet --require-hashes --only-binary=:all: --no-deps -r requirements-bootstrap.txt +if ($LASTEXITCODE -ne 0) { Fail "TLS dependency bootstrap failed." } & $venvPy -m pip install -r requirements.txt if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed. Scroll up for the pip error." } diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index ed8d5acff..b43eeca87 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -1185,6 +1185,9 @@ def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict: "subject": subject, "from": sender_name or sender_addr or sender, "from_address": sender_addr, + "to": str(row.get("to") or headers.get("To") or ""), + "cc": str(row.get("cc") or headers.get("Cc") or ""), + "references": str(row.get("references") or headers.get("References") or ""), "date": date_str, "date_epoch": date_epoch, "summary": body[:240], @@ -1557,6 +1560,7 @@ def _fixture_update_email(uid=None, source_folder="INBOX", account=None, **updat if not isinstance(rows, list): return False owner = _current_owner() + matches = [] for index, row in enumerate(rows, start=1): if not isinstance(row, dict): continue @@ -1568,14 +1572,19 @@ def _fixture_update_email(uid=None, source_folder="INBOX", account=None, **updat continue if not _fixture_folder_matches(row.get("folder") or "INBOX", source_folder): continue - for key, value in updates.items(): - if value is None: - row.pop(key, None) - else: - row[key] = value - path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - return True - return False + if not _fixture_row_matches_account(_fixture_email_record(row, index, owner or row_owner), account): + continue + matches.append(row) + if len(matches) != 1: + return False + row = matches[0] + for key, value in updates.items(): + if value is None: + row.pop(key, None) + else: + row[key] = value + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return True # ── Tool implementations ── @@ -2925,7 +2934,9 @@ def _create_email_draft_document( def _draft_reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None, title=None): """Create a threaded Odysseus reply draft document. Does not send.""" - fixture = _fixture_email_action_target(uid=uid, folder=folder, account=account) + fixture = _fixture_read_email(uid=uid, folder=folder, account=account) + if _fixture_email_enabled() and (fixture is None or fixture.get("error")): + return fixture or {"error": "No email fixture is available for the authenticated owner"} if fixture is not None: sender = str(fixture.get("from_address") or fixture.get("from") or "") _, sender_addr = email.utils.parseaddr(sender) @@ -3178,7 +3189,9 @@ def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): def _set_flag(uid, folder, flag, add=True, account=None): """Add or remove an IMAP flag (e.g. \\Seen, \\Answered, \\Deleted).""" - if _fixture_email_action_target(uid=uid, folder=folder, account=account) is not None: + if _fixture_email_enabled(): + if _fixture_email_action_target(uid=uid, folder=folder, account=account) is None: + return False if flag == "\\Seen": return _fixture_update_email(uid=uid, source_folder=folder, account=account, read=bool(add)) if flag == "\\Answered": diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt new file mode 100644 index 000000000..da1d62220 --- /dev/null +++ b/requirements-bootstrap.txt @@ -0,0 +1,8 @@ +# Integrity-pinned bootstrap for the October 2026 eMudhra CA incident. +# Install with --require-hashes --only-binary=:all: --no-deps before other pip work. +# pip vendors its own certifi; updating the application bundle cannot update it. +# Direct universal wheels avoid index resolution through the old trust store. +pip @ https://files.pythonhosted.org/packages/3f/06/3e4420302363fbb71be3c7f35ce042f54bfca330e3e66b3a2e3c4cca579f/pip-26.2.2-py3-none-any.whl \ + --hash=sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70 +certifi @ https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 diff --git a/requirements.txt b/requirements.txt index 7fabaae93..415b4b503 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,13 +3,16 @@ uvicorn python-multipart python-dotenv httpx +# HTTPS clients using certifi must distrust the compromised eMudhra CA paths. +certifi>=2026.10.23 httpcore>=1.0,<2.0 pydantic>=2.13.4 pydantic-settings>=2.14.1 SQLAlchemy pypdf pypdfium2 -Pillow +# Pillow 13 fixes known image-processing advisories; keep local embeddings compatible. +Pillow>=13.0.0 faster-whisper pdfplumber beautifulsoup4 @@ -21,7 +24,9 @@ numpy # chromadb-client is the lightweight HTTP client (talks to a standalone # ChromaDB service); fastembed runs local ONNX embeddings. chromadb-client -fastembed +# gRPC vendors independent roots; 1.84.1 removes the incident trust anchors. +grpcio>=1.84.1 +fastembed>=0.9.1 youtube-transcript-api yt-dlp # Markdown rendering for research reports (src/visual_report.py). diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index eb0bfc310..0d815309e 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -569,8 +569,72 @@ async def auto_name_session(session_manager, sess): if not first_msg: return + from src.owner_identity import is_request_sentinel_owner + owner = getattr(sess, "owner", None) + invalid_owner = ( + not isinstance(owner, str) or not owner.strip() + or owner != owner.strip() or is_request_sentinel_owner(owner) + ) + if invalid_owner: + session_manager.update_session_name(sess.id, fallback_session_title(first_msg)) + return + endpoint_url = str(getattr(sess, "endpoint_url", "") or "") model_name = str(getattr(sess, "model", "") or "") + + # The spec ("Ajax") model names its own chats from the user's first + # prompt. This function only runs after the first response stream has + # settled (auto_name_session_after_stream), so titling does not + # contend with the visible answer. Thinking is disabled and the + # budget kept small so the model returns a bare title. + try: + from src.model_profiles import uses_odysseus_progressive_thinking + _is_spec_model = bool(model_name) and uses_odysseus_progressive_thinking(model_name) + except Exception: + _is_spec_model = False + if _is_spec_model and endpoint_url: + try: + from src.database import SessionLocal + from src.endpoint_resolver import (build_chat_url, build_headers, + resolve_endpoint_runtime, resolve_owner_registered_endpoint) + from src.owner_identity import auth_disabled + + owner = getattr(sess, "owner", None) + if not owner and not auth_disabled(): + raise ValueError("Session has no owner") + db = SessionLocal() + try: + endpoint = resolve_owner_registered_endpoint(db, endpoint_url, owner) + base, key = resolve_endpoint_runtime(endpoint, owner=owner) + title_url = build_chat_url(base) + title_headers = build_headers(key, base) + finally: + db.close() + title = await llm_call_async( + title_url, + model_name, + [ + {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation \u2014 just the title."}, + {"role": "user", "content": first_msg}, + ], + temperature=0.3, + max_tokens=32, + headers=title_headers, + timeout=20, + thinking_mode="off", + ) + except Exception as e: + logger.warning("[auto-name] spec title call failed for %s: %s", sess.id, e) + title = "" + title = str(title or "").strip().strip('"\'').strip() + from src.text_helpers import strip_think + title = strip_think(title, prose=False, prompt_echo=False) + if not title or len(title) >= 80 or "\n" in title: + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named spec session {sess.id}: {title}") + return + if ( "ttft" in model_name.lower() or re.search(r":18\d{3}\b", endpoint_url) @@ -581,6 +645,14 @@ async def auto_name_session(session_manager, sess): return owner = getattr(sess, "owner", None) + from src.owner_identity import auth_disabled + if not owner and not auth_disabled(): + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named unowned session {sess.id} deterministically: {title}") + return + + from src.task_endpoint import resolve_task_endpoint t_url, t_model, t_headers = resolve_task_endpoint( sess.endpoint_url, sess.model, sess.headers, owner=owner ) @@ -588,24 +660,64 @@ async def auto_name_session(session_manager, sess): logger.debug("[auto-name] No model provided, skipping") return + sess_url = getattr(sess, "endpoint_url", None) + if t_url and sess_url and t_url == sess_url: + from src.database import SessionLocal + from src.endpoint_resolver import ( + build_chat_url, + build_headers, + resolve_endpoint, + resolve_endpoint_runtime, + resolve_owner_registered_endpoint, + ) + db = SessionLocal() + try: + endpoint = resolve_owner_registered_endpoint(db, t_url, owner) + base, key = resolve_endpoint_runtime(endpoint, owner=owner) + t_url = build_chat_url(base) + t_headers = build_headers(key, base) + except Exception as e: + logger.debug("[auto-name] session fallback endpoint %s unauthorized for %s: %s", t_url, owner, e) + t_url, t_model, t_headers = None, None, None + for prefix in ("utility", "default"): + try: + u, m, h = resolve_endpoint(prefix, owner=owner) + if u and m: + t_url, t_model, t_headers = u, m, h + break + except Exception: + pass + finally: + db.close() + + if not t_url or not t_model: + title = fallback_session_title(first_msg) + session_manager.update_session_name(sess.id, title) + logger.info(f"Auto-named session {sess.id} deterministically (no authorized model endpoint): {title}") + return + # max_tokens big enough that reasoning models (Minimax M2, # DeepSeek R1, QwQ, etc.) have headroom for … # plus the actual title — 200 used to clip them mid-reasoning # so strip_think left an empty string and no rename happened. # Timeout matches: 60s gives slow local reasoners room to finish. - title = await llm_call_async( - t_url, - t_model, - [ - {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."}, - {"role": "user", "content": first_msg}, - ], - temperature=0.3, - max_tokens=64, - headers=t_headers, - timeout=15, - reasoning_effort=effort_for_call(t_url, t_model, owner, sess), - ) + try: + title = await llm_call_async( + t_url, + t_model, + [ + {"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."}, + {"role": "user", "content": first_msg}, + ], + temperature=0.3, + max_tokens=64, + headers=t_headers, + timeout=15, + reasoning_effort=effort_for_call(t_url, t_model, owner, sess), + ) + except Exception as e: + logger.warning("[auto-name] non-spec title call failed for %s: %s", sess.id, e) + title = "" title = title.strip().strip('"\'').strip() # Strip / blocks (closed, dangling, or stray tags) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index 5a42d3351..c6f306e96 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -27,11 +27,13 @@ from src.llm_core import ( from src.agent_loop import ( stream_agent_loop, _configured_model_tool_surface, + _configured_model_tool_surface_raw, _local_media_needs_browser_render, _looks_like_workspace_coding_request, ) from src.agent_loop import _normalize_ody_qwen_text_artifacts from src import agent_runs +from src import shutdown as _shutdown from src.model_context import estimate_tokens from src.context_compactor import ( apply_compaction_state, @@ -329,6 +331,7 @@ def _is_external_discovery_request(text: str) -> bool: _URL_SCHEME_RE = re.compile(r"https?://", re.I) _PDF_URL_TAIL_RE = re.compile(r"\.pdf\b|/pdf/", re.I) +_EDITOR_SESSION_RE = re.compile(r"[A-Za-z0-9@._:-]{1,128}") def _mentions_pdf_url(value: str) -> bool: @@ -2688,6 +2691,23 @@ def setup_chat_routes( logger.info("chat→agent auto-escalation: explicit private browser workflow") active_doc_id = form_data.get("active_doc_id", "").strip() active_doc_state = form_data.get("active_doc_state", "").strip().casefold() + # Scoped photo_editor agent context: the Quick Edit popup (and any + # image-edit chat turn) may carry the open image-editor session id and a + # canvas snapshot so the editor tool family can drive it in-process. + # The id is quoted into a system message, so accept only an + # identifier; any other value is treated as no open editor. + editor_session = str( + form_data.get("editor_session") + or form_data.get("editor_session_id") + or "" + ).strip() + if not _EDITOR_SESSION_RE.fullmatch(editor_session): + editor_session = "" + photo_editor_requested = ( + str(form_data.get("photo_editor") or "").strip().lower() + in {"1", "true", "yes", "on"} + or bool(editor_session) + ) logger.info( "[doc-inject] chat_mode=%s, active_doc_id=%r, active_doc_state=%r", chat_mode, active_doc_id, active_doc_state, @@ -3270,7 +3290,34 @@ def setup_chat_routes( active_document=bool(active_doc), workspace=bool(workspace), image_attachment=any(str(a.get('mime') or '').startswith('image/') for a in (ctx.preprocessed.attachment_meta or [])), ) if _use_turn_contract else frozenset() - if 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta: + # Scoped photo_editor family: offered ONLY for an explicit editor + # request (Quick Edit popup / editor session) or an image-edit chat + # turn carrying an image. When active the turn offers just the editor + # tools plus ask_user, never the general tool list. + _photo_editor_active = bool(_use_turn_contract) and ( + photo_editor_requested + or ( + "image_editing" in _turn_capabilities + and bool(ctx.preprocessed.attachment_meta) + ) + ) + if _photo_editor_active: + if editor_session: + editor_context = {'role': 'system', 'content': + 'The image editor is open (session ' + editor_session + '). Use run_editor_command, ' + 'plan_edit and get_editor_screenshot to inspect and change the current canvas, and ' + 'ai_edit_image for model-based edits. Apply the user\'s requested edit to the open ' + 'editor and confirm the result with a screenshot.'} + else: + editor_context = {'role': 'system', 'content': + 'The user attached an image but no image editor is open, so the editor tools cannot ' + 'reach the canvas. Do not call editor tools. Reply briefly telling the user to open ' + 'the attached image in the editor (Quick Edit) and repeat the request; the editor ' + 'tools will then apply it.'} + ctx.messages.insert(0, editor_context) + if foreground_policy.enabled: + getattr(ctx, 'route_messages', ctx.messages).insert(0, dict(editor_context)) + elif 'image_editing' in _turn_capabilities and ctx.preprocessed.attachment_meta: image_refs = ['odysseus://attachment/' + str(a['id']) for a in ctx.preprocessed.attachment_meta if a.get('id') and str(a.get('mime') or '').startswith('image/')] if image_refs: @@ -3307,6 +3354,11 @@ def setup_chat_routes( # pasted revision request. This only offers permitted schemas; # it never requires or performs a document mutation. _turn_capabilities = _turn_capabilities | {"documents"} + # Scoped photo_editor family (see above): when active the turn offers + # just the editor tools plus ask_user, never the general tool list. + if _photo_editor_active: + _turn_capabilities = frozenset({"photo_editor"}) + _active_turn_capabilities = _turn_capabilities # Same decision that prepared the turn; it only stamps the contract # inside the agent-contract branch below. _clean_v3_preview = _compact_preview_turn @@ -3474,7 +3526,7 @@ def setup_chat_routes( if not _privs.get("can_use_documents", True): disabled_tools.update({"manage_documents", "create_document", "edit_document", "update_document", "suggest_document"}) if not _privs.get("can_generate_images", True): - disabled_tools.update({"generate_image", "edit_image"}) + disabled_tools.update({"generate_image", "edit_image", "ai_edit_image"}) if not _privs.get("can_manage_memory", True): disabled_tools.update({"manage_memory", "manage_skills"}) if not _privs.get("can_use_research", True): @@ -3556,9 +3608,22 @@ def setup_chat_routes( "document_editor": not { "manage_documents", "create_document", "edit_document", "update_document", }.issubset(disabled_tools), + # A photo-editing chat turn can only reach a canvas when the image + # editor is actually open. The compact runtime uses this to reply + # (open the image in the editor) instead of looping on editor tools + # that have no canvas to drive. + "photo_editor_open": bool(editor_session), } # Capture permission state before schema selection/reconciliation. # Only deterministic request intent supplies grants, never inventory. + # The strict router (spec) surface picks its tools in src/spec_router.py + # AFTER authority is built, so its admission is keyed off the surface. + try: + _spec_surface = _configured_model_tool_surface_raw( + getattr(sess, "endpoint_url", ""), getattr(sess, "model", ""), _user + ) == "spec" + except Exception: + _spec_surface = False _request_authority = request_authority_for_http( request, message, owner=_user, session_id=session, workspace=workspace, history=_turn_history, policy=tool_policy, @@ -3566,9 +3631,9 @@ def setup_chat_routes( active_document=bool(active_doc), image_attachment=any(str(a.get('mime') or '').startswith('image/') for a in (ctx.preprocessed.attachment_meta or [])), - capabilities=({'search_browser'} if ( + capabilities=(({"photo_editor"} if _photo_editor_active else set()) | ({'search_browser'} if ( _explicit_browser_intent or _external_discovery_intent - ) else ()), + ) else set())), ) if exact_tool_approval is not None: from src.agent_runtime.authority import RequestAuthority @@ -3758,7 +3823,9 @@ def setup_chat_routes( # personal manager; core shell/Web tools are not task # fallbacks. Other turns retain warm and workspace tools. extra_tools=( - frozenset() + frozenset({"ask_user"}) + if _active_turn_capabilities == frozenset({"photo_editor"}) + else frozenset() if _exact_selected_native_chain or _active_turn_capabilities == frozenset({"tasks"}) or ( _active_turn_capabilities in ( frozenset({"transcription"}), frozenset({"ocr"}), @@ -3960,7 +4027,7 @@ def setup_chat_routes( _heartbeat_counter = 0 _last_progress = {} _sent_avg = False - while True: + while not _shutdown.is_shutting_down(): status = research_handler.get_status(session) if not status or status["status"] != "running": break @@ -4222,8 +4289,6 @@ def setup_chat_routes( # indicator, but don't fold them into the saved # reply (mirrors the rewrite path below). if data.get("thinking"): - if thinking_mode == "off": - continue thinking_response += data["delta"] else: full_response += data["delta"] @@ -4667,8 +4732,6 @@ def setup_chat_routes( # Forward them for the live indicator, but keep # them out of the saved reply (same as chat mode). if data.get("thinking"): - if thinking_mode == "off": - continue thinking_response += data["delta"] else: full_response = _render_state.content @@ -4852,6 +4915,25 @@ def setup_chat_routes( _terminal_saved = True if _saved_id: yield f'data: {json.dumps(_render_state.message_saved(_saved_id))}\n\n' + run_post_response_tasks( + sess, session_manager, session, message, _saveable_no_tool_response, + _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager, + incognito=incognito, compare_mode=compare_mode, + character_name=ctx.preset.character_name, + agent_rounds=_agent_rounds, + agent_tool_calls=_agent_tool_calls, + skills_manager=skills_manager, + owner=_user, + extract_skills=( + user_requested_agent + and not tool_approval_continuation + ), + allow_background_extraction=_post_response_extraction_allowed( + tools_blocked=tool_policy.block_all_tool_calls, + tool_approval_continuation=tool_approval_continuation, + client_runtime_context=client_runtime_context, + ), + ) yield f'data: {json.dumps(_metrics_event)}\n\n' except json.JSONDecodeError: yield chunk @@ -4962,6 +5044,8 @@ def setup_chat_routes( raises before reaching a mode-specific finally block.""" try: async for chunk in stream_with_save(): + if _shutdown.is_shutting_down(): + break yield chunk finally: _active_streams.pop(session, None) @@ -5056,7 +5140,12 @@ def setup_chat_routes( if rec is None: if agent_runs.is_active(session_id): return {"status": "streaming", "detached": True} - raise HTTPException(404, "No active stream for this session") + # No active stream: report an explicit idle state with HTTP 200. + # A 404 here made every session load look like a failed request in + # the browser console. The client distinguishes "no stream" by the + # body (status != "streaming"), and still tolerates a 404 from an + # older server, so this stays backward compatible. + return {"status": "idle"} return rec # ------------------------------------------------------------------ # @@ -5155,6 +5244,8 @@ def setup_chat_routes( max_tokens=0, tools=None, ): + if _shutdown.is_shutting_down(): + break if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) diff --git a/routes/codex_routes.py b/routes/codex_routes.py index f42c6b632..542e44d49 100644 --- a/routes/codex_routes.py +++ b/routes/codex_routes.py @@ -35,6 +35,7 @@ CALENDAR_READ_SCOPES = {"calendar:read", "calendar:write"} CALENDAR_WRITE_SCOPES = {"calendar:write"} DOCS_READ_SCOPES = {"documents:read", "documents:write"} DOCS_WRITE_SCOPES = {"documents:write"} +CHAT_READ_SCOPES = {"chat"} WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"} @@ -161,6 +162,7 @@ def setup_codex_routes( memory_router: APIRouter | None = None, calendar_router: APIRouter | None = None, document_router: APIRouter | None = None, + history_router: APIRouter | None = None, ) -> APIRouter: router = APIRouter(prefix="/api/codex", tags=["codex"]) email_list_endpoint = _find_endpoint(email_router, "GET", "/api/email/list") @@ -174,6 +176,7 @@ def setup_codex_routes( documents_library_endpoint = _find_endpoint(document_router, "GET", "/api/documents/library") documents_get_endpoint = _find_endpoint(document_router, "GET", "/api/document/{doc_id}") documents_create_endpoint = _find_endpoint(document_router, "POST", "/api/document") + chat_history_endpoint = _find_endpoint(history_router, "GET", "/api/history/{session_id}") @router.get("/capabilities") def capabilities(request: Request): @@ -219,6 +222,11 @@ def setup_codex_routes( "launch": scoped(COOKBOOK_LAUNCH_SCOPES), "actions": ["tasks", "servers", "output", "serve", "stop"], }, + "chat": { + "read": scoped(CHAT_READ_SCOPES), + "actions": ["read"], + "available": chat_history_endpoint is not None, + }, }, "safety": { "email_send_requires_confirmation": True, @@ -226,6 +234,33 @@ def setup_codex_routes( }, } + @router.get("/chat/{session_id}") + async def read_chat( + request: Request, + session_id: str, + offset: int = 0, + limit: int = 100, + ): + _scope_owner(request, CHAT_READ_SCOPES) + if chat_history_endpoint is None: + raise HTTPException(503, "chat history unavailable") + offset, limit = _clamp_pagination( + offset, + limit, + default_limit=100, + max_limit=100, + ) + result = await chat_history_endpoint(request, session_id, limit, offset) + return { + "session_id": session_id, + "name": result["name"], + "model": result["model"], + "messages": result["history"], + "total": result["total"], + "offset": result["offset"], + "limit": result["limit"], + } + @router.get("/plugin.zip") def plugin_zip(request: Request): require_authenticated_request(request) diff --git a/routes/document/document_routes.py b/routes/document/document_routes.py index 9a7743e8a..2b987e140 100644 --- a/routes/document/document_routes.py +++ b/routes/document/document_routes.py @@ -1185,7 +1185,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: async def ai_tidy_documents(request: Request) -> Dict[str, Any]: """Use AI to judge if documents are junk/test/accidental, then delete them. Caches verdicts so previously-reviewed docs are skipped.""" - from src.task_endpoint import resolve_task_endpoint + from src.task_endpoint import resolve_task_endpoint, task_llm_call_async from src.endpoint_resolver import resolve_endpoint from src.llm_core import llm_call_async @@ -1228,13 +1228,13 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: + "\n".join(doc_list) ) - response = await llm_call_async( - url, model, + response = await task_llm_call_async( [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, {"role": "user", "content": prompt}], + fallback_url=url, fallback_model=model, fallback_headers=headers, + owner=user or None, temperature=0.1, max_tokens=200, - headers=headers, timeout=30, ) diff --git a/routes/editor_agent_routes.py b/routes/editor_agent_routes.py new file mode 100644 index 000000000..717c354e9 --- /dev/null +++ b/routes/editor_agent_routes.py @@ -0,0 +1,2018 @@ +"""Editor agent routes: natural-language planning for the image editor. + +Provides: + POST /api/editor/plan -> {reply, planner, steps:[{op,params}], done?} + POST /api/editor/bridge/command -> enqueue validated ops for the open editor + GET /api/editor/bridge/events -> SSE stream of queued commands (browser subscriber) + POST /api/editor/bridge/result -> browser posts results/screenshots back + POST /api/editor/bridge/await -> wait for a queued command's result + +The browser editor (static/js/editor/nl-edit.js) posts {prompt, state, mode, screenshot} +and consumes {reply, planner, steps} or {done:true}. When mode == "check" and the model +reports completion, the route returns {"done": true} to stop the client's self-check loop. + +The planner is wired to the app's configured chat model (vision-capable when a screenshot +is supplied) via set_editor_planner(); a deterministic keyword planner is the fallback so +the editor always gets a usable plan. +""" +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import math +import re +import uuid +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from src import editor_style_guide as style_guide +from src import editor_agent_model as editor_agent_model +from src import shutdown as _shutdown + +logger = logging.getLogger(__name__) + +# ── Op catalog (kept in sync with static/js/editor/editor-control.js) ── +OP_CATALOG = [ + "document.info", "document.importImage", + "document.crop", "document.resizeCanvas", "document.savePsd", + "layers.list", "layers.select", "layers.addRaster", "layers.placeImage", "layers.addAdjustment", + "layers.duplicate", "layers.delete", "layers.setOpacity", "layers.setBlend", + "layers.group", "layers.ungroup", "layers.addStyle", + "ai.generativeExpand", + "selection.selectAll", "selection.deselect", "selection.invert", + "selection.rectangle", "selection.ellipse", + "selection.subject", "selection.sky", "selection.background", "selection.samText", + "selection.colorRange", "selection.maskBounds", "selection.exportMask", + "adjustments.apply", "adjustments.applyMasked", + "filters.apply", "filters.applyMasked", + "transform.start", "transform.apply", "transform.cancel", + "text.add", + "history.mark", "history.collapse", "history.undo", "history.redo", + "export.png", "export.gallery", "screenshot", +] +# Retouch/selection ops are defined once in the shared style guide so the +# server planner and the browser fallback cannot drift; a plan naming one of +# these would otherwise be filtered out of the catalog. +for _extra_op in style_guide.NEW_OP_NAMES: + if _extra_op not in OP_CATALOG: + OP_CATALOG.append(_extra_op) +del _extra_op +_OP_SET = frozenset(OP_CATALOG) + +# Canonical argument shapes advertised to the model so it stops guessing. The +# browser client tolerates loose forms as a safety net, but the model should +# emit exactly these. +_ADJUSTMENT_TYPES = ( + "brightness-contrast {brightness,contrast} (multipliers, 1=neutral), " + "exposure {exposure,offset,gamma}, white-balance {temperature,tint}, " + "hue-saturation {hue,saturation,lightness}, vibrance {vibrance}, black-white, " + "shadows-highlights {shadows,highlights}, levels, curves, color-balance, " + "photo-filter {color,density,preserve_luminosity}, selective-color, gradient-map, " + "invert, posterize, threshold, channel-mixer, color-lookup" +) +_FILTER_IDS = ( + "blur-gaussian, blur-motion, blur-zoom, sharpen, sharpen-edges, unsharp-mask, " + "noise-add, noise-despeckle, pixelate-mosaic, pixelate-crystallize, stylize-emboss, " + "stylize-find-edges, stylize-solarize, distort-twirl, distort-pinch, distort-spherize, " + "distort-ripple, render-clouds, clarity {amount,radius}, vignette {amount,feather}" +) +_CANONICAL_GUIDANCE = ( + "Canonical op shapes (use exactly these, never invent keys or names): " + "adjustments.apply / adjustments.applyMasked params = " + "{\"type\": , \"params\": {...}}; " + "filters.apply / filters.applyMasked params = {\"id\": }. " + "Never nest a numeric bag under {\"adjustments\": {...}} with no type, and never invent " + "adjustment types or filter ids. \"clarity\" (params amount,radius) and \"vignette\" " + "(params amount,feather) are real filters — use them for local-contrast and framing. " + "Canvas: document.crop {x,y,width,height}, document.resizeCanvas {width,height,anchor}, " + "ai.generativeExpand {width,height,anchor,prompt,mode:'instruction',feather}. Text/styles: " + "text.add {text,x,y,color,size}, layers.addStyle {type,params}. Export: document.savePsd. " + "ai_edit_image takes a mode: 'instruction' (default; whole image, no mask, prompt names " + "what stays unchanged) for background/scene/season/style changes, adding an object to an " + "existing photo, and removing diffuse distractions; 'masked' (tight selection first) ONLY " + "for explicit regional removal of a clearly identified object. generate_image is ONLY for " + "creating a brand-new image." +) +# The shared style guide is the single source of truth for intent routing, +# calibrated strengths and the low-key/temperature/portrait rules. Both the +# server planner and the browser fallback read it (the client fetches +# GET /api/editor/style-guide). +_STYLE_GUIDANCE = style_guide.guidance() + +# EDITOR_PLANNER: async (messages: list[dict], owner: Optional[str]) -> str | dict. Set by app.py. +_EDITOR_PLANNER: Optional[Callable[[List[dict], Optional[str]], Any]] = None +_MAX_STEPS = 32 +_QUEUE_MAX = 64 +# Generative editor steps (SAM mask + local diffusion) take 30-110s each, so +# bridge waits and the agent's per-step wait must outlast the provider. +_BRIDGE_DEFAULT_TIMEOUT = 330.0 + +# ── Strict JSON ops schema (structured outputs / grammar) ── +# The planner (the local Qwen3.5-9B at :8097 and every fallback) is constrained +# to this schema so it can only emit valid editor ops. Allowed ops: +# * the deterministic editor ops (OP_CATALOG), +# * generate_edit -> image model (instruction, optional mask) [=ai_edit_image] +# * select_object -> SAM mask by description (seneca /v1/images/mask) +# [=selection.samText] +# * ask_user -> ask 2-3 concrete choices (never a silent tone default). +# The model-facing aliases are translated to the internal execution ops below, +# so the browser client and the executor keep their existing op vocabulary. +_MODEL_PLAN_OPS = tuple(dict.fromkeys( + list(OP_CATALOG) + ["generate_edit", "select_object", "ai_edit_image", + "generate_image", "ask_user"] +)) +_ALLOWED_PLAN_OPS = frozenset(OP_CATALOG) | {"ai_edit_image", "generate_image"} +_PLAN_OP_ALIASES = { + "generate_edit": "ai_edit_image", # image model: instruction + optional mask + "select_object": "selection.samText", # SAM mask by description +} +# Inner op values the ai_edit_image tool accepts (src/agent_tools/editor_tools.py +# `_AI_OPS`). A model-authored ai_edit_image step frequently omits the inner op or +# invents one (e.g. "restyle"), which the tool then rejects. Normalise it so +# instruction/restyle edits become op "generative" and masked region edits become +# "inpaint". +_AI_IMAGE_INNER_OPS = frozenset( + {"remove-bg", "upscale", "denoise", "enhance-face", "inpaint", "generative"} +) + + +def _normalize_ai_step(params: Any) -> Dict[str, Any]: + """Repair an ai_edit_image step's inner op so it always runs (restyle -> + generative, masked -> inpaint).""" + params = dict(params) if isinstance(params, dict) else {} + mode = str(params.get("mode") or "").strip().lower() + inner = str(params.get("op") or "").strip().lower() + if inner not in _AI_IMAGE_INNER_OPS: + inner = "inpaint" if mode == "masked" else "generative" + params["op"] = inner + if inner in ("inpaint", "generative") and not mode: + params["mode"] = "masked" if inner == "inpaint" else "instruction" + return params +PLAN_SCHEMA: Dict[str, Any] = { + "type": "object", + "properties": { + "reply": {"type": "string"}, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": list(_MODEL_PLAN_OPS), + "description": ( + "generate_edit = AI image model edit (params {prompt, " + "mode:'instruction'|'masked'}; do NOT add an inner 'op' — " + "instruction/restyle edits run as the image model's " + "'generative' op, masked region edits as 'inpaint'). " + "select_object = SAM mask by text (params {text}). " + "ask_user = ask with 2-3 choices (params {question, options})." + ), + }, + "params": {"type": "object"}, + }, + "required": ["op"], + }, + }, + "ask_user": { + "type": "object", + "properties": { + "question": {"type": "string"}, + "options": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["question", "options"], + }, + "done": {"type": "boolean"}, + }, + "required": ["reply", "steps"], +} +PLAN_RESPONSE_FORMAT: Dict[str, Any] = { + "type": "json_schema", + "json_schema": {"name": "editor_plan", "schema": PLAN_SCHEMA}, +} + + +def _normalize_ask_user(payload: Any) -> Optional[Dict[str, Any]]: + """Validate an ask_user payload: one question and 2-3 concrete choices.""" + if not isinstance(payload, dict): + return None + question = str(payload.get("question") or "").strip() + raw = payload.get("options") + labels: List[str] = [] + for opt in (raw or []): + if isinstance(opt, str): + lbl = opt.strip() + elif isinstance(opt, dict): + lbl = str(opt.get("label") or "").strip() + else: + lbl = "" + if lbl: + labels.append(lbl) + labels = labels[:3] + if not question or len(labels) < 2: + return None + # Match the shape the ask_user tool + the editor client expect: option + # objects with a label (and an optional description). + return { + "question": question, + "options": [{"label": lbl, "description": ""} for lbl in labels], + "multi": False, + } + + +def _validate_plan(data: Any) -> Optional[Dict[str, Any]]: + """Validate a raw planner plan against the strict ops schema. + + Returns a normalized plan (aliases translated, ops checked against the + allowed set, ask_user normalized to 2-3 choices) or None when the plan is + invalid or empty. Callers turn None into an ask_user plan — never a silent + tone default. + """ + if not isinstance(data, dict): + return None + reply = str(data.get("reply") or "") + ask_user = _normalize_ask_user(data.get("ask_user")) + raw_steps = data.get("steps") + if raw_steps is None: + raw_steps = [] + if not isinstance(raw_steps, list): + return None + steps: List[Dict[str, Any]] = [] + for s in raw_steps[:_MAX_STEPS]: + if not isinstance(s, dict): + continue # drop a malformed step rather than rejecting the whole plan + op = str(s.get("op") or "") + if op == "ask_user": + au = _normalize_ask_user(s.get("params")) + if au is not None and ask_user is None: + ask_user = au + continue + op = _PLAN_OP_ALIASES.get(op, op) + if op not in _ALLOWED_PLAN_OPS: + continue # drop a single unknown/hallucinated op; keep the valid steps + params = s.get("params") if isinstance(s.get("params"), dict) else {} + if op == "ai_edit_image": + params = _normalize_ai_step(params) + steps.append({"op": op, "params": params}) + # Guardrails apply to model plans too (no stacked lifts, no brighten after + # retouch.skin, no global adjustment after a masked recolour). + steps = style_guide.enforce_guards(steps) + done = bool(data.get("done")) + if not steps and not done and ask_user is None: + return None + out: Dict[str, Any] = {"reply": reply, "planner": "llm", "steps": steps} + if ask_user is not None: + out["ask_user"] = ask_user + if done: + out["done"] = True + return out + +# Generative edits (remove/add/replace content) require an image provider. When +# none is available the agent must stop and say so plainly rather than apply a +# deterministic approximation that silently does the wrong thing. +try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE as _PROVIDER_REQUIRED_MESSAGE +except Exception: # pragma: no cover - defensive + _PROVIDER_REQUIRED_MESSAGE = ( + "Removing/adding objects needs an image generation provider; enable one in " + "Settings > Models." + ) + +_AI_CONTENT_OPS = frozenset({"ai_edit_image", "generate_image"}) + +# Model ids that are editing/inpainting models rather than text-to-image ones. +_EDIT_MODEL_RE = re.compile(r"edit|inpaint|fill|kontext|controlnet|removal|qwen-image-edit", re.I) + +# Content nouns that signal a generative object edit rather than a tone/filter +# tweak. Kept narrow so "make it black and white" / "add a vignette" stay +# deterministic. +_GEN_OBJECTS = ( + r"(?:persons?|people|men|man|women|woman|guys?|girls?|boys?|someone|somebody|" + r"objects?|car|cars|sign|signs|logos?|watermarks?|trees?|buildings?|" + r"reflections?|animals?|dogs?|cats?|birds?|hands?|fingers?|moons?|suns?|clouds?|" + r"stars?|fire|smoke|snow|rain|flowers?|mountains?|planets?|balloons?|rockets?|hats?|" + r"glasses|sunglasses|shades|beard|beards|stickers?|frames?|borders?|" + r"rainbows?|wildflowers?|sunflowers?|tulips?|daisies?|leaves|pumpkins?|lanterns?|" + r"butterflies?|bees?|boats?|sailboats?|kites?|umbrellas?)" +) +_REMOVE_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|photoshop\s+out|paint\s+out)\b" + r"[^.!?\n]{0,60}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_ADD_RE = re.compile( + r"\b(?:add|insert|put|place|draw|paint|create|generate)\b" + r"[^.!?\n]{0,40}?\b" + _GEN_OBJECTS + r"\b", + re.I, +) +_REPLACE_RE = re.compile( + r"\b(?:replace|swap|substitute)\b|\b(?:change|turn)\b[^.!?\n]{0,40}?\b(?:to|into)\b", + re.I, +) +# Removing diffuse clutter ("remove the distractions") is generative too, but +# the specific-noun _REMOVE_RE above does not catch it. +_REMOVE_CLEANUP_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,40}?\b(?:distractions?|clutter|junk|mess|noise|" + r"unwanted\s+(?:objects?|items?|things?)|stuff|things|blemishes?|spots?)\b", + re.I, +) + +# Changing the whole background/scene (or placing the subject somewhere new) +# is a generative replacement, not the local remove-bg op. "make the +# background nicer/better" is included; plain "blur the background" is not. +_BG_RE = re.compile( + r"\b(?:replace|change|swap|put|place|set|make|improve|enhance|beautify|upgrade)\b" + r"[^.!?\n]{0,40}?\b(?:background|backdrop|scene)\b" + r"|\b(?:put|place)\b[^.!?\n]{0,30}?\b(?:on|in|at)\s+(?:a|an|the)\s+" + r"(?:beach|street|mountains?|forest|city|park|ocean|sea|desert|room|stage|field|snow|" + r"night|moon|mars|space|island|jungle)\b", + re.I, +) +# Season/weather changes and an explicit "generative fill" change scene +# content (not just tone), so they must be a generative edit, never a +# colour adjustment/filter step. +_SEASON_WEATHER_RE = re.compile( + r"\b(?:snow(?:y|ing|fall)?|winter|autumn(?:al)?|fall\s+(?:leaves|foliage)|spring|" + r"rain(?:y|ing)?|fog(?:gy)?|mist(?:y)?|storm(?:y)?|overcast)\b", + re.I, +) +_GENERATIVE_FILL_RE = re.compile(r"\bgenerative(?:\s+fill)?\b", re.I) +# Whole-image artistic RESTYLE requests (turn the photo into a painting, anime, +# pixel art, ...). These are a model instruction edit over the whole frame, not a +# local adjustment. The client has a short fast-path regex; the server must +# generalise so unknown restyle phrasings still reach the model. +_RESTYLE_RE = re.compile( + r"\b(?:re[\s-]?style|painting|painterly|water[\s-]?colou?r|oil\s+paint(?:ing)?|" + r"acrylic|gouache|pastel|sketch|pencil|charcoal|ink\s+drawing|line\s+art|comic|" + r"cartoon|anime|manga|illustration|3d\s+render|cgi|claymation|clay|pixel\s+art|" + r"voxel|low[\s-]?poly|mosaic|stained\s+glass|pop\s+art|art\s+nouveau|ukiyo[\s-]?e|" + r"surreal|fantasy\s+art|steampunk|cyberpunk|vaporwave|synthwave|graffiti|woodcut|" + r"linocut|etching|lithograph|impressionist|cubist|abstract|doodle|vector\s+art|" + r"neon\s+sign|psychedelic|gothic|comic[\s-]?book|storybook)\b", + re.I, +) + +# Text/title/PSD/format requests are deterministic edits, never generative. +_TEXT_RE = re.compile( + r"\b(?:title|caption|subtitle|label|heading|watermark|text|word(?:ing)?)\b", re.I, +) +_TEXT_ADD_RE = re.compile(r"\b(?:add|insert|put|place|write|overlay|create|set)\b", re.I) +_STYLE_RE = re.compile( + r"\b(drop[\s-]?shadow|outer[\s-]?glow|glow|stroke|outline|emboss|bevel)\b", re.I, +) +_PSD_RE = re.compile(r"\b(?:psd|photoshop file|layered file)\b", re.I) + +# Aspect -> exact output size. Instagram/format intents crop to the aspect and +# then resize to this size (then optionally add a title). +_ASPECT_TARGETS = ( + (re.compile(r"\b(?:1080\s*[x×]\s*1350|4\s*:\s*5|four[\s-]?fifths?|instagram)\b", re.I), (1080, 1350)), + (re.compile(r"\b(?:1080\s*[x×]\s*1080|1\s*:\s*1|square)\b", re.I), (1080, 1080)), + (re.compile(r"\b(?:1080\s*[x×]\s*1920|9\s*:\s*16|story|reel|tiktok|vertical)\b", re.I), (1080, 1920)), + (re.compile(r"\b(?:1920\s*[x×]\s*1080|16\s*:\s*9|widescreen|landscape video)\b", re.I), (1920, 1080)), +) +_DIM_RE = re.compile(r"\b(\d{3,5})\s*[x×]\s*(\d{3,5})\b") + + +def _is_text_request(text: str) -> bool: + return bool(_TEXT_RE.search(text) and _TEXT_ADD_RE.search(text)) + + +def _format_target(text: str) -> Optional[Tuple[int, int]]: + for pattern, dims in _ASPECT_TARGETS: + if pattern.search(text): + return dims + m = _DIM_RE.search(text) + if m: + try: + return (int(m.group(1)), int(m.group(2))) + except Exception: + return None + return None + + +def _extend_requested(text: str) -> bool: + if re.search(r"\b(?:extend|expand|wider|taller|letterbox)\b", text, re.I): + return True + return bool(re.search( + r"\badd\b[^.!?\n]{0,25}?\b(?:canvas|border|space|margin)\b", text, re.I, + )) + + +def _generative_kind(prompt: str) -> Optional[str]: + """Classify a prompt as a generative content edit: 'remove'/'add'/'replace'. + + Returns None for tone/colour/filter/text/selection edits that deterministic + editor ops handle correctly. 'remove the background' is deliberately not + matched (that is the local remove-bg op). + """ + text = str(prompt or "") + if not text.strip(): + return None + # Text/title, PSD and format requests are deterministic — never generative. + if _is_text_request(text) or _PSD_RE.search(text) or _format_target(text): + return None + # An explicit brand-new image ("create/generate/make a new image/picture/logo + # ...") is a text-to-image generation, never an edit of the current photo. + if _is_new_image_request(text): + return "add" + # Replacing the background/scene (or putting the subject somewhere new) is + # a generative replacement even though it is phrased like a background op. + if _BG_RE.search(text): + return "replace" + if re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of)\s+(?:the\s+)?background\b", + text, + re.I, + ): + return None + # Season/weather changes and explicit "generative fill" change content, not + # tone. Only treat them as generative when the user is not asking to REMOVE + # that element ("remove the rain" stays a removal). + if _GENERATIVE_FILL_RE.search(text) or _SEASON_WEATHER_RE.search(text): + if not re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\b", text, re.I + ): + return "replace" + if _RESTYLE_RE.search(text) and not _is_new_image_request(text): + # A whole-image artistic restyle is a model instruction edit (keep the + # subject/composition), never a local tone/colour adjustment. + return "restyle" + if _REPLACE_RE.search(text): + return "replace" + if _ADD_RE.search(text): + return "add" + if _REMOVE_RE.search(text) or _REMOVE_CLEANUP_RE.search(text): + return "remove" + return None + + +_NEW_IMAGE_RE = re.compile( + r"\b(?:create|generate|make|produce|draw|design)\b[^.!?\n]{0,30}?\b" + r"(?:new\s+)?(?:image|picture|photo|illustration|artwork|graphic|logo|icon|wallpaper|poster)\b", + re.I, +) +_EDIT_CONTEXT_RE = re.compile( + r"\b(?:this|that|these|it|the\s+(?:photo|image|picture)|attached|from\s+this|" + r"of\s+(?:this|the\s+photo)|edit|in\s+(?:this|the)\s+photo)\b", + re.I, +) +_KEEP_CLAUSE_RE = re.compile(r"\bkeep\b|\bunchanged\b|\bpreserve\b|\bsame\b|\bidentical\b", re.I) + + +def _is_new_image_request(text: str) -> bool: + """True only for a from-scratch image generation (not an edit of an existing photo).""" + if not _NEW_IMAGE_RE.search(text): + return False + return not _EDIT_CONTEXT_RE.search(text) + + +def _instruction_prompt(text: str) -> str: + """Append a keep-unchanged clause so instruction edits preserve the subject.""" + t = str(text or "").strip() + if not t: + return t + if _KEEP_CLAUSE_RE.search(t): + return t + return ( + t.rstrip(".") + + ". Keep the main subject and everything else in the image exactly the same." + ) + + +def _restyle_prompt(text: str) -> str: + """Whole-image artistic restyle instruction (subject/composition preserved).""" + t = str(text or "").strip() + if t: + base = ( + "Restyle the entire image as requested, applying the artistic style " + "across the whole frame: " + t + ) + else: + base = "Restyle the entire image in a new artistic style." + return _instruction_prompt(base) + + +def _generative_mode(kind: str, prompt: str) -> str: + """'masked' only for an explicit regional removal of a clearly identified object. + + Everything else generative (background/season/style replacement, adding an + object to the scene, removing diffuse distractions) uses 'instruction'. + """ + if kind == "remove": + text = str(prompt or "") + if _REMOVE_RE.search(text) and not _REMOVE_CLEANUP_RE.search(text): + return "masked" + return "instruction" + + +def _removal_prompt(text: str) -> str: + """A more explicit diffuse-removal instruction that names common clutter.""" + t = str(text or "").strip() or "Remove the distracting clutter from the photo" + return ( + t.rstrip(". ") + + ". Remove any distracting clutter such as roads, guardrails, utility poles, " + "signs, wires and debris, and fill the cleared area with the surrounding " + "landscape so it looks natural." + ) + + +# Rich, scene-specific descriptions so a background replacement produces a +# convincing scene (Qwen-Image-Edit is an instruction model and a bare "a beach" +# tends to yield a flat blue backdrop). Keyed by keywords in the request. +_SCENE_HINTS = ( + (r"beach|shore|seaside|coast|tropical", + "a sunny tropical beach", + "golden sand, turquoise sea with gentle waves, a clear blue sky with soft " + "clouds, and natural daylight"), + (r"mountain|alps|peak|valley", + "a scenic mountain landscape", + "distant peaks, natural daylight and soft haze"), + (r"forest|woods|jungle", + "a lush green forest", + "dappled natural daylight and depth"), + (r"city|street|urban|downtown", + "a realistic city street", + "recognisable city buildings and depth of field with natural perspective"), + (r"field|meadow|grass", + "an open grassy field", + "natural daylight and a distant horizon"), + (r"snow|winter", + "a snow-covered landscape", + "cold, soft daylight and a pale sky"), + (r"desert|dunes", + "a warm desert", + "sand dunes and a clear sky"), + (r"studio|plain|neutral|solid", + "a clean neutral studio backdrop", + "soft, even lighting"), +) +_SCENE_DEFAULT = ( + "a realistic natural background", + "matching the subject's lighting, perspective and shadows", +) +# Season/weather scene descriptions ("make it snowy/autumn"). +_SEASON_HINTS = ( + (r"snow|winter|snowy", "a snowy winter scene", + "fresh snow covering the ground and surfaces, soft cold daylight, a pale overcast sky"), + (r"autumn|autumnal|fall", "an autumn scene", + "vivid orange and golden foliage, warm low sunlight and fallen leaves"), + (r"spring", "a bright spring scene", + "fresh green foliage and blossoms under bright natural daylight"), + (r"rain|storm|overcast", "a rainy overcast scene", + "wet reflective surfaces, raindrops and a dramatic overcast sky"), + (r"fog|mist", "a foggy scene", + "soft drifting mist and diffuse light"), + (r"sky|clouds", "a new sky", + "realistic clouds and natural lighting matching the scene"), +) + + +def _scene_prompt(text: str) -> str: + """Deterministic, templated instruction for a background/scene change. + + Qwen-Image-Edit is an instruction model and a bare scene noun tends to yield + a flat backdrop, so the scene is expanded into concrete detail. The result is + passed to the provider unmodified. + """ + t = str(text or "").strip() + if not t: + return t + if _BG_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SCENE_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Replace the entire background behind the person with {scene}: {details}. " + "Keep the person exactly the same." + ) + if _SEASON_WEATHER_RE.search(t) or _GENERATIVE_FILL_RE.search(t): + scene, details = _SCENE_DEFAULT + for pattern, scene_name, description in _SEASON_HINTS: + if re.search(pattern, t, re.I): + scene, details = scene_name, description + break + return ( + f"Change the scene to {scene}: {details}. " + "Keep the main subject and everything else in the image exactly the same." + ) + return t + + +def _generative_step(kind: str, prompt: str) -> Dict[str, Any]: + """The AI step a generative intent must use instead of deterministic ops.""" + instruction = str(prompt or "").strip() + if kind == "restyle": + return { + "op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": _restyle_prompt(instruction), "steps": 8}, + } + if kind == "add": + # A brand-new image is the only case for generate_image; adding an object + # to an existing photo is an instruction edit of the whole image. + if _is_new_image_request(prompt): + return {"op": "generate_image", "params": {"prompt": instruction}} + return { + "op": "ai_edit_image", + "params": { + "op": "generative", + "mode": "instruction", + "prompt": _instruction_prompt(instruction), + "steps": 8, + }, + } + mode = _generative_mode(kind, prompt) + op = "inpaint" if kind == "remove" else "generative" + if mode == "instruction": + if kind == "remove": + base = _removal_prompt(instruction) + else: + base = _scene_prompt(instruction) + params: Dict[str, Any] = {"op": op, "mode": "instruction", "prompt": _instruction_prompt(base)} + else: + params = {"op": op, "mode": "masked", "prompt": instruction} + if kind in ("remove", "replace"): + # Removal and background-replacement edits need more sampler steps than + # the server's default of 4. + params["steps"] = 8 + return {"op": "ai_edit_image", "params": params} + + +_SKY_ADD_RE = re.compile( + r"\b(?:sky|moons?|suns?|sunset|sunrise|clouds?|stars?|night|aurora|stars?)\b", + re.I, +) +_REMOVE_NOUN_RE = re.compile( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out|clean\s+up|declutter|clear)\b" + r"[^.!?\n]{0,60}?\b(" + + _GEN_OBJECTS + + r"|distractions?|clutter|junk|mess|noise|stuff|things)\b", + re.I, +) + + +def _selection_step_for(kind: str, prompt: str) -> Optional[Dict[str, Any]]: + """A deterministic selection step that yields the mask a MASKED ai_edit_image needs. + + Instruction-mode edits operate on the whole image and need no mask, so this + returns None for them. Masked mode is only used for an explicit regional + object removal; the selection targets that object so ``selection.exportMask`` + has a tight region to export. + """ + text = str(prompt or "") + if kind == "remove" and _generative_mode(kind, text) == "masked": + m = _REMOVE_NOUN_RE.search(text) + label = (m.group(1).lower() if m else "person") + if label.endswith("s"): + label = label[:-1] + if label in ("people",): + label = "person" + return {"op": "selection.subject", "params": {"label": label}} + return None + + +def _generative_steps(kind: str, prompt: str) -> List[Dict[str, Any]]: + """The selection step (when needed) followed by the AI content step.""" + steps: List[Dict[str, Any]] = [] + selection = _selection_step_for(kind, prompt) + if selection: + steps.append(selection) + steps.append(_generative_step(kind, prompt)) + return steps + + +def _plan_uses_ai_content(steps: Any) -> bool: + return any(isinstance(s, dict) and s.get("op") in _AI_CONTENT_OPS for s in (steps or [])) + + +def _ensure_generative(prompt: str, steps: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Force a generative intent onto an AI content op, never a substitute. + + If the prompt is a generative content edit (remove/add/replace) but the + plan has no ai_edit_image/generate_image step, replace the plan with the + correct AI step so the agent cannot silently do an unrelated deterministic + edit. Non-generative prompts are returned unchanged. + """ + kind = _generative_kind(prompt) + if not kind: + return steps + # Background/scene replacement is deterministic: always use the templated + # scene prompt so a model rewrite cannot drop the scene details (a model + # ai_edit_image step would otherwise pass with its own vague prompt). + if kind == "replace" and ( + _BG_RE.search(str(prompt or "")) + or _SEASON_WEATHER_RE.search(str(prompt or "")) + or _GENERATIVE_FILL_RE.search(str(prompt or "")) + ): + return _generative_steps(kind, prompt) + # For a restyle (and any other generative intent) the planner is expected to + # choose generate_edit itself; the _RESTYLE_RE keyword list is only a fast + # path used when the model did not already pick an AI content op. + if not _plan_uses_ai_content(steps): + return _generative_steps(kind, prompt) + return steps + + +def _image_provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True # unknown: let the tool attempt and report its own error + + +def _state_dims(state: Any) -> Optional[Tuple[int, int]]: + """Current canvas size from the client-sent state (flat or nested).""" + if not isinstance(state, dict): + return None + candidates = [state] + for key in ("document", "document_info", "info"): + if isinstance(state.get(key), dict): + candidates.append(state[key]) + for src in candidates: + w = src.get("width") or src.get("imgWidth") + h = src.get("height") or src.get("imgHeight") + try: + if w and h: + return (int(round(float(w))), int(round(float(h)))) + except Exception: + continue + return None + + +def _title_text(prompt: str) -> Optional[str]: + quoted = re.search(r"[\"'\u201c\u201d\u2018\u2019]([^\"'\u201c\u201d\u2018\u2019]{1,40})[\"'\u201c\u201d\u2018\u2019]", prompt) + if quoted and quoted.group(1).strip(): + return quoted.group(1).strip() + caps = re.search(r"\b([A-Z][A-Z0-9]{1,20})\b", prompt) + if caps and caps.group(1).lower() not in {"add", "title", "text", "psd"}: + return caps.group(1) + return None + + +def _crop_rect_for_aspect(cur: Tuple[int, int], target: Tuple[int, int]) -> Optional[Dict[str, int]]: + cw, ch = cur + tw, th = target + if cw <= 0 or ch <= 0 or tw <= 0 or th <= 0: + return None + target_ratio = tw / th + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return None # already the right aspect + if cur_ratio > target_ratio: + # too wide -> trim the sides + new_w = int(round(ch * target_ratio)) + new_w = min(cw, max(1, new_w)) + return {"x": (cw - new_w) // 2, "y": 0, "width": new_w, "height": ch} + # too tall -> trim top/bottom + new_h = int(round(cw / target_ratio)) + new_h = min(ch, max(1, new_h)) + return {"x": 0, "y": (ch - new_h) // 2, "width": cw, "height": new_h} + + +def _extend_dims(cur: Tuple[int, int], target: Tuple[int, int]) -> Tuple[int, int]: + """Target canvas that grows (never crops) toward the target aspect.""" + cw, ch = cur + target_ratio = target[0] / target[1] + cur_ratio = cw / ch + if abs(cur_ratio - target_ratio) < 0.02: + return (cw, ch) + if cur_ratio < target_ratio: + return (int(round(ch * target_ratio)), ch) # add width + return (cw, int(round(cw / target_ratio))) # add height + + +def _plan_text_steps(prompt: str) -> List[Dict[str, Any]]: + title = _title_text(prompt) or "Title" + steps: List[Dict[str, Any]] = [{ + "op": "text.add", + "params": {"text": title, "x": 40, "y": 40, "color": "#ffffff", "size": 64}, + }] + if _STYLE_RE.search(prompt): + steps.append({"op": "layers.addStyle", "params": {"type": "drop-shadow", "params": {}}}) + return steps + + +def _plan_format_steps(prompt: str, state: Any, provider_available: bool) -> Optional[List[Dict[str, Any]]]: + target = _format_target(prompt) + if not target: + return None + cur = _state_dims(state) + if not cur: + # No canvas size known: ask the client to crop to the aspect itself. + return None + steps: List[Dict[str, Any]] = [] + if _extend_requested(prompt): + new_w, new_h = _extend_dims(cur, target) + if (new_w, new_h) == cur: + return None + if provider_available: + steps.append({"op": "ai.generativeExpand", "params": { + "width": new_w, "height": new_h, + "anchor": "center", "mode": "instruction", "feather": True, + "steps": 8, + "prompt": "extend the surrounding scene and sky naturally into the new border, " + "keep the original photo unchanged"}}) + else: + steps.append({"op": "document.resizeCanvas", "params": { + "width": new_w, "height": new_h, "anchor": "center"}}) + else: + rect = _crop_rect_for_aspect(cur, target) + if rect: + steps.append({"op": "document.crop", "params": rect}) + # Only resize when an exact output size was requested. + if abs(target[0] / target[1] - cur[0] / cur[1]) >= 0.02 or target in ((1080, 1350), (1080, 1080), (1080, 1920), (1920, 1080)): + steps.append({"op": "document.resizeCanvas", "params": { + "width": target[0], "height": target[1], "anchor": "center"}}) + if not steps: + return None + # Optional caption/title that came with the format request. + if _is_text_request(prompt): + steps.extend(_plan_text_steps(prompt)) + return steps + + +def _plan_psd_steps(prompt: str) -> Optional[List[Dict[str, Any]]]: + if _PSD_RE.search(prompt): + return [{"op": "document.savePsd", "params": {}}] + return None + + +def _plan_special( + prompt: str, state: Any, provider_available: bool, +) -> Optional[Tuple[List[Dict[str, Any]], str, str]]: + """Deterministic overrides for intents the model reliably botches. + + Canvas format/crop/extend, text + layer styles and PSD export must not be + left to the model. Returns (steps, reply, note) or None. These never need an + image provider except generative border fill, used only when one exists. + """ + # Targeted recolour is fully deterministic (object mask + masked hue shift) + # and must never be left to the model, which sometimes substitutes a + # generative placeImage. Highest priority so it can never vary. + recolor = style_guide.recolor_plan(prompt) + if recolor: + return list(recolor["steps"]), str(recolor["reply"]), "" + if _format_target(prompt): + steps = _plan_format_steps(prompt, state, provider_available) + if not steps: + return None + target = _format_target(prompt) + reply = f"Setting the canvas to {target[0]}\u00d7{target[1]}." + note = "" + if _extend_requested(prompt) and not provider_available: + note = ("No image generation provider is configured, so the new border " + "was extended with edge/transparent pixels.") + return steps, reply, note + if _PSD_RE.search(prompt): + steps = _plan_psd_steps(prompt) + return (steps, "Saving a layered PSD.", "") if steps else None + if _is_text_request(prompt): + steps = _plan_text_steps(prompt) + if steps: + reply = ("Adding the title with a drop shadow." + if _STYLE_RE.search(prompt) else "Adding the title text.") + return steps, reply, "" + # Structural ops (fill, text edit, align, rotate/flip, copy/paste, flatten, + # merge-down, colour overlay) — the guide's deterministic routing so the + # model never substitutes a generic adjustment/transform for them. + structural = style_guide.structural_plan(prompt) + if structural: + return list(structural["steps"]), str(structural["reply"]), "" + return None + + +_DIMS_CLAIM_RE = re.compile(r"(\d{2,5})\s*[x\u00d7]\s*(\d{2,5})") +_CANVAS_CHANGE_OPS = frozenset({ + "document.crop", "document.resizeCanvas", "ai.generativeExpand", "document.importImage", +}) +_TEXT_OPS = frozenset({"text.add", "layers.addStyle", "document.savePsd"}) + + +def _step_op_signature(step: Any) -> str: + """Identify an executed step by op plus its adjustment type / filter id.""" + if not isinstance(step, dict): + return "" + op = str(step.get("op") or "") + params = step.get("params") or {} + if op in ("adjustments.apply", "adjustments.applyMasked"): + return f"{op}:{params.get('type') or ''}" + if op in ("filters.apply", "filters.applyMasked"): + return f"{op}:{params.get('id') or ''}" + return op + + +# Phrases that name a specific edit. If the reply claims one of these but the +# executed steps do not contain it, the reply is rewritten from the real steps. +_OP_CLAIM_KEYWORDS: List[Tuple[str, Tuple[str, ...]]] = [ + ("photo.autoTone", ("auto tone", "auto-tone", "autotone", "auto-tune")), + ("adjustments.apply:shadows-highlights", ("shadow", "highlight")), + ("adjustments.apply:white-balance", ("white balance", "white-balance", + "temperature", "colour cast", "color cast")), + ("adjustments.apply:vibrance", ("vibrance",)), + ("adjustments.apply:brightness-contrast", ("brightness", "contrast")), + ("adjustments.apply:black-white", ("black and white", "black & white", + "grayscale", "greyscale", "monochrome")), + ("filters.apply:clarity", ("clarity", "local contrast")), + ("filters.apply:vignette", ("vignette",)), + ("filters.apply:sharpen", ("sharpen", "crisper")), + ("retouch.skin", ("skin", "complexion")), + ("retouch.teeth", ("teeth", "smile")), +] + +_OP_LABELS: Dict[str, str] = { + "photo.autoTone": "auto tone", + "adjustments.apply:shadows-highlights": "a shadow lift (highlights protected)", + "adjustments.apply:white-balance": "white balance", + "adjustments.apply:vibrance": "vibrance", + "adjustments.apply:brightness-contrast": "brightness/contrast", + "adjustments.apply:black-white": "black and white", + "filters.apply:clarity": "local contrast (clarity)", + "filters.apply:vignette": "a light vignette", + "filters.apply:sharpen": "sharpening", + "selection.face": "a face selection", + "selection.background": "a background selection", + "retouch.skin": "gentle skin retouching", + "retouch.teeth": "teeth whitening", +} + + +def _reply_from_ops(steps: List[Dict[str, Any]]) -> str: + """Describe what was actually executed, never what was planned.""" + labels: List[str] = [] + for step in steps or []: + sig = _step_op_signature(step) + label = _OP_LABELS.get(sig) + if label is None: + label = sig.split(":")[0] or "an edit" + if label not in labels: + labels.append(label) + if not labels: + return "Done." + return "Done \u2014 I applied " + ", ".join(labels) + "." + + +def _reply_matches_ops(planned_reply: str, steps: List[Dict[str, Any]]) -> bool: + """True when the reply does not claim an edit that was not executed.""" + text = (planned_reply or "").lower() + if not text: + return True + # "drop shadow" is a layer style, not a shadows/highlights adjustment. + text = text.replace("drop shadow", "drop-shadow-style") + executed = {_step_op_signature(step) for step in (steps or [])} + for sig, keywords in _OP_CLAIM_KEYWORDS: + if sig in executed: + continue + if any(keyword in text for keyword in keywords): + return False + return True + + + +def _verified_reply( + prompt: str, + steps: List[Dict[str, Any]], + results: List[Dict[str, Any]], + final_state: Any, + planned_reply: str, +) -> str: + """Build the final reply from ACTUAL executed results + final document state. + + Never let a plan's optimistic prose claim an edit that did not happen. + """ + failed = [ + r for r in (results or []) + if isinstance(r, dict) and r.get("ok") is False + ] + if failed: + op = str(failed[0].get("op") or "step") + err = failed[0].get("error") or "it did not complete" + return (f"I couldn't finish the {op} step ({err}). " + f"Nothing else was left changed — the image is back as it was.") + dims = _state_dims(final_state) + step_ops = [str(s.get("op") or "") for s in (steps or []) if isinstance(s, dict)] + claim = _DIMS_CLAIM_RE.search(planned_reply or "") + if claim and dims: + try: + claimed = (int(claim.group(1)), int(claim.group(2))) + except Exception: + claimed = None + if claimed and (abs(dims[0] - claimed[0]) > 2 or abs(dims[1] - claimed[1]) > 2): + if not any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (f"I planned to make it {claimed[0]}\u00d7{claimed[1]}, but the canvas is " + f"still {dims[0]}\u00d7{dims[1]} — the resize didn't apply. " + f"Please try again in the editor.") + if not steps and not results: + return planned_reply or "I didn't have a concrete edit to apply." + if not _is_text_request(prompt) and not _reply_matches_ops(planned_reply, steps): + return _reply_from_ops(steps) + if _is_text_request(prompt) and any(op in _TEXT_OPS for op in step_ops) and not failed: + return planned_reply or "Added the text." + if dims and any(op in _CANVAS_CHANGE_OPS for op in step_ops): + return (planned_reply or f"Done — the canvas is now {dims[0]}\u00d7{dims[1]}.") + return planned_reply + + +def set_editor_planner( + fn: Optional[Callable[[List[dict], Optional[str]], Any]], +) -> None: + global _EDITOR_PLANNER + _EDITOR_PLANNER = fn + + +_ROUTING_GUIDANCE = ( + "Routing rules (follow strictly): removing objects/people, adding new content " + "(e.g. a moon, an animal, a sign), or replacing/changing what is in the picture " + "are GENERATIVE edits -> use op \"ai_edit_image\" (op \"inpaint\" to remove, " + "\"generative\" to replace, with a descriptive prompt) or op \"generate_image\" " + "({prompt}) for added content. NEVER approximate these with selection.rectangle, " + "adjustments.apply, filters.apply, transform.* or any other deterministic op. " + "Use deterministic ops only for tone/colour/filter/crop/resize/text/selection " + "edits that do not invent or delete picture content. \"remove the background\" " + "is op \"ai_edit_image\" op \"remove-bg\". If a generative edit is requested and " + "no image provider is configured, do NOT substitute a deterministic edit: return " + "{\"reply\": \"" + _PROVIDER_REQUIRED_MESSAGE + "\", \"steps\": []}." +) + +# Strict ops vocabulary for the model (the schema is also sent as a structured +# output constraint, so the model can only emit these ops). +_SCHEMA_NOTE = ( + "Reply with ONLY a JSON object matching the ops schema. Valid step ops are the " + "deterministic editor ops above plus: \"generate_edit\" (a model image edit; " + "params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\", \"mask\": optional " + "object description}) for any edit that invents, removes, replaces, recolours or " + "restyles image content; \"select_object\" (params {\"text\": object description}) " + "to select an object by description; and \"ask_user\" (params {\"question\": str, " + "\"options\": [2-3 short choices]}) when the request is ambiguous. For " + "\"generate_edit\" set params {\"prompt\": str, \"mode\": \"instruction\"|\"masked\"} " + "and do NOT set an inner \"op\" field: the server maps instruction/restyle edits to " + "the image model's \"generative\" op and masked region edits to \"inpaint\". Prefer " + "generate_edit over deterministic approximations for content changes, and ask_user " + "with 2-3 concrete choices rather than guessing a tone tweak." +) + +_SYSTEM_PROMPT = ( + "You are a photo-editor planning agent. Convert the user's request into a JSON plan of " + "editor operations. Return ONLY JSON: {\"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Use selection.* before targeted edits when possible. Keep to <=6 steps. " + "If a screenshot is attached, use it to judge the current result. " + + _SCHEMA_NOTE +) + +_CHECK_SYSTEM_PROMPT = ( + "You are a photo-editor quality checker. A screenshot of the editor's CURRENT result is " + "attached along with the user's request and the iteration number. Decide whether the edit " + "satisfies the request. Return ONLY JSON: if it is complete, {\"done\": true, \"reply\": str}; " + "if more work is needed, {\"done\": false, \"reply\": str, \"steps\": [{\"op\": str, \"params\": {}}]}. " + "Available ops: " + ", ".join(OP_CATALOG) + ", plus ai_edit_image and generate_image. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Never repeat steps that already ran; <=4 steps. " + + _SCHEMA_NOTE +) + + +_GENERIC_TONE_OPS = frozenset({"photo.autoTone", "adjustments.apply", "filters.apply"}) + + +def _plan_is_generic_tone_only(steps: Any) -> bool: + """True when a plan is nothing but whole-image tone/colour/filter steps.""" + items = [s for s in (steps or []) if isinstance(s, dict)] + return bool(items) and all(s.get("op") in _GENERIC_TONE_OPS for s in items) + + +def _unsure_plan(prompt: str, plan: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Demote a bare whole-image tone plan to an ask when the prompt is unsure. + + When the planner produced ONLY generic tone ops (photo.autoTone / + adjustments.apply / filters.apply) for a prompt that matches no concrete + intent and names no concrete defect, the editor is guessing ("it just adds + vibrance and brightness"). Ask the user with 2-3 concrete choices instead. + """ + if not isinstance(plan, dict): + return plan + steps = plan.get("steps") or [] + if not _plan_is_generic_tone_only(steps): + return plan + if _generative_kind(prompt): + return plan + if style_guide.has_concrete_intent(prompt): + return plan + return style_guide.ask_user_plan() + + +def _fallback_plan(prompt: str) -> Dict[str, Any]: + kind = _generative_kind(prompt) + if kind: + label = {"remove": "object removal", "add": "content addition", + "replace": "content replacement", "restyle": "artistic restyle"}[kind] + return { + "reply": f"Planned an AI {label}.", + "planner": "fallback", + "steps": _generative_steps(kind, prompt), + } + guided = style_guide.fallback_plan(prompt) + if guided is not None: + guided["steps"] = style_guide.enforce_guards(guided.get("steps") or []) + return guided + # Nothing mapped to a concrete intent: ASK instead of silently applying a + # default tone tweak. The old fallback emitted a generic + # photo.autoTone/adjustments/filters stack here, which is the "it just adds + # vibrance and brightness" bug for unrecognized prompts. + return style_guide.ask_user_plan() + + +def _parse_plan(raw: Any) -> Optional[Dict[str, Any]]: + if isinstance(raw, dict): + data = raw + else: + text = raw if isinstance(raw, str) else "" + if not text: + return None + try: + data = json.loads(text) + except Exception: + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + return None + try: + data = json.loads(text[start:end + 1]) + except Exception: + return None + # Every plan is validated against the strict ops schema; an invalid or empty + # plan returns None so the caller falls back to ask_user (never a tone + # default). See _validate_plan. + return _validate_plan(data) + + +def _normalize_steps(body: Dict[str, Any]) -> List[Dict[str, Any]]: + if isinstance(body.get("steps"), list): + raw = body["steps"] + elif body.get("op"): + raw = [{"op": body["op"], "params": body.get("params") or {}}] + else: + raw = [] + out: List[Dict[str, Any]] = [] + for step in raw[:_MAX_STEPS]: + if not isinstance(step, dict): + continue + op = str(step.get("op") or "") + if op not in _OP_SET: + continue + params = step.get("params") + out.append({"op": op, "params": params if isinstance(params, dict) else {}}) + return out + + +async def _json_object(request: Request) -> Dict[str, Any]: + try: + body = await request.json() + except (ValueError, UnicodeDecodeError): + raise HTTPException(400, "Expected a JSON object") + if not isinstance(body, dict): + raise HTTPException(400, "Expected a JSON object") + return body + + +def _prompt(value: Any) -> str: + text = str(value or "").strip() + if len(text) > 4096: + raise HTTPException(413, "Editor prompt is too long") + return text + + +def _bridge_timeout(value: Any) -> float: + try: + timeout = float(value or _BRIDGE_DEFAULT_TIMEOUT) + except (TypeError, ValueError, OverflowError): + raise HTTPException(400, "Invalid bridge timeout") + if not math.isfinite(timeout) or timeout <= 0 or timeout > 900: + raise HTTPException(400, "Invalid bridge timeout") + return timeout + + +def _is_admin(request: Request) -> bool: + """Return True only for authorized administrator browser sessions or auth-disabled single-user mode. + + Delegated credentials (bearer API tokens) are explicitly rejected: + per src/tool_security.py, bearer tokens are long-lived credentials handed to + third parties/scripts and must remain capped at non-admin policies even + when minted by an administrator. + """ + # 1. Delegated credentials (bearer API tokens) must NEVER receive admin metadata privileges + if getattr(request.state, "api_token", False): + return False + + # 2. In-process internal tool bypass (loopback agent tools) + try: + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN, INTERNAL_TOOL_USER + import secrets + hdr = request.headers.get(INTERNAL_TOOL_HEADER) + if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN): + return True + if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER: + return True + except Exception: + pass + + # 3. Explicitly auth-disabled single-user operation (AUTH_ENABLED=false) + try: + from src.auth_helpers import _auth_disabled + if _auth_disabled(): + return True + except Exception: + pass + + # 4. Middleware require_admin + try: + from core.middleware import require_admin + require_admin(request) + return True + except Exception: + pass + + # 5. Direct session/user check when request.state.current_user is set + try: + auth_mgr = getattr(request.app.state, "auth_manager", None) + if auth_mgr and getattr(auth_mgr, "is_configured", False): + from src.auth_helpers import get_current_user + user = get_current_user(request) or getattr(request.state, "current_user", None) + if user and getattr(auth_mgr, "is_admin", None) and auth_mgr.is_admin(user): + return True + except Exception: + pass + + return False + + +def _user(request: Request) -> Optional[str]: + # Handle API bearer tokens explicitly: require 'chat' scope and attribute to real owner + if getattr(request.state, "api_token", False): + from src.auth_helpers import require_api_token_scope + return require_api_token_scope(request, "chat") + from src.auth_helpers import get_current_user, _auth_disabled + u = get_current_user(request) + if u: + return u + if _auth_disabled(): + return None + auth_mgr = getattr(request.app.state, "auth_manager", None) + if auth_mgr is not None and getattr(auth_mgr, "is_configured", False): + raise HTTPException(401, "Not authenticated") + client = getattr(request, "client", None) + host = (client.host if client else "") or "" + if host in ("127.0.0.1", "::1", "localhost", "testclient") or not host: + return None + raise HTTPException(401, "Not authenticated") + + +async def _default_editor_planner(messages: List[dict], owner: Optional[str]) -> Optional[str]: + """Plan with the local editor-agent model, falling back through the chain. + + Order: per-request ``agent_model`` override -> local Qwen3.5-9B (vision, + screenshot attached) -> the same model text-only with a VL caption of the + screenshot -> the hosted default/utility chat model. The model actually + used is recorded so the route can report it. + """ + result = await editor_agent_model.plan_with_editor_agent( + messages, owner, override=editor_agent_model.get_override(), + response_format=PLAN_RESPONSE_FORMAT, + ) + if result is None: + return None + raw, model_used, source = result + editor_agent_model.set_last_model(model_used, source) + return raw + + +def setup_editor_agent_routes() -> APIRouter: + router = APIRouter(tags=["editor-agent"]) + + if _EDITOR_PLANNER is None: + set_editor_planner(_default_editor_planner) + + # scoped session -> list of that session's live subscriber queues; id -> result future. + # A command is BROADCAST to every live subscriber (per-connection queue), never to a + # single shared queue: EventSource reconnects overlap, and with one shared queue a + # command could be consumed by a stale subscriber while the active page got nothing. + subscribers: Dict[Tuple[Optional[str], str], List["asyncio.Queue[dict]"]] = {} + backlog: Dict[Tuple[Optional[str], str], List[dict]] = {} + pending: Dict[str, Tuple[Optional[str], asyncio.Future]] = {} + max_pending = 1024 + + def _subscribe(session: str) -> "asyncio.Queue[dict]": + q: "asyncio.Queue[dict]" = asyncio.Queue(maxsize=_QUEUE_MAX) + subscribers.setdefault(session, []).append(q) + # Replay anything enqueued while this session had no live subscriber. + for cmd in backlog.pop(session, []) or []: + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + break + return q + + def _unsubscribe(session: str, q: "asyncio.Queue[dict]") -> None: + subs = subscribers.get(session) + if subs and q in subs: + subs.remove(q) + if not subs: + subscribers.pop(session, None) + + def _broadcast(session: str, cmd: dict) -> int: + subs = subscribers.get(session) or [] + if subs: + for q in list(subs): + try: + q.put_nowait(cmd) + except asyncio.QueueFull: + pass + return len(subs) + # No live subscriber yet: hold it so a subscriber that connects shortly + # after still receives the command (bounded ring buffer). + bl = backlog.setdefault(session, []) + bl.append(cmd) + if len(bl) > _QUEUE_MAX: + del bl[0: len(bl) - _QUEUE_MAX] + return 0 + + def _expire_command(session, cid, fut): + entry = pending.get(cid) + if entry is not None and entry[1] is fut: + pending.pop(cid, None) + if not fut.done(): + fut.cancel() + queued = backlog.get(session) + if queued: + backlog[session] = [cmd for cmd in queued if cmd.get("id") != cid] + if not backlog[session]: + backlog.pop(session, None) + + def _scoped(session: Optional[str], owner: Optional[str]) -> Tuple[Optional[str], str]: + # Usernames and session ids may both contain colons. + return (owner, str(session or 'default')) + + def _scope_label(scope: Tuple[Optional[str], str]) -> str: + return f"{scope[0] or 'anon'}:{scope[1]}" + + @router.post("/api/editor/plan") + async def plan(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + prompt = _prompt(body.get("prompt")) + state = body.get("state") or {} + mode = str(body.get("mode") or "plan").strip().lower() + screenshot = body.get("screenshot") + agent_model = str(body.get("agent_model") or "").strip() + if not prompt and mode != "check": + return JSONResponse({"reply": "No prompt provided.", "planner": "none", "steps": [], + "agent_model": agent_model or editor_agent_model.local_model()}) + + system = _CHECK_SYSTEM_PROMPT if mode == "check" else _SYSTEM_PROMPT + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": content}, + ] + + raw: Any = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if inspect.isawaitable(raw): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + parsed = _parse_plan(raw) + if _used and _used[0]: + model_used = _used[0] + elif parsed: + model_used = agent_model or editor_agent_model.local_model() + else: + model_used = "fallback" if agent_model else editor_agent_model.local_model() + if isinstance(parsed, dict): + parsed["agent_model"] = model_used + if mode != "check": + _special = _plan_special(prompt, state, _image_provider_available(owner)) + if _special: + _special_steps, _special_reply, _ = _special + return JSONResponse({ + "reply": _special_reply, + "planner": "deterministic", + "steps": style_guide.enforce_guards(_special_steps) or _special_steps, + "agent_model": model_used, + }) + if parsed and mode != "check": + _strengthened = style_guide.strengthen(prompt, list(parsed.get("steps") or [])) + if _strengthened != parsed.get("steps"): + parsed["steps"] = _strengthened + parsed["planner"] = str(parsed.get("planner") or "llm") + "+guide" + # Keep /plan and /agent consistent: a generative intent must never be + # left on a deterministic substitute (e.g. a model returning only + # selection/adjustment steps for 'remove the person on the left'). + _generative = _ensure_generative(prompt, list(parsed.get("steps") or [])) + if _generative != parsed.get("steps"): + parsed["steps"] = _generative + parsed["planner"] = str(parsed.get("planner") or "llm") + "+generative" + # Same safety guards the /agent path applies, so /plan never returns + # a stacked-lift or a global adjustment after a masked recolour. + parsed["steps"] = style_guide.enforce_guards(list(parsed.get("steps") or [])) + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + _maybe = _unsure_plan(prompt, parsed) + if _maybe is not parsed and isinstance(_maybe, dict): + _maybe["agent_model"] = model_used + parsed = _maybe + + if mode == "check": + # Never let a self-check re-plan substitute a generative placeImage + # for a deterministic recolour; re-issue the masked-hue plan instead. + _recolor = style_guide.recolor_plan(prompt) + if _recolor and _plan_uses_ai_content(list((parsed or {}).get("steps") or [])): + return JSONResponse({ + "reply": str(_recolor["reply"]), + "planner": "deterministic", + "steps": style_guide.enforce_guards(list(_recolor["steps"])), + "agent_model": model_used, + }) + if parsed and (parsed.get("done") or parsed.get("steps")): + return JSONResponse(parsed) + return JSONResponse({ + "done": True, + "reply": (parsed or {}).get("reply") or "The edit looks complete.", + "planner": (parsed or {}).get("planner") or "check", + "steps": [], + "agent_model": model_used, + }) + if parsed: + return JSONResponse(parsed) + _fallback = _fallback_plan(prompt) + _fallback["agent_model"] = model_used + return JSONResponse(_fallback) + + @router.post("/api/editor/bridge/command") + async def bridge_command(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + session = _scoped(body.get("session"), owner) + steps = _normalize_steps(body) + wait = bool(body.get("wait")) + timeout = _bridge_timeout(body.get("timeout")) + ids: List[str] = [] + if len(pending) + len(steps) > max_pending: + raise HTTPException(429, "Too many pending editor commands") + for step in steps: + cid = uuid.uuid4().hex + ids.append(cid) + fut = asyncio.get_event_loop().create_future() + pending[cid] = (owner, fut) + _broadcast(session, {"id": cid, "op": step["op"], "params": step["params"]}) + + if wait and len(steps) == 1: + cid = ids[0] + entry = pending.get(cid) + fut = entry[1] + try: + res = await asyncio.wait_for(fut, timeout=timeout) + if isinstance(res, dict): + res.setdefault("status", "completed" if res.get("ok") else "failed") + res.setdefault("completed", bool(res.get("ok"))) + return JSONResponse(res) + return JSONResponse({"id": cid, "ok": True, "status": "completed", "completed": True, "result": res}) + except asyncio.TimeoutError: + return JSONResponse({"ok": False, "error": "timeout", "status": "timed_out", "completed": False, "id": cid}, status_code=504) + finally: + _expire_command(session, cid, fut) + else: + for cid in ids: + entry = pending.get(cid) + if entry is not None: + asyncio.get_running_loop().call_later(timeout, _expire_command, session, cid, entry[1]) + + return JSONResponse({ + "queued": ids, + "session": _scope_label(session), + "status": "queued", + "completed": False, + "ok": False, + }) + + @router.get("/api/editor/bridge/events") + async def bridge_events(request: Request, session: str = "default") -> StreamingResponse: + owner = _user(request) + scoped = _scoped(session, owner) + + async def gen(): + q = _subscribe(scoped) + try: + yield ": connected\n\n" + while not _shutdown.is_shutting_down(): + try: + cmd = await asyncio.wait_for(q.get(), timeout=15) + yield f"data: {json.dumps(cmd)}\n\n" + except asyncio.TimeoutError: + yield ": keepalive\n\n" + finally: + _unsubscribe(scoped, q) + + return StreamingResponse(gen(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.post("/api/editor/bridge/result") + async def bridge_result(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + cid = str(body.get("id") or "") + entry = pending.get(cid) + if entry is None: + return JSONResponse({"ok": True}) + if entry[0] != owner: + return JSONResponse({"ok": False, "error": "unknown command"}, status_code=404) + _, fut = entry + if fut is not None and not fut.done(): + is_ok = bool(body.get("ok")) + fut.set_result({ + "id": cid, + "ok": is_ok, + "status": "completed" if is_ok else "failed", + "completed": is_ok, + "result": body.get("result"), + "error": body.get("error"), + "screenshot": body.get("screenshot"), + }) + return JSONResponse({"ok": True}) + + @router.post("/api/editor/bridge/await") + async def bridge_await(request: Request) -> JSONResponse: + owner = _user(request) + body = await _json_object(request) + cid = str(body.get("id") or "") + timeout = _bridge_timeout(body.get("timeout")) + entry = pending.get(cid) + if entry is None or entry[0] != owner: + return JSONResponse({"ok": False, "error": "unknown command", "status": "unknown", "completed": False}, status_code=404) + fut = entry[1] + try: + result = await asyncio.wait_for(asyncio.shield(fut), timeout=timeout) + if pending.get(cid) == (owner, fut): + pending.pop(cid, None) + res = dict(result) if isinstance(result, dict) else {"result": result, "ok": True} + res.setdefault("status", "completed" if res.get("ok") else "failed") + res["completed"] = bool(res.get("ok")) + return JSONResponse(res) + except asyncio.TimeoutError: + return JSONResponse({"ok": False, "error": "timeout", "status": "timed_out", "completed": False, "id": cid}) + except asyncio.CancelledError: + return JSONResponse({"ok": False, "error": "cancelled", "status": "cancelled", "completed": False, "id": cid}) + + _AGENT_SYSTEM_PROMPT = ( + "You are the photo editor agent. You drive the open image editor and " + "may also run AI image operations. Available editor ops: " + + ", ".join(OP_CATALOG) + + ", plus ai_edit_image and generate_image. " + "Respond with ONLY a JSON object: {\"reply\": , " + "\"steps\": [{\"op\": , \"params\": {...}}]}. " + + _ROUTING_GUIDANCE + " " + + _CANONICAL_GUIDANCE + " " + _STYLE_GUIDANCE + " " + "Prefer AI content ops for generative requests: \"remove the person on the left\" " + "-> op \"ai_edit_image\" op \"inpaint\" with a prompt; \"add a full moon in the sky\" " + "-> op \"generate_image\" with a prompt. " + "Use deterministic editor ops (adjustments.apply, filters.apply, selection.*, " + "layers.placeImage, text.add, ...) only for non-generative, precise edits. " + "Keep steps minimal and ordered. " + + _SCHEMA_NOTE + ) + + def _scoped_editor_tools() -> List[str]: + """The exact tool list offered to the model for a photo_editor turn.""" + fallback = [ + "plan_edit", "run_editor_command", "get_editor_screenshot", + "ai_edit_image", "generate_image", "ask_user", + ] + try: + from src.tool_policy import ToolPolicy + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS + from src.turn_contract import resolve_turn_contract + + contract = resolve_turn_contract( + capabilities={"photo_editor"}, + schemas=FUNCTION_TOOL_SCHEMAS, + policy=ToolPolicy(), + required_capabilities={"photo_editor"}, + ) + return sorted(contract.offered) or fallback + except Exception: + return fallback + + @router.post("/api/editor/agent") + async def editor_agent(request: Request) -> JSONResponse: + """Scoped photo_editor agent turn (Quick Edit). + + Offers ONLY the editor tool family plus ask_user, plans edits, and + drives the open editor through the owner-scoped bridge. AI edit and + generation actions land as new layers; the original is never + overwritten. + """ + owner = _user(request) + body = await _json_object(request) + from src.settings import get_setting + disabled = get_setting("disabled_tools", []) or [] + if not isinstance(disabled, (list, tuple, set)): + disabled = [] + if "plan_edit" in disabled: + raise HTTPException(403, "Editor tool is disabled") + prompt = _prompt(body.get("prompt")) + session = str(body.get("editor_session") or body.get("session") or "default") + state = body.get("state") if isinstance(body.get("state"), dict) else {} + screenshot = body.get("screenshot") + # Optional user-picked image model (from the editor model selector). + model_choice = str(body.get("model") or body.get("image_model") or "").strip() + # Optional user-picked PLANNER model (the editor agent/planner model). + agent_model = str(body.get("agent_model") or "").strip() + tools_offered = [tool for tool in _scoped_editor_tools() if tool not in disabled] + scoped = _scoped(session, owner) + trace: List[Dict[str, Any]] = [ + {"type": "session", "session": _scope_label(scoped), "tools_offered": tools_offered}, + ] + if not prompt: + return JSONResponse({ + "ok": False, "reply": "No prompt provided.", + "tools_offered": tools_offered, "steps": [], "trace": trace, + "agent_model": agent_model or editor_agent_model.local_model(), + }) + text = json.dumps({"prompt": prompt, "state": state}, ensure_ascii=False) + content: Any = text + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + content = [ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": {"url": screenshot}}, + ] + messages = [ + {"role": "system", "content": _AGENT_SYSTEM_PROMPT}, + {"role": "user", "content": content}, + ] + raw = None + planner = _EDITOR_PLANNER + editor_agent_model.set_override(agent_model) + editor_agent_model.set_last_model("", "") + if planner is not None: + try: + raw = planner(messages, owner) + if hasattr(raw, "__await__"): + raw = await raw + except Exception: + raw = None + _used = editor_agent_model.get_last_model() + parsed = _parse_plan(raw) + if _used and _used[0]: + model_used = _used[0] + elif parsed: + model_used = agent_model or editor_agent_model.local_model() + else: + model_used = "fallback" if agent_model else editor_agent_model.local_model() + plan = parsed or _fallback_plan(prompt) + _strengthened = style_guide.strengthen(prompt, list(plan.get("steps") or [])) + if _strengthened != plan.get("steps"): + plan = {**plan, "steps": _strengthened, + "planner": f"{plan.get('planner') or 'llm'}+guide"} + kind = _generative_kind(prompt) + provider_available = _image_provider_available(owner) + # Deterministic overrides for intents the model reliably botches: canvas + # format/crop/extend, text + layer styles, and PSD export. + _special = _plan_special(prompt, state, provider_available) + special_note = "" + if _special: + special_steps, special_reply, special_note = _special + steps = style_guide.enforce_guards(special_steps) or list(special_steps) + reply = special_reply + kind = None + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+deterministic"} + else: + steps = _ensure_generative(prompt, list(plan.get("steps") or [])) + steps = style_guide.enforce_guards(steps) + if steps != (plan.get("steps") or []): + plan = {**plan, "steps": steps, + "planner": f"{plan.get('planner') or 'llm'}+generative"} + reply = str(plan.get("reply") or "") + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. + # A bare whole-image tone plan for an unspecified prompt is a guess: + # ask the user instead of silently tweaking. Only replace when the + # guard actually changed the plan (identity), so a model-supplied + # ask_user never wipes out a concrete generative edit. + _base_plan = {**plan, "steps": steps} + _maybe = _unsure_plan(prompt, _base_plan) + if _maybe is not _base_plan and isinstance(_maybe, dict) and _maybe.get("ask_user"): + plan = _maybe + steps = [] + reply = str(_maybe.get("reply") or "") + trace.append({ + "type": "plan", "tool": "plan_edit", + "planner": plan.get("planner"), "steps": steps, + "image_provider": provider_available, "agent_model": model_used, + }) + # A vague prompt with no concrete defect asks the user instead of + # guessing a token edit (2-3 concrete options). + ask_user_payload = plan.get("ask_user") + if isinstance(ask_user_payload, dict) and not steps and not _special: + trace.append({"type": "ask_user", + "question": ask_user_payload.get("question")}) + return JSONResponse({ + "ok": True, + "ask_user": ask_user_payload, + "ask_user_pending": True, + "reply": reply or str(ask_user_payload.get("question") or ""), + "tools_offered": tools_offered, + "steps": [], + "trace": trace, + "results": [], + "agent_model": model_used, + }) + # This route dispatches tools directly, outside execute_tool_block. + # Apply the same administrator tool switches before any step runs. + needed = {"plan_edit"} + for step in steps: + op = step.get("op") + needed.add(op if op in {"ai_edit_image", "generate_image"} else "run_editor_command") + if needed.intersection(disabled): + raise HTTPException(403, "Editor tool is disabled") + # No image provider: stop before any edit runs and tell the user plainly + # (zero canvas changes; nothing to roll back). + if _plan_uses_ai_content(steps): + from src.auth_helpers import require_privilege + require_privilege(request, "can_generate_images") + if (kind or _plan_uses_ai_content(steps)) and not provider_available: + trace.append({"type": "blocked", "reason": "image_provider_unavailable"}) + return JSONResponse({ + "ok": False, + "blocked": True, + "needs_provider": True, + "reply": _PROVIDER_REQUIRED_MESSAGE, + "message": _PROVIDER_REQUIRED_MESSAGE, + "image_provider": False, + "tools_offered": tools_offered, + "steps": steps, + "trace": trace, + "results": [], + "agent_model": model_used, + }) + needed = {"plan_edit"} + for step in steps: + op = step.get("op") + needed.add(op if op in {"ai_edit_image", "generate_image"} else "run_editor_command") + if needed.intersection(disabled): + raise HTTPException(403, "Editor tool is disabled") + results: List[Dict[str, Any]] = [] + for step in steps: + op = step.get("op") + params = step.get("params") or {} + if op in ("ai_edit_image", "generate_image"): + try: + from src.agent_tools.editor_tools import AiEditImageTool, GenerateImageLayerTool + except Exception: + results.append({"op": op, "ok": False, "error": "editor tools unavailable"}) + break + tool_payload = dict(params) + # Planner output is untrusted: it cannot redirect this turn to + # another editor session or override an explicit model choice. + tool_payload["session"] = session + # Route to the user-picked model when one is selected. + if model_choice: + tool_payload["model"] = model_choice + # Hand the AI step the canvas image the client already captured + # for this request. Without it the tool falls back to a separate + # bridge screenshot, which is not reliably answered right after a + # selection step (the masked path was losing the canvas image). + if isinstance(screenshot, str) and screenshot.startswith("data:image/"): + tool_payload["image"] = screenshot + else: + tool_payload.pop("image", None) + tool_cls = AiEditImageTool if op == "ai_edit_image" else GenerateImageLayerTool + try: + _desc, res = await tool_cls().execute( + json.dumps(tool_payload), + {"owner": owner, "session_id": _scope_label(scoped), "editor_session": session, + "incognito": body.get("incognito") is True}, + ) + except Exception: + logger.exception("Editor image operation failed") + res = {"error": "Editor image operation failed", "exit_code": 1} + res = res if isinstance(res, dict) else {} + ok = res.get("exit_code", 0) == 0 + step_result = { + k: res.get(k) for k in ("image_id", "image_url", "placed", "output", "model") + if k in res + } + results.append({ + "op": op, "ok": ok, "result": step_result, + # Surface the real provider/tool failure so the reply can + # say what went wrong instead of a bare ok:true/ok:false. + "error": None if ok else (res.get("error") or res.get("output") or "it did not complete"), + }) + trace.append({"type": "step", "op": op, "ok": ok, "result": step_result}) + if not ok: + break + continue + cid = uuid.uuid4().hex + fut = asyncio.get_event_loop().create_future() + pending[cid] = (owner, fut) + _broadcast(scoped, {"id": cid, "op": op, "params": params}) + try: + res = await asyncio.wait_for(pending[cid][1], timeout=_BRIDGE_DEFAULT_TIMEOUT) + except asyncio.TimeoutError: + results.append({"op": op, "ok": False, "error": "timeout"}) + trace.append({"type": "step", "op": op, "ok": False, "error": "timeout"}) + break + finally: + _expire_command(scoped, cid, fut) + results.append(res) + trace.append({"type": "step", "op": op, "ok": res.get("ok"), "result": res.get("result")}) + if res.get("ok") is False: + break + # Post-run verification: read the ACTUAL canvas back and base the reply + # on executed results + final document.info, never the plan. + final_state: Dict[str, Any] = {} + if steps and not any(r.get("ok") is False for r in results): + try: + _info_id = uuid.uuid4().hex + _info_fut = asyncio.get_event_loop().create_future() + pending[_info_id] = (owner, _info_fut) + _broadcast(scoped, {"id": _info_id, "op": "document.info", "params": {}}) + _info = await asyncio.wait_for(pending[_info_id][1], timeout=60) + if isinstance(_info, dict) and isinstance(_info.get("result"), dict): + final_state = _info["result"] + except Exception: + final_state = {} + finally: + _expire_command(scoped, _info_id, _info_fut) + planned_reply = reply + reply = _verified_reply(prompt, steps, results, final_state, planned_reply) + if special_note and special_note.strip() not in reply: + reply = (reply + " " + special_note.strip()).strip() + return JSONResponse({ + "ok": not any(r.get("ok") is False for r in results), + "reply": reply or "Done.", + "planned_reply": planned_reply, + "image_provider": provider_available, + "final_state": final_state, + "tools_offered": tools_offered, + "tools": ["plan_edit"], + "steps": steps, + "trace": trace, + "results": results, + "agent_model": model_used, + }) + + @router.get("/api/editor/style-guide") + async def style_guide_route() -> JSONResponse: + """The shared intent -> ops guide (single source of truth). + + The browser fallback planner fetches this so the client and server + routing rules cannot drift. + """ + return JSONResponse(style_guide.as_json()) + + @router.get("/api/editor/image-models") + async def editor_image_models(request: Request) -> JSONResponse: + """List the image models available from configured, owner-visible providers. + + Each entry is ``{id, label, kind, provider}`` where ``id`` is the model id + to pass as ``model`` to ai_edit_image / generate_image, and ``kind`` is + ``edit`` (editing/inpainting models) or ``generate`` (text-to-image). + """ + owner = _user(request) + is_admin = _is_admin(request) + models: List[Dict[str, Any]] = [] + seen: set = set() + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter + from src.endpoint_resolver import _endpoint_enabled_models + from src.image_model_ids import looks_like_image_generation_model + + db = SessionLocal() + try: + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + for ep in query.all(): + # Image-only endpoints (the local Qwen server) list their + # models verbatim. Gateway/chat endpoints (OpenRouter, …) + # mix chat and image models, so expose only ids that are + # recognisably image-capable — the same test the request + # router uses. + image_endpoint = getattr(ep, "model_type", None) == "image" + endpoint_id = getattr(ep, "id", None) + provider = getattr(ep, "name", None) + if not provider: + provider = ( + getattr(ep, "base_url", "") + if is_admin + else (f"Endpoint {endpoint_id}" if endpoint_id else "image") + ) + provider = provider or "image" + for name in _endpoint_enabled_models(ep): + if not image_endpoint and not looks_like_image_generation_model(name): + continue + key = (str(endpoint_id or provider), name) + if key in seen: + continue + seen.add(key) + entry = { + "id": name, + "label": name, + "kind": "edit" if _EDIT_MODEL_RE.search(name) else "generate", + "provider": provider, + "endpoint_id": endpoint_id, + } + if is_admin: + entry["base_url"] = getattr(ep, "base_url", "") or "" + models.append(entry) + finally: + db.close() + except Exception: + logger.warning("editor image-models lookup failed", exc_info=True) + return JSONResponse({"models": models}) + + @router.get("/api/editor/agent-models") + async def editor_agent_models(request: Request) -> JSONResponse: + """List the models the editor agent/planner can run on. + + Each entry is ``{id, label, source, vision}`` where ``id`` is + passed as ``agent_model`` to /api/editor/plan or /api/editor/agent, and + ``source`` is ``local`` (the local Qwen3.5-9B vision server) or + ``hosted`` (the configured default/utility chat model that the planner + falls back to). + For non-admin users, private URLs and internal probe status are omitted. + """ + owner = _user(request) + is_admin = _is_admin(request) + try: + models = await editor_agent_model.list_editor_agent_models(owner) + if not is_admin: + sanitized = [] + for m in models: + m_copy = dict(m) + m_copy.pop("base_url", None) + sanitized.append(m_copy) + models = sanitized + except Exception: + logger.warning("editor agent-models lookup failed", exc_info=True) + models = [] + + probe_info = {} + if is_admin: + probe_info = editor_agent_model.probe_status() + if not probe_info: + editor_agent_model.startup_probe() + probe_info = editor_agent_model.probe_status() + + return JSONResponse({ + "models": models, + "probe": probe_info, + }) + + @router.post("/api/editor/self-check") + async def editor_self_check(request: Request) -> JSONResponse: + """Same-origin VL self-check: compare BEFORE/AFTER against the intent. + + A configured vision server may not provide browser CORS headers. This route + proxies the two images + intent to the agent VLM and returns + ``{done, issues, retry_hint, text}`` (``done=True`` with + ``reason="unavailable"`` on any failure so the client never loops). + """ + owner = _user(request) + body = await _json_object(request) + result = await editor_agent_model.self_check( + body.get("before"), + body.get("after"), + _prompt(body.get("prompt") or body.get("intent")), + model=str(body.get("model") or "").strip() or None, + owner=owner, + ) + return JSONResponse(result) + + return router diff --git a/routes/email/email_routes.py b/routes/email/email_routes.py index 172790a6c..6060cd568 100644 --- a/routes/email/email_routes.py +++ b/routes/email/email_routes.py @@ -25,13 +25,16 @@ import json import re import html import io +import tempfile import zipfile -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse from html.parser import HTMLParser as _HTMLParser import logging import uuid -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path +from email.message import EmailMessage +from email import policy as email_policy from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart @@ -40,6 +43,7 @@ from starlette.concurrency import run_in_threadpool from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request from fastapi.responses import FileResponse, StreamingResponse from src.constants import DATA_DIR +from src import shutdown as _shutdown from src.path_confinement import confine from src.llm_core import llm_call_async @@ -84,6 +88,17 @@ _SERVER_OWNED_OAUTH_FIELDS = { _CACHE_HOOKS: dict = {"invalidate_list": None} +def _check_mail_host(host, owner: str, label: str) -> str | None: + """Reject unsafe or unresolved mail hosts before saving or testing them.""" + from src.url_safety import check_outbound_host + ok, reason = check_outbound_host(host, block_private=_mail_private_blocked(owner)) + if ok: + return None + logger.warning("Rejected %s mail host for owner=%r: %s", label, owner, reason) + # Resolver and exception details belong in server logs, never the response. + return f"Rejected {label} host: server address is not allowed or does not resolve" + + def _normalized_mail_host(value) -> str: """Normalize a mail hostname for exact provider-bound comparisons.""" return str(value or "").strip().lower().rstrip(".") @@ -636,10 +651,15 @@ def _message_id_chain(*values: str) -> list[str]: def _uid_from_fetch_meta(meta_b: bytes) -> str: - m = re.search(rb"\bUID\s+(\d+)\b", meta_b) + m = re.search(rb"\bUID\s+(\d+)\b", meta_b, re.IGNORECASE) return m.group(1).decode() if m else "" +def _size_from_fetch_meta(meta_b: bytes) -> int | None: + m = re.search(rb"\bRFC822\.SIZE\s+(\d+)\b", meta_b, re.IGNORECASE) + return int(m.group(1)) if m else None + + def _parse_list_unsubscribe_header(value: str | None) -> list[dict]: """Parse RFC List-Unsubscribe entries into safe reviewable actions. @@ -1807,20 +1827,75 @@ def setup_email_routes(): # a cached message body. return (account_id or "", folder, str(uid), owner) + _LIST_STALE = {} + _LIST_LOCK = _threading.RLock() + _LIST_CACHE_LOCK = _LIST_LOCK + # Invalidation fences reads already in flight, including background refresh. + _LIST_EPOCH = [0] + _LIST_CACHE_GENERATION = _LIST_EPOCH + def _list_cache_get(key): - v = _LIST_CACHE.get(key) - if not v: return None - if v[0] < _time.monotonic(): - _LIST_CACHE.pop(key, None) - return None - return v[1] + with _LIST_LOCK: + v = _LIST_CACHE.get(key) + if not v: return None + if v[0] < _time.monotonic(): + _LIST_CACHE.pop(key, None) + return None + return v[1] + + def _list_stale_get(key): + with _LIST_LOCK: + return _LIST_STALE.get(key) + + def _list_stale_put(key, value): + _LIST_STALE[key] = value + if len(_LIST_STALE) > 96: + for k in list(_LIST_STALE.keys())[:-48]: + _LIST_STALE.pop(k, None) + + _LIST_REFRESHING = set() def _list_cache_put(key, value): - _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value) - # Cap size - if len(_LIST_CACHE) > 64: - for k in list(_LIST_CACHE.keys())[:-32]: - _LIST_CACHE.pop(k, None) + with _LIST_LOCK: + _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value) + # Also snapshot the last-good response for stale-while-revalidate. + _list_stale_put(key, value) + if len(_LIST_CACHE) > 64: + for k in list(_LIST_CACHE.keys())[:-32]: + _LIST_CACHE.pop(k, None) + + def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner): + """Refresh a list snapshot off the event loop, at most once per key. + + Runs the blocking IMAP fetch in a daemon thread, then repopulates both + the TTL cache and the stale snapshot so the next list view is warm. + """ + with _LIST_LOCK: + if cache_key in _LIST_REFRESHING: + return + _LIST_REFRESHING.add(cache_key) + epoch = _LIST_EPOCH[0] + + def _run(): + try: + refreshed = _list_emails_sync(*sync_args) + if refreshed and not refreshed.get("error"): + with _LIST_LOCK: + if epoch != _LIST_EPOCH[0]: + return + _list_cache_put(cache_key, refreshed) + try: + _schedule_recent_email_warm( + refreshed.get("emails") or [], folder, account_id, owner) + except Exception: + pass + except Exception as exc: # pragma: no cover - background best effort + logger.debug("email list background refresh failed: %s", exc) + finally: + with _LIST_LOCK: + _LIST_REFRESHING.discard(cache_key) + + _threading.Thread(target=_run, daemon=True, name="email-list-refresh").start() def _folder_cache_get(account_id, owner): key = (account_id or "", owner or "") @@ -1855,37 +1930,74 @@ def setup_email_routes(): hook(account_id, folder) except Exception: pass - if account_id is None and folder is None: - _LIST_CACHE.clear() - return - for k in list(_LIST_CACHE.keys()): - k_acct = k[0] if len(k) > 0 else "" - k_folder = k[1] if len(k) > 1 else "" - if (account_id is None or k_acct == (account_id or "")) and \ - (folder is None or k_folder == folder): - _LIST_CACHE.pop(k, None) + with _LIST_LOCK: + _LIST_EPOCH[0] += 1 + if account_id is None and folder is None: + _LIST_CACHE.clear() + _LIST_STALE.clear() + return + for cache in (_LIST_CACHE, _LIST_STALE): + for k in list(cache.keys()): + k_acct = k[0] if len(k) > 0 else "" + k_folder = k[1] if len(k) > 1 else "" + if (account_id is None or k_acct == (account_id or "")) and \ + (folder is None or k_folder == folder): + cache.pop(k, None) + + def _index_update_flags(owner, account_id, folder, uid, flag, add): + with _LIST_LOCK: + _email_index_update_flags(owner, account_id, folder, uid, flag, add) + _invalidate_list_cache(account_id, folder) + + def _index_delete(owner, account_id, folder, uid): + with _LIST_LOCK: + _email_index_delete(owner, account_id, folder, uid) + _invalidate_list_cache(account_id, folder) def _update_list_cache_seen(account_id, folder, uid, seen: bool): uid_s = str(uid) - for key, (expires_at, value) in list(_LIST_CACHE.items()): - if key[0] != (account_id or "") or key[1] != folder: - continue - emails = list((value or {}).get("emails") or []) - changed = False - if seen and key[2] == "unread": - kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] - if len(kept) != len(emails): - value = dict(value) - value["emails"] = kept - value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) - changed = True - else: - for e in emails: - if str((e or {}).get("uid") or "") == uid_s: - e["is_read"] = bool(seen) + with _LIST_LOCK: + for key, (expires_at, value) in list(_LIST_CACHE.items()): + if key[0] != (account_id or "") or key[1] != folder: + continue + emails = list((value or {}).get("emails") or []) + changed = False + if seen and key[2] == "unread": + kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] + if len(kept) != len(emails): + value = dict(value) + value["emails"] = kept + value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) changed = True - if changed: - _LIST_CACHE[key] = (expires_at, value) + else: + for e in emails: + if str((e or {}).get("uid") or "") == uid_s: + e["is_read"] = bool(seen) + changed = True + if changed: + _LIST_CACHE[key] = (expires_at, value) + if key in _LIST_STALE: + _LIST_STALE[key] = value + + for key, value in list(_LIST_STALE.items()): + if key[0] != (account_id or "") or key[1] != folder: + continue + emails = list((value or {}).get("emails") or []) + changed = False + if seen and key[2] == "unread": + kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s] + if len(kept) != len(emails): + value = dict(value) + value["emails"] = kept + value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept))) + changed = True + else: + for e in emails: + if str((e or {}).get("uid") or "") == uid_s: + e["is_read"] = bool(seen) + changed = True + if changed: + _LIST_STALE[key] = value def _read_cache_get(key): v = _READ_CACHE.get(key) @@ -2265,6 +2377,8 @@ def setup_email_routes(): SECURITY: `owner` is propagated so when `account_id` is missing, the fallback config lookup is scoped to this user's accounts only. """ + with _LIST_CACHE_LOCK: + index_generation = _LIST_CACHE_GENERATION[0] conn = None conn_ok = False try: @@ -2559,7 +2673,9 @@ def setup_email_routes(): if parsed: fetched_emails.append(parsed) cached_rows[parsed["uid"]] = parsed - _email_index_upsert(owner, account_id, folder, fetched_emails) + with _LIST_CACHE_LOCK: + if index_generation == _LIST_CACHE_GENERATION[0]: + _email_index_upsert(owner, account_id, folder, fetched_emails) for uid_num in uid_order: row = cached_rows.get(uid_num) if not row: @@ -2848,6 +2964,48 @@ def setup_email_routes(): owner, account_id or "", folder, filter, limit, offset, elapsed_ms, ) return cached + # Stale-while-revalidate: never block a plain list view on IMAP while + # we already hold a recent snapshot or a durable index. Serve it now + # with a `stale` marker and refresh in the background so the next view + # is warm. Blocking IMAP only remains for a genuine cold start with no + # snapshot and no indexed rows (e.g. a brand-new account/folder). + if not manual_refresh and not from_addr and not has_attachments and not (date_from or date_to): + snapshot = _list_stale_get(ck) + if snapshot is None: + try: + indexed_emails, indexed_total, indexed_at = _email_index_list( + owner, account_id, folder, filter, limit, offset, False, + ) + except Exception: + indexed_emails, indexed_total, indexed_at = [], 0, None + if indexed_total: + snapshot = { + "emails": indexed_emails, + "total": indexed_total, + "folder": folder, + "offset": offset, + "sync": { + "source": "index", + "indexed": indexed_total, + "updated_at": indexed_at, + }, + } + if snapshot is not None: + payload = dict(snapshot) + payload["emails"] = list(payload.get("emails") or []) + _hide_unlinked_calendar_tags(payload["emails"]) + sync_meta = dict(payload.get("sync") or {}) + sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index" + sync_meta["stale"] = True + payload["sync"] = sync_meta + _schedule_list_refresh( + (folder, limit, offset, filter, account_id, from_addr, + bool(has_attachments), owner, True, date_from or "", date_to or ""), + ck, folder, account_id, owner, + ) + return payload + with _LIST_CACHE_LOCK: + cache_generation = _LIST_CACHE_GENERATION[0] result = await _asyncio.to_thread( _list_emails_sync, folder, limit, offset, filter, account_id, from_addr, bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "", @@ -2856,7 +3014,9 @@ def setup_email_routes(): if offset == 0 and not from_addr and not has_attachments and filter in ("all", "unread", "unanswered", "undone"): _record_email_received_events(owner, account_id, folder, result.get("emails") or []) _schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner) - _list_cache_put(ck, result) + with _LIST_CACHE_LOCK: + if cache_generation == _LIST_CACHE_GENERATION[0]: + _list_cache_put(ck, result) elapsed_ms = int((_time.monotonic() - started_at) * 1000) if elapsed_ms > 1500: logger.warning( @@ -3204,7 +3364,7 @@ def setup_email_routes(): conn.select(_q(folder)) moved = _move_email_message(conn, uid, "Junk", role="junk") if moved: - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) except Exception: logger.debug("unsubscribe move-to-spam skipped", exc_info=True) @@ -3221,7 +3381,7 @@ def setup_email_routes(): if deleted: conn.expunge() if deleted: - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) except Exception as exc: delete_error = "Email was unsubscribed but could not be moved to Trash" @@ -3303,7 +3463,7 @@ def setup_email_routes(): if moved: changed += 1 cleaned_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) else: failed += 1 except Exception: @@ -3392,6 +3552,8 @@ def setup_email_routes(): folder (Gmail does), we transparently swap to All Mail so the search surfaces archived / labelled emails too. Plain IMAP accounts fall back to whatever folder the caller specified.""" + with _LIST_CACHE_LOCK: + index_generation = _LIST_CACHE_GENERATION[0] if not q or len(q) < 2: return {"emails": [], "total": 0, "query": q} # CRLF in q would terminate the IMAP command early — reject defensively. @@ -3535,7 +3697,9 @@ def setup_email_routes(): except Exception as e: logger.warning(f"Error parsing search result {uid}: {e}") continue - _email_index_upsert(owner, account_id, effective_folder, fetched_emails) + with _LIST_CACHE_LOCK: + if index_generation == _LIST_CACHE_GENERATION[0]: + _email_index_upsert(owner, account_id, effective_folder, fetched_emails) return { "emails": emails, @@ -3672,8 +3836,7 @@ def setup_email_routes(): # accepted it, so the index and list cache cannot drift ahead of # the mailbox. if mark_seen and not mark_seen_failed: - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) - _update_list_cache_seen(account_id, folder, uid, True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) related_attachments = [] if full and not _has_visible_attachments(msg): @@ -3812,8 +3975,7 @@ def setup_email_routes(): status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)") if status != "OK": return False - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) - _update_list_cache_seen(account_id, folder, uid, True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) return True except Exception as e: logger.warning(f"mark-seen after cached read failed uid={uid}: {e}") @@ -4185,6 +4347,414 @@ def setup_email_routes(): headers={"Content-Disposition": f'attachment; filename="{zip_name}"'}, ) + @router.post("/messages-download-bulk") + async def download_bulk_messages(request: Request, owner: str = Depends(require_owner)): + """Export selected original email messages as .eml files in a ZIP.""" + try: + payload = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + messages = payload.get("messages") if isinstance(payload, dict) else None + if not isinstance(messages, list) or not messages: + raise HTTPException(status_code=400, detail="No emails selected") + if len(messages) > 250: + raise HTTPException(status_code=400, detail="Too many emails selected") + + category = str(payload.get("category") or "selected").strip().lower() + category = re.sub(r"[^a-z0-9_-]+", "-", category).strip("-") or "selected" + selected_dates = sorted({ + str(row.get("date") or "").strip() + for row in messages if isinstance(row, dict) + and re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(row.get("date") or "").strip()) + }) + date_span = "" + if selected_dates: + date_span = selected_dates[0] if len(selected_dates) == 1 else f"{selected_dates[0]} - {selected_dates[-1]}" + export_folder = f"emails-{category}" + (f" ({date_span})" if date_span else "") + + # 512 MiB safety cap prevents unbounded memory growth when exporting + # many large messages. + max_total_bytes = 512 * 1024 * 1024 + max_message_bytes = 64 * 1024 * 1024 + spool_threshold_bytes = 32 * 1024 * 1024 + used_names: set[str] = set() + exported = 0 + skipped = 0 + total_bytes = 0 + validated_accounts: set[str] = set() + + def _email_filename(row: dict, uid: str) -> str: + date = str(row.get("date") or "").strip() + date = date if re.fullmatch(r"\d{4}-\d{2}-\d{2}", date) else "" + subject = str(row.get("subject") or "email").strip() + subject = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]+', "_", subject) + subject = re.sub(r"\s+", " ", subject).strip(" .")[:100] or "email" + safe_uid = re.sub(r"[^A-Za-z0-9_-]+", "_", uid).strip("_") or "message" + prefix = f"{date} - " if date else "" + return _safe_attachment_zip_name(f"{prefix}{subject} - {safe_uid}.eml", "email.eml") + + def _fixture_message_bytes(fixture: dict, uid: str, folder: str) -> bytes: + msg = EmailMessage() + sender_name = str(fixture.get("from_name") or "").strip() + sender_address = str(fixture.get("from_address") or "").strip() + sender = email.utils.formataddr((sender_name, sender_address)) if sender_address else sender_name + for key, value in ( + ("From", sender), + ("To", fixture.get("to") or owner), + ("Subject", fixture.get("subject") or "(no subject)"), + ("Message-ID", fixture.get("message_id") or f""), + ): + clean_value = re.sub(r"[\r\n]+", " ", str(value or "")).strip() + if clean_value: + msg[key] = clean_value + raw_date = str(fixture.get("date") or "").strip() + if raw_date: + try: + parsed_date = datetime.fromisoformat(raw_date.replace("Z", "+00:00")) + if parsed_date.tzinfo is None: + parsed_date = parsed_date.replace(tzinfo=timezone.utc) + except Exception: + try: + parsed_date = email.utils.parsedate_to_datetime(raw_date) + except Exception: + parsed_date = None + if parsed_date is not None: + msg["Date"] = email.utils.format_datetime(parsed_date) + msg.set_content(str(fixture.get("body") or "")) + for attachment in fixture.get("attachments") or []: + try: + index = int(attachment.get("index")) + source = _fixture_attachment_source(uid, index, owner, folder) + if not source: + continue + _row, raw_attachment = source + content_type = str(raw_attachment.get("content_type") or "application/octet-stream") + maintype, _, subtype = content_type.partition("/") + if not subtype: + maintype, subtype = "application", "octet-stream" + msg.add_attachment( + str(raw_attachment.get("content") or "").encode("utf-8"), + maintype=maintype, + subtype=subtype, + filename=str(raw_attachment.get("filename") or attachment.get("filename") or f"attachment-{index}"), + ) + except Exception: + logger.debug("Skipping malformed fixture email attachment uid=%s", uid, exc_info=True) + return msg.as_bytes(policy=email_policy.SMTP) + + # Optimization: single message exports return raw .eml directly without ZIP compression. + if len(messages) == 1: + row = messages[0] + if not isinstance(row, dict): + raise HTTPException(status_code=404, detail="No selected emails could be exported") + uid = str(row.get("uid") or "").strip() + if not uid: + raise HTTPException(status_code=404, detail="No selected emails could be exported") + folder = str(row.get("folder") or "INBOX").strip() or "INBOX" + account_id = str(row.get("account_id") or row.get("account") or "").strip() or None + if account_id: + _assert_owns_account(account_id, owner) + + filename = _email_filename(row, uid) + message_bytes: bytes | None = None + + fixture = _fixture_email_read(uid, folder, owner) + if fixture is not None and not fixture.get("error"): + message_bytes = _fixture_message_bytes(fixture, uid, folder) + elif re.fullmatch(r"\d+", uid): + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder), readonly=True) + uid_b = uid.encode() + st_sz, sz_data = None, None + try: + st_sz, sz_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822.SIZE)") + except Exception as sz_err: + logger.warning("Single message size preflight query error for uid=%s: %s", uid, sz_err) + + if st_sz != "OK": + logger.warning("Single message size preflight status=%s for uid=%s", st_sz, uid) + raise HTTPException(status_code=502, detail="Unable to verify email size safety") + + if not sz_data: + # Message does not exist in mailbox (missing mail) + raise HTTPException(status_code=404, detail="No selected emails could be exported") + + sz: int | None = None + for item in sz_data: + meta = item[0] if isinstance(item, tuple) else item + if isinstance(meta, bytes): + u = _uid_from_fetch_meta(meta) + if u == uid: + sz = _size_from_fetch_meta(meta) + break + + if sz is None or sz < 0: + logger.warning("Single message size preflight returned invalid/malformed size for uid=%s", uid) + raise HTTPException(status_code=502, detail="Invalid email size metadata from server") + + if sz > max_message_bytes or sz > max_total_bytes: + raise HTTPException(status_code=413, detail="Export too large") + + # Preflight size safety established; perform RFC822 fetch. + st, msg_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822)") + if st == "OK" and msg_data: + for meta, raw in _group_uid_fetch_records(msg_data): + if _uid_from_fetch_meta(meta) == uid and raw: + message_bytes = bytes(raw) + break + except HTTPException: + raise + except Exception as exc: + logger.warning("Single email export failed for uid=%s folder=%s: %s", uid, folder, exc) + + if not message_bytes: + raise HTTPException(status_code=404, detail="No selected emails could be exported") + + if len(message_bytes) > max_message_bytes or len(message_bytes) > max_total_bytes: + raise HTTPException(status_code=413, detail="Export too large") + + return StreamingResponse( + io.BytesIO(message_bytes), + media_type="message/rfc822", + headers={ + "Content-Disposition": f"attachment; filename=\"email.eml\"; filename*=UTF-8''{quote(filename, safe='')}", + "X-Exported-Emails": "1", + "X-Skipped-Emails": "0", + }, + ) + + seen_items: set[tuple[str | None, str, str]] = set() + remote_groups: dict[tuple[str | None, str], dict[str, tuple[str, dict]]] = {} + spooled_file = tempfile.SpooledTemporaryFile(max_size=spool_threshold_bytes, mode="w+b") + response_started = False + try: + with zipfile.ZipFile(spooled_file, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for row in messages: + if not isinstance(row, dict): + skipped += 1 + continue + uid = str(row.get("uid") or "").strip() + if not uid: + skipped += 1 + continue + folder = str(row.get("folder") or "INBOX").strip() or "INBOX" + account_id = str(row.get("account_id") or row.get("account") or "").strip() or None + if account_id and account_id not in validated_accounts: + _assert_owns_account(account_id, owner) + validated_accounts.add(account_id) + + item_key = (account_id, folder, uid) + if item_key in seen_items: + skipped += 1 + continue + seen_items.add(item_key) + + fixture = _fixture_email_read(uid, folder, owner) + if fixture is not None and not fixture.get("error"): + filename = _email_filename(row, uid) + unique_name = filename + suffix = 2 + while unique_name.lower() in used_names: + unique_name = f"{Path(filename).stem}-{suffix}.eml" + suffix += 1 + used_names.add(unique_name.lower()) + message_bytes = _fixture_message_bytes(fixture, uid, folder) + if len(message_bytes) > max_message_bytes: + raise HTTPException(status_code=413, detail="Export too large") + total_bytes += len(message_bytes) + if total_bytes > max_total_bytes: + raise HTTPException(status_code=413, detail="Export too large") + try: + zf.writestr(f"{export_folder}/{unique_name}", message_bytes) + except Exception as zip_err: + logger.error("ZIP entry write failed for fixture: %s", zip_err) + raise HTTPException(status_code=500, detail="Failed to create export archive") + exported += 1 + continue + if not re.fullmatch(r"\d+", uid): + skipped += 1 + continue + group = remote_groups.setdefault((account_id, folder), {}) + group.setdefault(uid, (_email_filename(row, uid), row)) + + preflight_error: HTTPException | None = None + + # Fetch original RFC822 data in batches and store it byte-for-byte. + # Note on IMAP memory-safety: Python's standard imaplib buffers socket responses into + # memory before returning literals to application code. Preflight verification ensures + # we do not intentionally fetch messages exceeding limits, but an untrusted or non-compliant + # IMAP server sending unexpectedly large literals cannot be bounded at the transport level + # without custom socket streaming. Post-fetch checks guard application-level storage. + for (account_id, folder), rows_by_uid in remote_groups.items(): + rows = list(rows_by_uid.items()) + remaining_rows = len(rows) + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder), readonly=True) + + # Size preflight query to catch oversized exports early and bound batch sizes. + uid_set_all = ",".join(uid for uid, _ in rows).encode() + st_sz, sz_data = None, None + try: + st_sz, sz_data = _imap_uid_fetch(conn, uid_set_all, "(UID RFC822.SIZE)") + except Exception as sz_err: + logger.warning("IMAP size preflight query failed for folder=%s: %s", folder, sz_err) + + if st_sz != "OK": + logger.warning("IMAP size preflight status=%s for folder=%s account=%s", st_sz, folder, account_id) + preflight_error = HTTPException(status_code=502, detail="Unable to verify email size safety") + skipped += remaining_rows + remaining_rows = 0 + continue + + if not sz_data: + # All requested UIDs in this mailbox are missing on server + skipped += remaining_rows + remaining_rows = 0 + continue + + reported_sizes: dict[str, int | None] = {} + for item in sz_data: + meta = item[0] if isinstance(item, tuple) else item + if isinstance(meta, bytes): + u = _uid_from_fetch_meta(meta) + if u: + reported_sizes[u] = _size_from_fetch_meta(meta) + + # Categorize requested UIDs + malformed_uids = [ + uid for uid, _ in rows + if uid in reported_sizes and (reported_sizes[uid] is None or reported_sizes[uid] < 0) + ] + if malformed_uids: + logger.warning("IMAP size preflight returned invalid/malformed size for uids=%s in folder=%s", malformed_uids, folder) + preflight_error = HTTPException(status_code=502, detail="Invalid email size metadata from server") + skipped += remaining_rows + remaining_rows = 0 + continue + + # Identify missing mail (requested UIDs not returned in preflight) + missing_uids = {uid for uid, _ in rows if uid not in reported_sizes} + if missing_uids: + skipped += len(missing_uids) + remaining_rows -= len(missing_uids) + rows = [item for item in rows if item[0] not in missing_uids] + + if not rows: + continue + + size_by_uid: dict[str, int] = { + uid: reported_sizes[uid] for uid, _ in rows + } + + for u, s in size_by_uid.items(): + if s > max_message_bytes: + raise HTTPException(status_code=413, detail="Export too large") + if total_bytes + sum(size_by_uid.values()) > max_total_bytes: + raise HTTPException(status_code=413, detail="Export too large") + + # Form batches with at most 20 messages and capped at 10 MiB payload. + batches: list[list[tuple[str, tuple[str, dict]]]] = [] + cur_batch: list[tuple[str, tuple[str, dict]]] = [] + cur_batch_bytes = 0 + max_batch_bytes = 10 * 1024 * 1024 + for item in rows: + uid_val = item[0] + sz = size_by_uid[uid_val] + if cur_batch and ( + len(cur_batch) >= 20 or + (cur_batch_bytes + sz > max_batch_bytes) + ): + batches.append(cur_batch) + cur_batch = [] + cur_batch_bytes = 0 + cur_batch.append(item) + cur_batch_bytes += sz + if cur_batch: + batches.append(cur_batch) + + for batch in batches: + wanted = {uid: filename for uid, (filename, _row) in batch} + uid_set = ",".join(wanted).encode() + status, msg_data = _imap_uid_fetch(conn, uid_set, "(UID RFC822)") + if status != "OK" or not msg_data: + skipped += len(batch) + remaining_rows -= len(batch) + continue + found: set[str] = set() + for meta, raw_message in _group_uid_fetch_records(msg_data): + uid = _uid_from_fetch_meta(meta) + if not uid or uid not in wanted or not raw_message: + continue + if uid in found: + # P2: Reject duplicate IMAP response records for an already-processed UID + logger.debug("Ignoring duplicate IMAP FETCH record for uid=%s in folder=%s", uid, folder) + continue + filename = wanted[uid] + unique_name = filename + suffix = 2 + while unique_name.lower() in used_names: + unique_name = f"{Path(filename).stem}-{suffix}.eml" + suffix += 1 + used_names.add(unique_name.lower()) + message_bytes = bytes(raw_message) + if len(message_bytes) > max_message_bytes: + raise HTTPException(status_code=413, detail="Export too large") + total_bytes += len(message_bytes) + if total_bytes > max_total_bytes: + raise HTTPException(status_code=413, detail="Export too large") + try: + zf.writestr(f"{export_folder}/{unique_name}", message_bytes) + except Exception as zip_err: + logger.error("ZIP entry write failed for remote message: %s", zip_err) + raise HTTPException(status_code=500, detail="Failed to create export archive") + found.add(uid) + exported += 1 + remaining_rows -= 1 + missing = len(batch) - len(found) + skipped += len(batch) - len(found) + remaining_rows -= missing + except HTTPException: + raise + except Exception as exc: + skipped += remaining_rows + logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, remaining_rows, exc) + + if preflight_error is not None: + raise preflight_error + + if exported <= 0: + raise HTTPException(status_code=404, detail="No selected emails could be exported") + + spooled_file.seek(0) + zip_name = _safe_attachment_zip_name(f"{export_folder}.zip", "selected-emails.zip") + logger.info("Bulk email export complete owner=%s selected=%s exported=%s skipped=%s", owner, len(messages), exported, skipped) + + def _iter_spooled(f, chunk_size=64 * 1024): + try: + while True: + chunk = f.read(chunk_size) + if not chunk: + break + yield chunk + finally: + f.close() + + response_started = True + return StreamingResponse( + _iter_spooled(spooled_file), + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{zip_name}"', + "X-Exported-Emails": str(exported), + "X-Skipped-Emails": str(skipped), + }, + ) + finally: + if not response_started: + spooled_file.close() + @router.get("/inline-image/{uid}") async def inline_image( uid: str, @@ -4637,7 +5207,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=False): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", False) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -4657,7 +5227,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on)) + _index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on)) _invalidate_list_cache(account_id, folder) return {"success": True, "flagged": bool(on)} except Exception as e: @@ -4675,7 +5245,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Seen", add=True): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) + _index_update_flags(owner, account_id, folder, uid, "\\Seen", True) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -4695,7 +5265,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, "Archive", role="archive"): return {"success": False, "error": "Email not found"} - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -4726,7 +5296,7 @@ def setup_email_routes(): # click or by another mailbox client; it is already gone # from the requested folder, so do not trap the UI on a # permanent “Email not found” error. - _email_index_delete(owner, account_id, folder, str(uid)) + _index_delete(owner, account_id, folder, str(uid)) _invalidate_list_cache(account_id, folder) return {"success": True, "already_deleted": True} trash_folder = _resolve_mail_folder(conn, "Trash", "trash") @@ -4744,9 +5314,9 @@ def setup_email_routes(): if not _move_email_message(conn, resolved_uid, trash_folder, role="trash"): logger.warning("Email delete Trash move failed uid=%s resolved_uid=%s folder=%s trash=%s", uid, resolved_uid, folder, trash_folder) return {"success": False, "error": "Could not move email to Trash"} - _email_index_delete(owner, account_id, folder, resolved_uid) + _index_delete(owner, account_id, folder, resolved_uid) if resolved_uid != str(uid): - _email_index_delete(owner, account_id, folder, str(uid)) + _index_delete(owner, account_id, folder, str(uid)) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -4762,7 +5332,7 @@ def setup_email_routes(): if not _store_email_flag(conn, uid, "\\Deleted", add=True): return {"success": False, "error": "Email not found"} conn.expunge() - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -4830,7 +5400,7 @@ def setup_email_routes(): pending_expunge.append(uid) continue deleted_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) except Exception: failed_uids.append(uid) logger.debug("Bulk email delete failed for uid=%s", uid, exc_info=True) @@ -4852,7 +5422,7 @@ def setup_email_routes(): if expunge_ok: for uid in pending_expunge: deleted_uids.append(uid) - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) else: # Prioritize preservation over cleanup: unflag \Deleted from our # copied messages so future EXPUNGE operations do not destroy them. @@ -4971,7 +5541,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _move_email_message(conn, uid, dest): return {"success": False, "error": f"Failed to move to {dest}"} - _email_index_delete(owner, account_id, folder, uid) + _index_delete(owner, account_id, folder, uid) _invalidate_list_cache(account_id) return {"success": True} except Exception as e: @@ -5057,7 +5627,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=True): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True) + _index_update_flags(owner, account_id, folder, uid, "\\Answered", True) _clear_done_response_tags(owner, account_id, folder, uid) _invalidate_list_cache(account_id, folder) return {"success": True} @@ -5073,7 +5643,7 @@ def setup_email_routes(): conn.select(_q(folder)) if not _store_email_flag(conn, uid, "\\Answered", add=False): return {"success": False, "error": "Email not found"} - _email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False) + _index_update_flags(owner, account_id, folder, uid, "\\Answered", False) _invalidate_list_cache(account_id, folder) return {"success": True} except Exception as e: @@ -5740,7 +6310,7 @@ def setup_email_routes(): try: st, _sel = imap.select(_q(_source_folder), readonly=False) if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True): - _email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True) + _index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True) _clear_done_response_tags(owner, _account_id, _source_folder, _source_uid) _invalidate_list_cache(_account_id, _source_folder) logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}") @@ -6290,7 +6860,7 @@ def setup_email_routes(): await queue.put({'type': 'result', **result}) task = asyncio.create_task(run()) try: - while True: + while not _shutdown.is_shutting_down(): item = await queue.get() yield 'data: ' + json.dumps(item) + '\n\n' if item.get('type') == 'result': @@ -6723,6 +7293,12 @@ def setup_email_routes(): if account_id: _assert_owns_account(account_id, owner) + for field, label in (("imap_host", "IMAP"), ("smtp_host", "SMTP")): + if data.get(field): + host_error = await run_in_threadpool(_check_mail_host, data[field], owner, label) + if host_error: + return {"ok": False, "error": host_error} + # Non-reply automation flags stay global. Away/auto-reply settings are # account-scoped when an account_id is supplied by the Email Settings UI. settings = _load_settings() @@ -6863,6 +7439,11 @@ def setup_email_routes(): smtp_port, port_err = _coerce_port(data.get("smtp_port"), 465) if port_err: return {"ok": False, "error": port_err} + for field, label in (("imap_host", "IMAP"), ("smtp_host", "SMTP")): + if data.get(field): + host_error = await run_in_threadpool(_check_mail_host, data[field], owner, label) + if host_error: + return {"ok": False, "error": host_error} db = SessionLocal() try: _lock_email_account_owner_mutation(db, owner) @@ -6908,6 +7489,11 @@ def setup_email_routes(): """Update an email account. Passwords only overwrite if non-empty.""" # Path param account_id — dep validated via Query, re-check the path-param value. _assert_owns_account(account_id, owner) + for field, label in (("imap_host", "IMAP"), ("smtp_host", "SMTP")): + if data.get(field): + host_error = await run_in_threadpool(_check_mail_host, data[field], owner, label) + if host_error: + return {"ok": False, "error": host_error} from core.database import SessionLocal, EmailAccount db = SessionLocal() try: diff --git a/routes/gallery/gallery_routes.py b/routes/gallery/gallery_routes.py index 276ac90a5..690c22044 100644 --- a/routes/gallery/gallery_routes.py +++ b/routes/gallery/gallery_routes.py @@ -1,14 +1,17 @@ """Gallery routes — browsable library for photos and AI-generated images.""" import os +import asyncio import base64 +import gc import hashlib import io import logging import re +import time import uuid from pathlib import Path -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Tuple from fastapi import APIRouter, HTTPException, Query, Request @@ -34,6 +37,241 @@ _SAM_STATE: Dict[str, Any] = {} _GROUNDING_STATE: Dict[str, Any] = {} +def _rss_mb() -> float: + """Current process RSS in MB (from /proc; -1.0 when unavailable).""" + try: + with open("/proc/self/status", "r") as fh: + for line in fh: + if line.startswith("VmRSS:"): + return round(int(line.split()[1]) / 1024.0, 1) + except Exception: + pass + return -1.0 + + +def _image_op_log_rss(op: str, phase: str) -> None: + logger.info("image-op %s %s rss_mb=%.1f", op, phase, _rss_mb()) + + +def _local_image_models_enabled() -> bool: + """Whether GB-scale local image models may be loaded IN-PROCESS. + + Default OFF: loading rembg/SAM/OWL-ViT/Real-ESRGAN in the web process + routinely OOM-kills memory-constrained hosts. Enable with + ``ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS=1`` or point ``ODYSSEUS_IMAGE_OPS_BASE`` + at a remote service that implements the same ``/api/image/*`` endpoints. + """ + return os.getenv("ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS", "").strip().lower() in ( + "1", "true", "yes", "on", + ) + + +def _remote_image_ops_base() -> str: + return (os.getenv("ODYSSEUS_IMAGE_OPS_BASE", "") or "").strip().rstrip("/") + + +def _outpaint_timeout() -> float: + """Provider timeout for the outpaint/extend proxy. + + Tall extensions can take up to ~330 s (the provider's own timeout is + 900 s), so the proxy waits 900 s by default — otherwise a long extend is + cut off at the proxy. Override with ``ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT``. + """ + try: + return float(os.getenv("ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT", "900") or 900) + except (TypeError, ValueError): + return 900.0 + + +def _unload_image_models() -> None: + """Drop lazily-loaded torch models so their RAM returns to the OS.""" + if not _SAM_STATE and not _GROUNDING_STATE: + return + _SAM_STATE.clear() + _GROUNDING_STATE.clear() + try: + import torch # type: ignore + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + gc.collect() + _image_op_log_rss("models", "unloaded") + + +async def _route_heavy_image_op(path: str, body: dict, *, label: str): + """Proxy a heavy image op to the remote ops service when configured, else + run it in-process only when explicitly enabled. + + Returns the remote JSON on success, or ``None`` to signal the caller should + run it in-process; raises HTTPException(503) with a clear message otherwise + so the app never loads a GB-scale model on a memory-constrained host. + """ + base = _remote_image_ops_base() + if base: + _image_op_log_rss(label, f"proxy->{base}") + try: + import httpx + + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + path, json=body) + if resp.status_code == 200: + return resp.json() + logger.warning("image-op %s remote %s -> %s", label, path, resp.status_code) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service returned {resp.status_code}).", + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op %s remote %s failed: %s", label, path, exc) + raise HTTPException( + 503, + f"{label} is temporarily unavailable (remote service unreachable).", + ) from exc + if _local_image_models_enabled(): + return None + raise HTTPException(503, f"{label} is temporarily unavailable on this server.") + + +def _xywh_to_xyxy(bbox): + """Convert a remote image-ops [x, y, w, h] bbox to this app's [x1,y1,x2,y2].""" + try: + x, y, w, h = (float(v) for v in bbox) + return [int(x), int(y), int(x + w), int(y + h)] + except Exception: + return bbox + + +async def _remote_remove_background(body: dict): + """Proxy background removal to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/remove-background (multipart: image, + optional hint_mask, model, return_mask). Returns {"image": } + to match this route's local response shape, or None when the caller should + run in-process (local models explicitly enabled); raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Background removal is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + hint_bytes = None + if body.get("hint_mask"): + hint_bytes = _b64_bytes(body.get("hint_mask"), label="hint_mask") + _image_op_log_rss("remove-bg", f"proxy->{base}") + try: + import httpx + + files = {"image": ("image.png", img_bytes, "image/png")} + if hint_bytes: + files["hint_mask"] = ("hint.png", hint_bytes, "image/png") + data = {"model": str(body.get("model") or ""), "return_mask": "true"} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/remove-background", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + items = payload.get("data") or [] + if not items or not items[0].get("b64_json"): + raise HTTPException(503, "Background removal is temporarily unavailable (remote returned no image).") + out = {"image": items[0]["b64_json"]} + if items[0].get("mask_b64"): + out["mask"] = items[0]["mask_b64"] + if items[0].get("bbox"): + out["bbox"] = items[0]["bbox"] + return out + raise HTTPException( + 503, f"Background removal is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op remove-bg remote failed: %s", exc) + raise HTTPException(503, "Background removal is temporarily unavailable (remote service unreachable).") from exc + + +async def _remote_smart_mask(body: dict): + """Proxy SAM/grounding masking to the remote ops service (seneca contract). + + Endpoint: POST {base}/v1/images/mask (multipart: image + one of prompt/box/ + points). Returns {"mask": , "bbox": [...], "model", "device"} to + match this route's local response shape, or None when the caller should run + in-process; raises 503 otherwise. + """ + base = _remote_image_ops_base() + if not base: + if _local_image_models_enabled(): + return None + raise HTTPException(503, "Smart mask is temporarily unavailable on this server.") + img_bytes = _b64_bytes(body.get("image")) + points = body.get("points") or [] + box = body.get("box") + text = (body.get("text") or body.get("query") or "").strip() + if not points and not box and not text: + raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", f"proxy->{base}") + try: + import httpx + + data: Dict[str, Any] = {} + if text: + data["prompt"] = text + if box: + try: + x1, y1, x2, y2 = (float(v) for v in box) + data["box"] = ",".join(str(int(v)) for v in (x1, y1, x2 - x1, y2 - y1)) + except Exception as exc: + raise HTTPException(400, "Box must be [x1, y1, x2, y2]") from exc + if points: + import json as _json + + data["points"] = _json.dumps( + [[float(p["x"]), float(p["y"]), int(p.get("label", 1))] for p in points] + ) + if body.get("multimask") is not None: + data["multimask"] = str(bool(body.get("multimask"))).lower() + if body.get("dilate_px") is not None: + data["dilate_px"] = str(int(body.get("dilate_px"))) + files = {"image": ("image.png", img_bytes, "image/png")} + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post(base + "/v1/images/mask", files=files, data=data) + if resp.status_code == 200: + payload = resp.json() + masks = payload.get("masks") or [] + candidates = [m for m in masks if m.get("b64_json")] + if not candidates: + # Empty selection is a normal 200 outcome (never a 400): the + # editor shows a "nothing selected" message and changes nothing. + return { + "mask": None, + "bbox": None, + "masks": [], + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + "error": "Nothing selected for that selection.", + } + best = max(candidates, key=lambda m: float(m.get("score") or 0.0)) + return { + "mask": best["b64_json"], + "bbox": _xywh_to_xyxy(best.get("bbox")), + "label": best.get("label"), + "masks": masks, + "model": payload.get("model") or "sam+grounding", + "device": payload.get("device", ""), + } + raise HTTPException( + 503, f"Smart mask is temporarily unavailable (remote service returned {resp.status_code})." + ) + except HTTPException: + raise + except Exception as exc: + logger.warning("image-op mask remote failed: %s", exc) + raise HTTPException(503, "Smart mask is temporarily unavailable (remote service unreachable).") from exc + + def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): if not image_b64: raise HTTPException(400, "Missing image") @@ -50,12 +288,79 @@ def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"): raise HTTPException(400, "Invalid image") from exc +def _b64_bytes(value: str, *, label: str = "image") -> bytes: + """Decode raw base64 OR a ``data:image/...;base64,`` URL to bytes. + + The image-edit tools post data URLs, while the browser posts raw base64, + so every decode site must accept both. + """ + if not value: + raise HTTPException(400, f"Missing {label}") + text = str(value) + if "," in text and text.split(",", 1)[0].startswith("data:"): + text = text.split(",", 1)[1] + try: + return base64.b64decode(text) + except Exception as exc: + raise HTTPException(400, f"Invalid {label}") from exc + + def _pil_image_to_b64(img, *, fmt: str = "PNG") -> str: buf = io.BytesIO() img.save(buf, format=fmt) return base64.b64encode(buf.getvalue()).decode("ascii") +def _prepare_local_edit(src_b64, mask_b64, max_side): + """Decode/normalize/cap an edit's inputs to PNG bytes at a model-friendly size. + + CPU-bound (a 4000x3000 decode+resize is ~0.2 s); run it via a thread off the + event loop. Returns (source_png_bytes, mask_png_bytes, model_w, model_h, orig_size). + """ + from PIL import Image + + img_bytes = _b64_bytes(src_b64) + source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") + if mask_b64: + mask_bytes = _b64_bytes(mask_b64, label="mask") + mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") + else: + mask_png = Image.new("L", source_png.size, 255) + orig_size = source_png.size + if max(orig_size) > max_side: + scale = float(max_side) / max(orig_size) + model_w = max(16, int(round(orig_size[0] * scale / 16.0)) * 16) + model_h = max(16, int(round(orig_size[1] * scale / 16.0)) * 16) + else: + model_w, model_h = orig_size + if (model_w, model_h) != orig_size: + source_work = source_png.resize((model_w, model_h), Image.LANCZOS) + mask_work = mask_png.resize((model_w, model_h), Image.LANCZOS) + else: + source_work, mask_work = source_png, mask_png + src_buf = io.BytesIO() + source_work.save(src_buf, format="PNG") + mask_buf = io.BytesIO() + mask_work.save(mask_buf, format="PNG") + return src_buf.getvalue(), mask_buf.getvalue(), model_w, model_h, orig_size + + +def _restore_local_b64(raw_b64, orig_size, model_size): + """Resize a provider result b64 back to the original dimensions (CPU-bound).""" + if tuple(model_size) == tuple(orig_size): + return raw_b64 + from PIL import Image + + try: + im = Image.open(io.BytesIO(base64.b64decode(raw_b64))).convert("RGBA") + im = im.resize(tuple(orig_size), Image.LANCZOS) + ob = io.BytesIO() + im.save(ob, format="PNG") + return base64.b64encode(ob.getvalue()).decode("ascii") + except Exception: + return raw_b64 + + def _load_sam_backend(): model_id = os.getenv("ODYSSEUS_SAM_MODEL", "facebook/sam-vit-base") cached = _SAM_STATE.get(model_id) @@ -301,7 +606,61 @@ def _first_visible_image_endpoint(db, owner: str | None): for ep in endpoints: if getattr(ep, "owner", None) == owner: return ep - return endpoints[0] if endpoints else None + if endpoints: + return endpoints[0] + # Fallback: the image model may live on a *mixed* endpoint (model_type != + # "image"), e.g. qwen-image-edit / gpt-image-1 on a shared OpenAI-compatible + # gateway or a diffusion server registered as "chat". Chat image generation + # already resolves its model by name across every enabled endpoint, so the + # editor must find the same model here instead of reporting "no endpoint". + return _first_mixed_image_endpoint(db, owner) + + +def _first_mixed_image_endpoint(db, owner: str | None): + """Best-effort image endpoint when no image-only endpoint is registered. + + Returns an enabled, owner-visible endpoint whose configured model list + includes an image-generation model, preferring the admin's configured + ``image_model`` when it is one of them. + """ + import json + from src.auth_helpers import owner_filter + from src.image_model_ids import looks_like_image_generation_model + + preferred = "" + try: + from src.settings import load_settings + preferred = str((load_settings() or {}).get("image_model") or "").strip() + except Exception: + preferred = "" + + query = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), # noqa: E712 + ModelEndpoint, + owner, + ) + fallback = None + for ep in query.all(): + if getattr(ep, "model_type", None) == "image": + continue + models = set() + for field in ("cached_models", "pinned_models"): + try: + values = getattr(ep, field, None) or [] + values = json.loads(values) if isinstance(values, str) else values + models.update(v for v in values if isinstance(v, str)) + except (TypeError, ValueError): + pass + matches = {m for m in models if looks_like_image_generation_model(m)} + if not matches: + continue + if preferred and preferred in matches: + return ep + if owner and getattr(ep, "owner", None) == owner: + return ep + if fallback is None: + fallback = ep + return fallback def _visible_image_endpoint_for_base(db, base: str, owner: str | None, model: str = ""): @@ -324,9 +683,15 @@ def _visible_image_endpoint_for_base(db, base: str, owner: str | None, model: st configured.update(v for v in values if isinstance(v, str)) except (TypeError, ValueError): pass - if model not in configured or not looks_like_image_generation_model(model): - continue - if _normalize_image_endpoint_base(getattr(ep, "base_url", "")) == target: + if model: + if model not in configured or not looks_like_image_generation_model(model): + continue + else: + if not any(looks_like_image_generation_model(v) for v in configured): + continue + ep_id = str(getattr(ep, "id", "") or "") + ep_base = _normalize_image_endpoint_base(getattr(ep, "base_url", "")) + if (ep_id and (base == ep_id or target == ep_id)) or (ep_base and ep_base == target): if owner and getattr(ep, "owner", None) == owner: return ep if fallback is None: @@ -353,13 +718,102 @@ async def _fetch_result_image_b64(url: str) -> Optional[str]: ) if not ok: raise HTTPException(502, f"Upstream returned an unsafe image URL: {reason}") - async with httpx.AsyncClient(timeout=60) as c2: + async with httpx.AsyncClient(timeout=60, follow_redirects=False) as c2: ir = await c2.get(url) if ir.status_code == 200: return base64.b64encode(ir.content).decode() return None +def _resolve_image_op_endpoint(db, owner, requested_base: str, chosen_model: str = "") -> Tuple[str, Optional[str]]: + """Resolve the provider base (with /v1) + api_key for a self-hosted image op. + + Mirrors inpaint_proxy: a client-supplied base must map to a registered, + owner-visible image endpoint (no direct use of request-body URLs). + """ + if requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, owner, chosen_model) + if not ep: + raise HTTPException(403, "Choose a registered image endpoint") + else: + ep = _first_visible_image_endpoint(db, owner) + if not ep: + raise HTTPException(400, "No image generation endpoint configured. Serve a diffusion model via Cookbook first.") + base = ep.base_url.rstrip("/") + if not base.endswith("/v1"): + base += "/v1" + return base, ep.api_key + + +async def _forward_seneca_image_op( + base: str, + api_key: Optional[str], + provider_path: str, + image_bytes: bytes, + fields: Dict[str, str], + *, + label: str, + timeout: float = 300.0, +) -> Dict[str, Any]: + """POST a single-image multipart op to the provider and return ``{image}``. + + The provider returns OpenAI-style ``{data:[{b64_json|url}]}`` or a raw + ``{image}``. ``request_id`` in ``fields`` keys the provider's progress route. + """ + import httpx + + url = base.rstrip("/") + provider_path + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + files = {"image": ("source.png", image_bytes, "image/png")} + _t0 = time.perf_counter() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + r = await client.post(url, data=fields, files=files, headers=headers) + _provider_ms = (time.perf_counter() - _t0) * 1000.0 + except httpx.TimeoutException: + raise HTTPException(504, f"{label} timed out ({int(timeout)}s)") + except Exception as exc: + logger.warning("%s provider call failed: %s", label, exc) + raise HTTPException(503, f"{label} is temporarily unavailable (remote service unreachable).") + if r.status_code != 200: + logger.warning("%s provider status %s", label, r.status_code) + detail = f"{label} failed" + try: + j = r.json() + if isinstance(j, dict) and (j.get("error") or j.get("detail")): + detail = str(j.get("error") or j.get("detail"))[:200] + except Exception: + pass + raise HTTPException(r.status_code if r.status_code in (400, 404, 422) else 502, detail) + try: + result = r.json() + except ValueError as exc: + raise HTTPException(502, "Image endpoint returned invalid JSON") from exc + raw_b64: Optional[str] = None + used_model = "" + if isinstance(result, dict): + used_model = str(result.get("model") or "") + data = result.get("data") + if isinstance(data, list) and data: + item = data[0] if isinstance(data[0], dict) else {} + raw_b64 = item.get("b64_json") or None + if not raw_b64 and item.get("url"): + raw_b64 = await _fetch_result_image_b64(item["url"]) + if not raw_b64 and result.get("image"): + raw_b64 = result["image"] + if not raw_b64: + raise HTTPException(502, "Image endpoint returned no image") + logger.info( + "%s timing model_ms=%.1f total_ms=%.1f request_id=%s", + label, _provider_ms, + (time.perf_counter() - _t0) * 1000.0, fields.get("request_id") or "-", + ) + payload: Dict[str, Any] = {"image": raw_b64} + if used_model: + payload["model"] = used_model + return payload + + def setup_gallery_routes() -> APIRouter: router = APIRouter(tags=["gallery"]) @@ -602,7 +1056,7 @@ def setup_gallery_routes() -> APIRouter: # Use img2img endpoint if available, otherwise upscale via canvas on client try: - async with httpx.AsyncClient(timeout=120) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/upscale", json={ "image": b64, "scale": scale, }) @@ -645,7 +1099,7 @@ def setup_gallery_routes() -> APIRouter: base_url += "/v1" try: - async with httpx.AsyncClient(timeout=180) as client: + async with httpx.AsyncClient(timeout=300) as client: resp = await client.post(f"{base_url}/images/generations", json={ "prompt": prompt, "image": b64, @@ -1282,7 +1736,7 @@ def setup_gallery_routes() -> APIRouter: requested_base = (body.pop("_endpoint", "") or "").rstrip("/") # SSRF hardening: validate a client-supplied endpoint before any # outbound request (mirrors routes/embedding_routes.py). - if requested_base: + if requested_base and "://" in requested_base: from src.url_safety import check_outbound_url ok, reason = check_outbound_url( requested_base, @@ -1291,6 +1745,10 @@ def setup_gallery_routes() -> APIRouter: if not ok: raise HTTPException(400, f"Rejected endpoint URL: {reason}") chosen_model = (body.pop("_model", "") or "").strip() + # Optional client-supplied id so GET /api/image/edit-progress can poll the + # provider's step progress for this op. Ignored by providers that do not + # implement the progress contract. + progress_request_id = (body.get("request_id") or "").strip() api_key = None if not requested_base: db = SessionLocal() @@ -1338,8 +1796,10 @@ def setup_gallery_routes() -> APIRouter: raise HTTPException(500, "Pillow not installed on server") try: - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) + img_bytes = _b64_bytes(body.get("image")) + if not body.get("mask"): + raise HTTPException(400, "A mask is required for OpenAI-compatible image edits") + mask_bytes = _b64_bytes(body.get("mask"), label="mask") source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") # luminance # Build OpenAI mask: RGBA where alpha=255 means keep, 0 means regenerate. @@ -1390,7 +1850,11 @@ def setup_gallery_routes() -> APIRouter: } headers = {"Authorization": f"Bearer {api_key}"} try: - async with httpx.AsyncClient(timeout=120) as client: + logger.debug( + "inpaint_proxy provider edit -> model=%r size=%r base=%s", + oa_model, size, base, + ) + async with httpx.AsyncClient(timeout=300) as client: if is_openrouter: # Reference-based editing, then composite locally so # pixels outside the user's mask remain unchanged. @@ -1410,7 +1874,16 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), headers=headers, data=data, files=files) if r.status_code != 200: logger.error("inpaint_proxy OpenAI edit: status %s", r.status_code) - raise HTTPException(r.status_code, "OpenAI edit failed") + _detail = "OpenAI edit failed" + try: + _j = r.json() + if isinstance(_j, dict): + _detail = _j.get("error") or _j.get("detail") or _detail + if isinstance(_detail, dict): + _detail = _detail.get("message") or _detail + except Exception: + pass + raise HTTPException(r.status_code, f"{_detail} (provider HTTP {r.status_code})") result = r.json() raw_b64 = None if result.get("data"): @@ -1441,13 +1914,16 @@ def setup_gallery_routes() -> APIRouter: blended = Image.composite(generated, source_png, mask_png) out_buf = io.BytesIO() blended.save(out_buf, format="PNG") - return {"image": base64.b64encode(out_buf.getvalue()).decode()} + return {"image": base64.b64encode(out_buf.getvalue()).decode(), "model": oa_model} except Exception as comp_err: # Never return a full-image edit when masking fails. logger.warning(f"Inpaint compose failed: {comp_err}") raise HTTPException(502, "Could not apply the edit within the selected region") except httpx.TimeoutException: - raise HTTPException(504, "OpenAI inpaint timed out (120s)") + raise HTTPException(504, "OpenAI inpaint timed out (300s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: OpenAI endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") # Self-hosted diffusion server path. Newer Odysseus image # wrappers expose the OpenAI-compatible /v1/images/edits @@ -1459,45 +1935,96 @@ def setup_gallery_routes() -> APIRouter: # supports multiple models per process. Harmless if ignored. if chosen_model: body["model"] = chosen_model - async with httpx.AsyncClient(timeout=240) as client: + async with httpx.AsyncClient(timeout=300) as client: + _t0 = time.perf_counter() + _restore_ms = [0.0] try: import base64, io from PIL import Image - img_bytes = base64.b64decode(body["image"]) - mask_bytes = base64.b64decode(body["mask"]) - # Normalize both inputs to PNG bytes. Local MLX and - # Diffusers wrappers expect white mask pixels to mean - # "edit this region", which matches the editor's mask. - source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA") - mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L") - src_buf = io.BytesIO() - source_png.save(src_buf, format="PNG") - mask_buf = io.BytesIO() - mask_png.save(mask_buf, format="PNG") + # Decode/resize/encode off the event loop: a 4000x3000 input + # costs ~0.2 s of CPU that would otherwise block every other + # request (and stack under concurrent edits). + _max_side = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_SIDE", "768") or 768) + _src_png, _mask_png, _model_w, _model_h, _orig_size = await asyncio.to_thread( + _prepare_local_edit, body.get("image"), body.get("mask"), _max_side + ) files = { - "image": ("source.png", src_buf.getvalue(), "image/png"), - "mask": ("mask.png", mask_buf.getvalue(), "image/png"), + "image": ("source.png", _src_png, "image/png"), + "mask": ("mask.png", _mask_png, "image/png"), } data = { "model": chosen_model or body.get("model") or "", "prompt": body.get("prompt", ""), - "size": f"{int(body.get('width') or source_png.width)}x{int(body.get('height') or source_png.height)}", + "size": f"{_model_w}x{_model_h}", "n": "1", } + if progress_request_id: + data["request_id"] = progress_request_id + # Let the caller's step budget through up to a cap (quality + # edits such as removal/background-replacement/expand pass 8 + # on purpose); only fall back to a moderate default when no + # count is requested. The size cap above is what keeps the + # op responsive. + _max_steps = int(os.getenv("ODYSSEUS_IMAGE_EDIT_MAX_STEPS", "8") or 8) + if body.get("steps"): + try: + data["steps"] = str(min(max(1, int(body["steps"])), _max_steps)) + except (TypeError, ValueError): + data["steps"] = "4" + else: + data["steps"] = "4" + + async def _restore_b64(_raw: str) -> str: + """Resize a provider PNG b64 back to the original size (off-loop).""" + if (_model_w, _model_h) == _orig_size: + return _raw + _rs = time.perf_counter() + try: + return await asyncio.to_thread( + _restore_local_b64, _raw, _orig_size, (_model_w, _model_h) + ) + finally: + _restore_ms[0] += (time.perf_counter() - _rs) * 1000.0 + logger.debug( + "inpaint_proxy self-hosted /images/edits -> model=%r " + "size=%r steps=%r endpoint=%s", + data.get("model"), data.get("size"), + data.get("steps"), base, + ) + _prep_ms = (time.perf_counter() - _t0) * 1000.0 + + def _finish(_payload: Dict[str, Any]) -> Dict[str, Any]: + # model_ms is the provider round-trip (its own queue + + # diffusion time); prep_ms is our decode/resize; restore_ms + # is resizing the result back. total_ms is the whole op. + logger.info( + "inpaint-proxy timing prep_ms=%.1f model_ms=%.1f restore_ms=%.1f " + "total_ms=%.1f size=%s steps=%s model=%s request_id=%s", + _prep_ms, _provider_ms[0], _restore_ms[0], + (time.perf_counter() - _t0) * 1000.0, + data.get("size"), data.get("steps"), data.get("model"), + progress_request_id or "-", + ) + return _payload + + _t_post = time.perf_counter() r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), data=data, files=files) + _provider_ms = [(time.perf_counter() - _t_post) * 1000.0] if r.status_code == 200: result = r.json() + _used_model = result.get("model") if isinstance(result, dict) else None + _used_model = _used_model or data.get("model") or "" if isinstance(result, dict) and result.get("data"): item = result["data"][0] if item.get("b64_json"): - return {"image": item["b64_json"]} + return _finish({"image": await _restore_b64(item["b64_json"]), "model": _used_model}) if item.get("url"): raw_b64 = await _fetch_result_image_b64(item["url"]) if raw_b64: - return {"image": raw_b64} + return _finish({"image": await _restore_b64(raw_b64), "model": _used_model}) if isinstance(result, dict) and result.get("image"): - return {"image": result["image"]} + return _finish({"image": await _restore_b64(result["image"]), "model": _used_model}) raise HTTPException(502, "Image edit endpoint returned no image") if r.status_code not in (404, 405): logger.warning("inpaint_proxy self-hosted edits: status %s", r.status_code) @@ -1528,16 +2055,170 @@ def setup_gallery_routes() -> APIRouter: r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body) if r.status_code != 200: logger.error("inpaint_proxy diffusion: status %s", r.status_code) - raise HTTPException(r.status_code, "Inpaint request failed") + detail = "Inpaint request failed" + try: + j = r.json() + if isinstance(j, dict): + detail = j.get("detail") or j.get("error") or detail + except Exception: + pass + raise HTTPException(r.status_code, f"{detail} (provider HTTP {r.status_code})") return r.json() except httpx.TimeoutException: raise HTTPException(504, "Inpaint request timed out (240s)") + except httpx.TransportError as exc: + logger.warning("inpaint_proxy: image endpoint unreachable: %s", exc) + raise HTTPException(502, f"Image server unreachable: {type(exc).__name__}") except HTTPException: raise except Exception: logger.exception("inpaint_proxy: request failed") raise HTTPException(502, "Inpaint request failed") + async def _resolve_and_decode(body: Dict[str, Any], user): + """Shared prep for the seneca single-image ops: validate endpoint + decode. + + Returns (base, api_key, image_bytes, chosen_model). The base64 decode + runs off the event loop (large images). + """ + requested_base = (body.pop("_endpoint", "") or "").rstrip("/") + if requested_base and "://" in requested_base: + from src.url_safety import check_outbound_url + ok, reason = check_outbound_url( + requested_base, + block_private=os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") + chosen_model = (body.pop("_model", "") or "").strip() + db = SessionLocal() + try: + base, api_key = _resolve_image_op_endpoint(db, user, requested_base, chosen_model) + finally: + db.close() + image_bytes = await asyncio.to_thread(_b64_bytes, body.get("image")) + return base, api_key, image_bytes, chosen_model + + @router.post("/api/image/replace-background") + async def replace_background(request: Request): + """Proxy seneca's subject-mask background replacement (~/v1/images/replace-background).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if body.get("harmonise") is not None: + fields["harmonise"] = "true" if body.get("harmonise") else "false" + if body.get("shadow") is not None: + fields["shadow"] = "true" if body.get("shadow") else "false" + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/replace-background", image_bytes, fields, + label="replace-background", + ) + + @router.post("/api/image/outpaint") + async def outpaint(request: Request): + """Proxy seneca's canvas outpaint/extend (~/v1/images/outpaint, alias /extend).""" + user = require_privilege(request, "can_generate_images") + body = await request.json() + base, api_key, image_bytes, chosen_model = await _resolve_and_decode(body, user) + fields: Dict[str, str] = { + "prompt": str(body.get("prompt") or ""), + "steps": str(body.get("steps") or 4), + } + for key in ("ratio", "size", "context"): + if body.get(key) is not None and str(body.get(key)) != "": + fields[key] = str(body[key]) + if body.get("request_id"): + fields["request_id"] = str(body["request_id"]).strip() + if chosen_model: + fields["model"] = chosen_model + return await _forward_seneca_image_op( + base, api_key, "/images/outpaint", image_bytes, fields, + label="outpaint", timeout=_outpaint_timeout(), + ) + + @router.post("/api/image/extend") + async def outpaint_extend(request: Request): + """Alias of /api/image/outpaint (the provider exposes /outpaint as /extend too).""" + return await outpaint(request) + + @router.get("/api/image/edit-progress") + async def edit_progress(request: Request): + """Poll a running image edit's provider-side step progress. + + Contract v1 (image-op progress): the provider exposes + ``GET /v1/images/progress/{request_id}`` -> ``{stage, step, steps, + eta_s, progress}``. This proxy resolves the configured image endpoint + (owner-scoped) and returns that JSON, or ``{"available": false}`` when + the provider has no progress route yet, so the client can keep an + indeterminate bar without breaking. + """ + import httpx + user = require_privilege(request, "can_generate_images") + request_id = (request.query_params.get("request_id") or "").strip() + if not request_id: + raise HTTPException(400, "Missing request_id") + # request_id is spliced into the outbound URL, so it must be a safe token + # (no path separators/host chars). The base itself is a DB-validated row. + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", request_id): + raise HTTPException(400, "Invalid request_id") + requested_base = (request.query_params.get("base") or "").strip().rstrip("/") + endpoint_id = (request.query_params.get("endpoint_id") or "").strip() + db = SessionLocal() + try: + ep = None + if endpoint_id: + ep = _visible_image_endpoint_for_base(db, endpoint_id, user) + if ep is None: + return {"available": False, "reason": "endpoint not found or disabled"} + elif requested_base: + ep = _visible_image_endpoint_for_base(db, requested_base, user) + if ep is None: + return {"available": False, "reason": "endpoint not found or disabled"} + else: + ep = _first_visible_image_endpoint(db, user) + if ep is None: + return {"available": False, "reason": "no image endpoint configured"} + base = ep.base_url.rstrip("/") + finally: + db.close() + + from src.url_safety import check_outbound_url + block_private = os.getenv("IMAGE_BLOCK_PRIVATE_IPS", "false").lower() == "true" + ok, reason = check_outbound_url(base, block_private=block_private) + if not ok: + # CodeQL #1203: Never expose internal addresses, resolver errors or raw + # exception details to clients or logs. Use a stable diagnostic message + # without interpolating untrusted exception content or destination strings. + logger.warning("edit-progress endpoint rejected: outbound URL policy check failed") + return {"available": False, "reason": "endpoint URL rejected"} + + if not base.endswith("/v1"): + base += "/v1" + # Dynamic path segment (validated token above), so not routed through + # _join_checked_gallery_endpoint's exact-match whitelist. + url = f"{base}/images/progress/{request_id}" + try: + async with httpx.AsyncClient(timeout=10, follow_redirects=False) as client: + r = await client.get(url) + if r.status_code == 200: + data = r.json() + if isinstance(data, dict): + data.setdefault("available", True) + return data + return {"available": True, "progress": data} + return {"available": False, "status": r.status_code} + except Exception as exc: + logger.debug("edit-progress poll failed: %s", exc) + return {"available": False, "reason": "provider unreachable"} + # ---- POST /api/image/harmonize — proper img2img call ---- # Earlier version routed through inpaint with a full-white mask, but # most backends interpret "100% mask coverage" as "regenerate from @@ -1563,7 +2244,7 @@ def setup_gallery_routes() -> APIRouter: # below, so validate it first (mirrors routes/embedding_routes.py). # Local-first means loopback/LAN is allowed by default; the cloud # metadata range and non-HTTP(S) schemes are always rejected. - if requested_base: + if requested_base and "://" in requested_base: from src.url_safety import check_outbound_url ok, reason = check_outbound_url( requested_base, @@ -1590,7 +2271,7 @@ def setup_gallery_routes() -> APIRouter: # so the outbound URL never depends directly on request-body input. db = SessionLocal() try: - ep = _visible_image_endpoint_for_base(db, requested_base, user) + ep = _visible_image_endpoint_for_base(db, requested_base, user, model) if not ep: raise HTTPException(403, "Choose a registered image endpoint") base = ep.base_url.rstrip("/") @@ -1689,10 +2370,10 @@ def setup_gallery_routes() -> APIRouter: headers["Authorization"] = f"Bearer {api_key}" last_err = None - # Cold-start SDXL inpaint can take 60-90s on first request (loading - # weights to GPU). 240s gives headroom for both that and a full - # 1024×1024 inference pass on slower setups. - async with httpx.AsyncClient(timeout=240) as client: + # Cold-start image servers can take 90-110s for a full pass (and + # 60-90s more on first request while weights load to GPU). 300s gives + # headroom for both on slower setups. + async with httpx.AsyncClient(timeout=300) as client: for path, kind, payload in candidates: _effective_base = base_root if path.startswith("/sdapi") else base target = _join_checked_gallery_endpoint(_effective_base, path) @@ -1755,7 +2436,7 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Unsharp mask: radius=2, percent=amount*200, threshold=3 @@ -1776,6 +2457,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/denoise", body, label="AI denoise") + if _proxied is not None: + return _proxied try: strength = float(body.get("strength", 0.5)) except Exception: @@ -1788,7 +2472,7 @@ def setup_gallery_routes() -> APIRouter: except ImportError: raise HTTPException(500, "Server missing a required dependency") # Decode source image (RGB; Real-ESRGAN doesn't preserve alpha). - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1827,6 +2511,9 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") if not image_b64: raise HTTPException(400, "No image provided") + _proxied = await _route_heavy_image_op("/api/image/upscale-local", body, label="AI upscale") + if _proxied is not None: + return _proxied try: scale = int(body.get("scale", 2)) except Exception: @@ -1838,7 +2525,7 @@ def setup_gallery_routes() -> APIRouter: import numpy as np except ImportError: raise HTTPException(500, "Server missing a required dependency") - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) src = Image.open(io.BytesIO(img_bytes)).convert("RGB") try: patch_realesrgan_torchvision_compat() @@ -1865,6 +2552,74 @@ def setup_gallery_routes() -> APIRouter: logger.warning("AI upscale failed", exc_info=True) return {"error": "AI upscale failed"} + # ---- POST /api/image/generate ---- + @router.post("/api/image/generate") + async def generate_image(request: Request): + """Text-to-image generation for the editor's Quick Edit 'Image' mode. + + Generates a brand-new image from a prompt using the admin-configured + image model, returned as base64 so the client can drop it in as a layer. + """ + import json as _json + user = require_privilege(request, "can_generate_images") + try: + body = await request.json() + except Exception: + body = {} + prompt = str((body or {}).get("prompt") or "").strip() + if not prompt: + raise HTTPException(400, "A prompt is required") + try: + width = int((body or {}).get("width") or 0) + height = int((body or {}).get("height") or 0) + except (TypeError, ValueError): + width = height = 0 + if height > width: + size = "1024x1536" + elif width > height: + size = "1536x1024" + else: + size = "1024x1024" + try: + from src.ai_interaction import do_generate_image + except Exception: + raise HTTPException(502, "Image generation is unavailable") + # The editor's model picker may choose a specific (cloud) image model; + # forward it so settings defaults only apply when none is selected. + model_choice = str((body or {}).get("model") or "").strip() + payload = {"prompt": prompt, "size": size} + if model_choice: + payload["model"] = model_choice + content = _json.dumps(payload) + result = await do_generate_image(content, owner=user, incognito=True) + if not isinstance(result, dict) or result.get("error"): + msg = (result.get("error") if isinstance(result, dict) else None) or "Image generation failed" + low = str(msg).lower() + # Plain-language status codes the editor maps to fixed messages: + # 503 -> generation is switched off on the server + # 502 -> the image server could not be reached (or failed) + if "disabled" in low or "switched off" in low or "not enabled" in low: + raise HTTPException(503, "Image generation is switched off on the server at the moment.") + if "timed out" in low or "timeout" in low: + raise HTTPException(504, f"Image server timed out: {msg}") + if "connection" in low or "connect" in low or "unreachable" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + # A configured image model that cannot be resolved means its endpoint + # did not answer, i.e. the image server is down, not a bad prompt. + if "no endpoint found with image model" in low: + raise HTTPException(502, f"Image server unreachable: {msg}") + raise HTTPException(502, msg) + image_url = result.get("image_url") or "" + b64 = image_url.split(",", 1)[1] if image_url.startswith("data:") else image_url + if not b64: + raise HTTPException(502, "Image generation returned no image") + return { + "image": b64, + "prompt": prompt, + "model": result.get("image_model"), + "size": result.get("image_size") or size, + } + # ---- POST /api/image/remove-bg ---- @router.post("/api/image/mask") async def smart_mask(request: Request): @@ -1876,6 +2631,9 @@ def setup_gallery_routes() -> APIRouter: """ require_privilege(request, "can_generate_images") body = await request.json() + _proxied = await _remote_smart_mask(body) + if _proxied is not None: + return _proxied image = _b64_to_pil_image(body.get("image") or "", mode="RGB") points = body.get("points") or [] box = body.get("box") @@ -1889,6 +2647,7 @@ def setup_gallery_routes() -> APIRouter: if not points and not box: raise HTTPException(400, "Provide at least one point, box, or object text") + _image_op_log_rss("mask", "start") backend = _load_sam_backend() torch = backend["torch"] processor = backend["processor"] @@ -1985,6 +2744,8 @@ def setup_gallery_routes() -> APIRouter: } if grounded: result["grounding"] = grounded + _image_op_log_rss("mask", "end") + _unload_image_models() return result except HTTPException: raise @@ -2010,11 +2771,14 @@ def setup_gallery_routes() -> APIRouter: body = await request.json() image_b64 = body.get("image") hint_b64 = body.get("hint_mask") + _proxied = await _remote_remove_background(body) + if _proxied is not None: + return _proxied from PIL import Image import base64, io - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGBA") W, H = img.size @@ -2022,7 +2786,7 @@ def setup_gallery_routes() -> APIRouter: bbox = None if hint_b64: try: - hint_bytes = base64.b64decode(hint_b64) + hint_bytes = _b64_bytes(hint_b64, label="hint_mask") hint = Image.open(io.BytesIO(hint_bytes)).convert("L") # Resize the hint to match if dimensions disagree if hint.size != img.size: @@ -2046,6 +2810,7 @@ def setup_gallery_routes() -> APIRouter: else: crop = img + _image_op_log_rss("remove-bg", "start") try: from rembg import remove from starlette.concurrency import run_in_threadpool @@ -2061,6 +2826,9 @@ def setup_gallery_routes() -> APIRouter: cut = tmp except Exception: return {"error": "No background removal model available. Install rembg: pip install rembg"} + finally: + _image_op_log_rss("remove-bg", "end") + _unload_image_models() # Compose the cropped result back into a full-size transparent canvas. if bbox: @@ -2100,11 +2868,13 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter, ImageEnhance import numpy as np - img_bytes = base64.b64decode(image_b64) + img_bytes = _b64_bytes(image_b64) img = Image.open(io.BytesIO(img_bytes)).convert("RGB") # Try GFPGAN first (AI face restoration) try: + if not _local_image_models_enabled(): + raise ImportError("local image models disabled") from gfpgan import GFPGANer import cv2 diff --git a/routes/mcp/mcp_routes.py b/routes/mcp/mcp_routes.py index 5d14ba8c3..5a8ae9c92 100644 --- a/routes/mcp/mcp_routes.py +++ b/routes/mcp/mcp_routes.py @@ -5,6 +5,7 @@ import os import uuid import urllib.parse import html +import secrets from pathlib import Path from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import RedirectResponse, HTMLResponse @@ -12,7 +13,7 @@ import logging import httpx from core.database import McpServer, SessionLocal -from core.middleware import require_admin +from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN, require_admin from src.constants import DATA_DIR, MCP_OAUTH_DIR from src.mcp_manager import McpManager @@ -424,6 +425,11 @@ def setup_mcp_routes(mcp_manager: McpManager): Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]} """ require_admin(request) + internal_token = request.headers.get(INTERNAL_TOOL_HEADER) + if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN): + # Same boundary as POST /api/tools: an agent turn must not be able + # to re-enable MCP tools an admin switched off. + raise HTTPException(403, "Change MCP tool policy in Settings > Integrations, not from chat") db = SessionLocal() try: srv = db.query(McpServer).filter(McpServer.id == server_id).first() diff --git a/routes/model_routes.py b/routes/model_routes.py index 3466d4b9b..4535bbafe 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -7,6 +7,7 @@ import json import hashlib import ipaddress import socket +import secrets import time as _time import logging import threading @@ -47,6 +48,7 @@ from src.endpoint_resolver import ( build_headers, ) from src.auth_helpers import _auth_disabled, effective_user, owner_filter +from src.url_safety import check_outbound_url logger = logging.getLogger(__name__) @@ -472,7 +474,7 @@ def _truthy(value: str | None) -> bool: _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"} _REFRESH_MODES = {"auto", "manual", "disabled"} -_MODEL_TOOL_MODES = {"none", "compact", "full"} +_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"} _MODEL_TOOL_MODE_ALIASES = { "regular": "full", "odysseus_compact": "compact", @@ -552,10 +554,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float: val = 0 if val > 0: return float(max(1, min(60, val))) - # llama.cpp and other local OpenAI-compatible servers can block briefly - # while warming/loading. A 2s local timeout makes working endpoints flicker - # offline before /v1/models is ready. - return 10.0 if category == "local" else 2.0 + # Keep the background refresh short so a single dead endpoint cannot stall + # the model picker. A timed-out refresh never clears a non-empty model + # cache, so a briefly-warming local server is simply retried next round. + return 2.0 def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float: @@ -1510,6 +1512,9 @@ def setup_model_routes(model_discovery): _refresh_inflight = {"v": False, "done": None} # coarse single-flight guard _REFRESH_FAILURE_BASE = 300.0 _REFRESH_FAILURE_MAX = 3600.0 + # Even a user-triggered refresh must not re-probe an endpoint that just + # failed/timed out on every click; keep a short cooldown. + _REFRESH_FAILURE_MIN = 60.0 def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str: # Session-backed endpoints carry no static key; include their auth id @@ -1557,14 +1562,20 @@ def setup_model_routes(model_discovery): if mode in ("manual", "disabled") and not force: return False, info fails = int(state.get("fail_count") or 0) - if fails and not force: + if fails: last_failure = float(state.get("last_failure") or 0.0) empty_local = ( not cached and category == "local" and str(getattr(ep, "id", "") or "").startswith("local-") ) - if now - last_failure < _failure_delay(fails, empty_local=empty_local): + delay = _failure_delay(fails, empty_local=empty_local) + if force: + # A user-triggered refresh gets a short cooldown instead of the + # long background backoff, but still skips a just-failed + # endpoint so repeated clicks cannot stall the picker. + delay = min(delay, _REFRESH_FAILURE_MIN) + if now - last_failure < delay: return False, info if cached and not force: interval = _endpoint_refresh_interval(ep, category) @@ -2163,6 +2174,19 @@ def setup_model_routes(model_discovery): # keep those container-local when the frontend marks them as such. base_url = _rewrite_loopback_for_docker(base_url, container_local=_truthy(container_local)) + # SSRF hardening: validate the user-supplied URL before any outbound + # request. Local-first means loopback/LAN endpoints are allowed by + # default; non-HTTP(S) schemes and the cloud metadata range are always + # rejected. Set MODELENDPOINT_BLOCK_PRIVATE_IPS=true for full lockdown. + # Note: the outbound fetches rely on httpx's default + # follow_redirects=False — a redirect target is never dereferenced. + ok, reason = check_outbound_url( + base_url, + block_private=os.getenv("MODELENDPOINT_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") + is_featherless = _host_match(base_url, "featherless.ai") or _safe_detect_provider(base_url) == "featherless" # Auto-generate name from URL if not provided if not name.strip(): @@ -2399,6 +2423,16 @@ def setup_model_routes(model_discovery): from src.endpoint_resolver import resolve_url base_url = resolve_url(base_url) base_url = _rewrite_loopback_for_docker(base_url) + # SSRF hardening: validate the user-supplied URL before any outbound + # request. Local-first means loopback/LAN endpoints are allowed by + # default; non-HTTP(S) schemes and the cloud metadata range are always + # rejected. Set MODELENDPOINT_BLOCK_PRIVATE_IPS=true for full lockdown. + ok, reason = check_outbound_url( + base_url, + block_private=os.getenv("MODELENDPOINT_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") requested_kind = _normalize_endpoint_kind(endpoint_kind) is_featherless = _host_match(base_url, "featherless.ai") or _safe_detect_provider(base_url) == "featherless" if is_featherless and requested_kind == "auto": @@ -2889,6 +2923,16 @@ def setup_model_routes(model_discovery): _new_base = _new_base[: -len(_suffix)].rstrip("/") _new_base = _normalize_base(_new_base) if _new_base: + # SSRF hardening: PATCH rewrites the stored base_url, so + # it needs the same guard as POST create/test — no + # bypass through the update path. Follow-up fetches + # rely on httpx's default follow_redirects=False. + ok, reason = check_outbound_url( + _new_base, + block_private=os.getenv("MODELENDPOINT_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise HTTPException(400, f"Rejected endpoint URL: {reason}") ep.base_url = _new_base else: ep.is_enabled = not ep.is_enabled @@ -3048,6 +3092,12 @@ def setup_model_routes(model_discovery): def update_tools(body: ToolsUpdate, request: Request): """Update which tools are disabled.""" require_admin(request) + from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN + internal_token = request.headers.get(INTERNAL_TOOL_HEADER) + if internal_token and secrets.compare_digest(internal_token, INTERNAL_TOOL_TOKEN): + # Owner impersonation can carry an admin username, so check the + # authenticated transport rather than request.state.current_user. + raise HTTPException(403, "Change tool policy in Settings > Agent Tools, not from chat") settings = _load_settings() settings["disabled_tools"] = body.disabled _save_settings(settings) @@ -3055,4 +3105,10 @@ def setup_model_routes(model_discovery): router._should_refresh_endpoint = _should_refresh_endpoint router._search_endpoint_catalog = search_endpoint_catalog + # Exposed for tests: the in-memory backoff state, its key function, and the + # background refresh driver (probes endpoints in parallel with the short + # per-endpoint timeout). + router._refresh_state = _refresh_state + router._refresh_key = _refresh_key + router._refresh_caches_bg = _refresh_caches_bg return router diff --git a/routes/research/research_routes.py b/routes/research/research_routes.py index 98897a875..89f426715 100644 --- a/routes/research/research_routes.py +++ b/routes/research/research_routes.py @@ -17,6 +17,7 @@ from src.endpoint_resolver import resolve_endpoint from src.auth_helpers import _auth_disabled, get_current_user from src.owner_identity import REQUEST_SENTINEL_OWNERS from src.constants import DEEP_RESEARCH_DIR +from src import shutdown as _shutdown _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$") @@ -285,11 +286,17 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: async def research_status(session_id: str, request: Request): user = _require_user(request) _validate_session_id(session_id) + # No in-flight or persisted research for this session is a normal state + # (most sessions have none), not an error. Return HTTP 200 with an + # explicit empty status so loading a session does not log a spurious + # 404; every client treats a non-"running" status as idle anyway. + # Research owned by ANOTHER user answers identically, so the response + # does not reveal whether that research exists. if not _owns_in_memory(session_id, user): - raise HTTPException(404, "No research found for this session") + return {"status": "none"} status = research_handler.get_status(session_id) if status is None: - raise HTTPException(404, "No research found for this session") + return {"status": "none"} try: source_state = research_handler.get_source_state(session_id) if isinstance(source_state, str) and source_state: @@ -657,7 +664,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter: raise HTTPException(404, "No research found for this session") async def _generate(): last_payload = None - while True: + while not _shutdown.is_shutting_down(): status = research_handler.get_status(session_id) if status is None: yield f"data: {json.dumps({'status': 'not_found'})}\n\n" diff --git a/routes/session_routes.py b/routes/session_routes.py index 9f6cc3659..1c903f194 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -1320,7 +1320,7 @@ def setup_session_routes( after Phase 1 — used by the "Tidy (no AI)" UI affordance so users can clean junk without spending tokens. """ - from src.llm_core import llm_call + from src.llm_core import llm_call, llm_call_with_fallback from src.utility_effort import effort_for_call user = effective_user(request) single_user_mode = not user and _auth_disabled() @@ -1467,7 +1467,7 @@ def setup_session_routes( return {"status": "skipped", "reason": "No unfiled sessions to sort"} # Pick an endpoint — prefer admin-configured task endpoint - from src.task_endpoint import resolve_task_endpoint + from src.task_endpoint import resolve_task_endpoint, resolve_task_candidates url, model, headers = resolve_task_endpoint(owner=user) if not url: url, model, headers = _pick_endpoint_for_sort(owner=user) @@ -1493,9 +1493,12 @@ def setup_session_routes( # 16384 (was 4096): with many chats the folder JSON is large, and a # reasoning model spends tokens thinking first — 4096 truncated the # JSON mid-output, so it never parsed ("invalid JSON for auto-sort"). - raw = llm_call(url, model, [{"role": "user", "content": prompt}], - temperature=0.3, max_tokens=16384, headers=headers, timeout=120, - reasoning_effort=effort_for_call(url, model, user)) + raw = llm_call_with_fallback( + resolve_task_candidates(fallback_url=url, fallback_model=model, + fallback_headers=headers, owner=user), + [{"role": "user", "content": prompt}], + temperature=0.3, max_tokens=16384, timeout=120, + reasoning_effort=effort_for_call(url, model, user)) logger.info(f"Auto-sort raw response ({len(raw)} chars): {raw[:300]}") # Extract JSON from response — handle markdown fences, leading text, # reasoning-model blocks, and trailing commas. diff --git a/routes/shell_routes.py b/routes/shell_routes.py index 8e2e969e0..b4684f779 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -27,6 +27,7 @@ from src.host_docker_access import ( from src.optional_deps import prepare_optional_dependency_import from src.auth_helpers import _auth_disabled from src import process_lifecycle +from src import shutdown as _shutdown # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist # on Windows, so importing them unconditionally crashed app startup there @@ -1025,7 +1026,7 @@ async def _generate_tmux(cmd: str, request: Request): lines_sent = 0 exit_code = None - while True: + while not _shutdown.is_shutting_down(): # Check client disconnect if await request.is_disconnected(): # tmux keeps running — that's the whole point @@ -1142,7 +1143,7 @@ async def _generate_win_detached(cmd: str, request: Request): lines_sent = 0 exit_code = None - while True: + while not _shutdown.is_shutting_down(): if await request.is_disconnected(): yield f"data: {json.dumps({'stream': 'stdout', 'data': f'Disconnected. Background job {session_id} continues running.'})}\n\n" return @@ -1267,7 +1268,7 @@ def setup_shell_routes() -> APIRouter: """Read chunks, split on \\n or \\r for progress bar support.""" try: buf = b"" - while True: + while not _shutdown.is_shutting_down(): chunk = await stream.read(4096) if not chunk: if buf: diff --git a/routes/upload_routes.py b/routes/upload_routes.py index 347da487e..92bbb84ea 100644 --- a/routes/upload_routes.py +++ b/routes/upload_routes.py @@ -25,11 +25,12 @@ from src.auth_helpers import effective_user, is_delegated_credential from src.owner_identity import auth_disabled, is_request_sentinel_owner from src.attachment_refs import attachment_refs_from_metadata from src.constants import GENERATED_IMAGES_DIR -from src.path_confinement import is_inside +from src.path_confinement import PathEscape, canonical_root, confine, is_inside from src.upload_handler import ( UploadCleanupSafetyError, count_recent_uploads, extract_upload_ids, + secure_filename, ) logger = logging.getLogger(__name__) @@ -153,30 +154,47 @@ def setup_upload_routes(upload_handler): def _upload_root() -> str: from src.constants import UPLOAD_DIR - return os.path.realpath(getattr(upload_handler, "upload_dir", UPLOAD_DIR)) + return canonical_root(getattr(upload_handler, "upload_dir", UPLOAD_DIR)) def _path_inside_upload_dir(path: str) -> bool: return is_inside(_upload_root(), path) def _resolve_upload_path(file_id: str) -> str: - from src.constants import UPLOAD_DIR - upload_root = getattr(upload_handler, "upload_dir", UPLOAD_DIR) - direct = os.path.join(upload_root, file_id) + safe_id = secure_filename(os.path.basename(file_id)) + if not safe_id: + raise HTTPException(400, "Invalid file ID") + root = _upload_root() + + direct = os.path.normpath(os.path.join(root, safe_id)) if os.path.lexists(direct): - if not _path_inside_upload_dir(direct): + try: + canonical_direct = confine(root, direct, allow_root=False) + except (PathEscape, ValueError): raise HTTPException(403, "Access denied") - if os.path.isfile(direct): - return direct + if os.path.isfile(canonical_direct): + return canonical_direct raise HTTPException(404, "File not found") - for root, _dirs, files in os.walk(upload_root, followlinks=False): - if file_id not in files: - continue - path = os.path.join(root, file_id) - if not _path_inside_upload_dir(path): + info = _load_upload_info(safe_id) + stored_path = (info or {}).get("path") + if stored_path: + try: + canonical_stored = confine(root, stored_path, allow_root=False) + if os.path.isfile(canonical_stored): + return canonical_stored + except (PathEscape, ValueError): raise HTTPException(403, "Access denied") - if os.path.isfile(path): - return path + + for walk_root, _dirs, files in os.walk(root, followlinks=False): + if safe_id not in files: + continue + cand = os.path.normpath(os.path.join(walk_root, safe_id)) + try: + canonical_cand = confine(root, cand, allow_root=False) + except (PathEscape, ValueError): + raise HTTPException(403, "Access denied") + if os.path.isfile(canonical_cand): + return canonical_cand raise HTTPException(404, "File not found") raise HTTPException(404, "File not found") @@ -435,52 +453,79 @@ def setup_upload_routes(upload_handler): logger.error(f"Failed to get upload stats: {e}") raise HTTPException(500, "Failed to get upload statistics") - @router.get("/{file_id}") + @router.api_route("/{file_id}", methods=["GET", "HEAD"]) async def download_file(request: Request, file_id: str, thumb: int = 0): - """Serve an uploaded file by its ID. `?thumb=1` returns a small cached - JPEG thumbnail for images (used by chat attachment previews) so the - client isn't downloading the full-resolution photo just to show it tiny.""" + """Serve an uploaded file by its ID or probe via HEAD. `?thumb=1` returns + a small cached JPEG thumbnail for images (used by chat attachment previews) + so the client isn't downloading the full-resolution photo just to show it tiny.""" if not upload_handler.validate_upload_id(file_id): raise HTTPException(400, "Invalid file ID") + safe_file_id = secure_filename(os.path.basename(file_id)) import mimetypes as _mt # Look up original filename and owner from uploads.json - original_name = file_id + original_name = safe_file_id # _load_upload_index() tolerates a missing/corrupt uploads.json (it falls # back to the .bak sibling, then to {}), so a truncated DB degrades to # "no metadata" instead of a 500 from an unhandled JSONDecodeError. db = upload_handler._load_upload_index() - info = next((fi for fi in db.values() if fi.get("id") == file_id), None) + info = next((fi for fi in db.values() if fi.get("id") == safe_file_id), None) if info: - original_name = info.get("name", file_id) + original_name = info.get("name", safe_file_id) file_owner = info.get("owner") if info else None _verify_upload_access(request, file_owner) - path = _resolve_upload_path(file_id) + upload_root = _upload_root() + path = _resolve_upload_path(safe_file_id) + try: + path = confine(upload_root, path, allow_root=False) + except (PathEscape, ValueError): + raise HTTPException(403, "Access denied") + if not path.startswith(upload_root + os.sep) or not _path_inside_upload_dir(path): + raise HTTPException(403, "Access denied") mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "application/octet-stream" from fastapi.responses import FileResponse # Downscaled thumbnail for image previews — generated once and cached. + # Skip generation during HEAD probe to avoid loading/processing whole image. if thumb and mime.startswith("image/"): + thumb_dir = os.path.normpath(os.path.join(upload_root, ".thumbs")) + thumb_path = os.path.normpath(os.path.join(thumb_dir, safe_file_id + ".jpg")) try: - from PIL import Image, ImageOps - thumb_dir = os.path.join(_upload_root(), ".thumbs") - os.makedirs(thumb_dir, exist_ok=True) - thumb_path = os.path.join(thumb_dir, file_id + ".jpg") - if (not os.path.exists(thumb_path) - or os.path.getmtime(thumb_path) < os.path.getmtime(path)): - im = Image.open(path) - # iPhone / camera JPEGs encode rotation in EXIF rather than - # the pixel data. Browsers honour that on the original via - # image-orientation:from-image, but PIL strips EXIF when it - # saves the JPEG thumb, leaving the pixels sideways. Bake - # the rotation into the pixels before thumbnailing. - im = ImageOps.exif_transpose(im) - im.thumbnail((320, 320)) - if im.mode not in ("RGB", "L"): - im = im.convert("RGB") - im.save(thumb_path, "JPEG", quality=80) - return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) - except Exception as e: - logger.warning(f"Thumbnail generation failed for {file_id}: {e}") - # Fall through to the full image. + thumb_path = confine(upload_root, thumb_path, allow_root=False) + except (PathEscape, ValueError): + raise HTTPException(403, "Access denied") + if not thumb_path.startswith(thumb_dir + os.sep) or not _path_inside_upload_dir(thumb_path): + raise HTTPException(403, "Access denied") + if request.method == "HEAD": + if (os.path.isfile(thumb_path) + and os.path.getmtime(thumb_path) >= os.path.getmtime(path)): + return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) + # GET may generate a JPEG or fall back to the original on failure. + # Neither representation is known without decoding the image. HEAD + # must not advertise the original/stale file's headers as a thumb. + from fastapi.responses import Response + response = Response(status_code=200, headers=UPLOAD_RESPONSE_HEADERS) + del response.headers["content-length"] + return response + else: + try: + from PIL import Image, ImageOps + os.makedirs(thumb_dir, exist_ok=True) + if (not os.path.exists(thumb_path) + or os.path.getmtime(thumb_path) < os.path.getmtime(path)): + im = Image.open(path) + # iPhone / camera JPEGs encode rotation in EXIF rather than + # the pixel data. Browsers honour that on the original via + # image-orientation:from-image, but PIL strips EXIF when it + # saves the JPEG thumb, leaving the pixels sideways. Bake + # the rotation into the pixels before thumbnailing. + im = ImageOps.exif_transpose(im) + im.thumbnail((320, 320)) + if im.mode not in ("RGB", "L"): + im = im.convert("RGB") + im.save(thumb_path, "JPEG", quality=80) + return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) + except Exception as e: + logger.warning(f"Thumbnail generation failed for {file_id}: {e}") + # Fall through to the full image. return FileResponse( path, media_type=mime, @@ -496,9 +541,10 @@ def setup_upload_routes(upload_handler): return next((fi for fi in db.values() if fi.get("id") == file_id), None) def _vision_cache_path(file_id: str) -> str: - cache_dir = os.path.join(_upload_root(), ".vision") + safe_id = secure_filename(os.path.basename(file_id)) + cache_dir = os.path.normpath(os.path.join(_upload_root(), ".vision")) os.makedirs(cache_dir, exist_ok=True) - return os.path.join(cache_dir, file_id + ".txt") + return os.path.normpath(os.path.join(cache_dir, safe_id + ".txt")) def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None: """Copy upload OCR/vision text onto the promoted gallery image row.""" diff --git a/scripts/check_brotli.py b/scripts/check_brotli.py new file mode 100644 index 000000000..3e8a52346 --- /dev/null +++ b/scripts/check_brotli.py @@ -0,0 +1,122 @@ +"""Fail image builds that select an unfixed Debian native Brotli decoder. + +Debian Node links libbrotlidec rather than bundling upstream Node's decoder. +Use Debian's backport revision, not BrotliDecoderVersion() or Python wheel +metadata, to establish the package floor for CVE-2026-111497: +https://security-tracker.debian.org/tracker/CVE-2026-111497 +""" + +import argparse +import ctypes +from pathlib import Path +import platform +import re +import subprocess +import sys + + +DEBIAN_FIXED = {"12": "1.0.9-2+deb12u1", "13": "1.1.0-2+deb13u1"} +DECODER_SONAME = "libbrotlidec.so.1" + + +def command(*args): + return subprocess.check_output(args, text=True).strip() + + +def check_package(): + release = platform.freedesktop_os_release() + floor = DEBIAN_FIXED.get(release.get("VERSION_ID")) + if release.get("ID") != "debian" or floor is None: + raise ValueError("Brotli package floor is not defined for this distribution") + record = command( + "dpkg-query", "-W", "-f=${Status}\n${Version}", "libbrotli1" + ).splitlines() + if len(record) != 2 or any(not value.strip() for value in record): + raise ValueError("invalid libbrotli1 package record: expected status and version") + status, version = record + if status != "install ok installed": + raise ValueError(f"libbrotli1 is not fully installed (status: {status})") + if subprocess.run( + ["dpkg", "--compare-versions", version, "ge", floor], check=False + ).returncode != 0: + raise ValueError(f"libbrotli1 {version} does not meet Debian's fixed floor {floor}") + owned = { + Path(line).resolve() + for line in command("dpkg-query", "-L", "libbrotli1").splitlines() + if Path(line).name == DECODER_SONAME + } + if not owned: + raise ValueError("libbrotli1 has no packaged native decoder") + return version, owned + + +def loaded_path(library): + # Inspect the library actually selected by the dynamic loader. An installed + # fixed package alone would not catch LD_LIBRARY_PATH or /usr/local shadows. + class DlInfo(ctypes.Structure): + _fields_ = [("filename", ctypes.c_char_p), ("base", ctypes.c_void_p), + ("symbol", ctypes.c_char_p), ("address", ctypes.c_void_p)] + + dladdr = ctypes.CDLL(None).dladdr + dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)] + dladdr.restype = ctypes.c_int + info = DlInfo() + if not dladdr(ctypes.cast(library.BrotliDecoderDecompress, ctypes.c_void_p), ctypes.byref(info)): + raise ValueError("cannot identify the loaded Brotli decoder") + return Path(info.filename.decode()).resolve() + + +def check_decoder(library): + # The fix adds error -17. Older decoders return INVALID here, including + # upstream 1.1.0 and 1.2.0. Fixed Debian 12 and 13 backports expose the same + # marker with different error-string prefixes. + # No malformed attacker-controlled stream is needed for this build check. + library.BrotliDecoderErrorString.argtypes = [ctypes.c_int] + library.BrotliDecoderErrorString.restype = ctypes.c_char_p + if library.BrotliDecoderErrorString(-17) not in (b"BLOCK_SWITCH", b"_ERROR_FORMAT_BLOCK_SWITCH"): + raise ValueError("native decoder lacks the CVE-2026-111497 BLOCK_SWITCH fix marker") + + library.BrotliDecoderDecompress.argtypes = [ + ctypes.c_size_t, ctypes.c_void_p, ctypes.POINTER(ctypes.c_size_t), ctypes.c_void_p + ] + library.BrotliDecoderDecompress.restype = ctypes.c_int + # A small, valid uncompressed Brotli stream must still decode correctly. + encoded = bytes.fromhex("0b058068656c6c6f20776f726c6403") + output = ctypes.create_string_buffer(11) + size = ctypes.c_size_t(len(output)) + result = library.BrotliDecoderDecompress(len(encoded), encoded, ctypes.byref(size), output) + if result != 1 or size.value != 11 or output.raw != b"hello world": + raise ValueError("native Brotli decoder failed the valid-stream control") + + +def check_node(node, decoder): + # ldd inspects the distro executable and its libnode dependency without + # executing JavaScript or relying on process.versions.brotli. + linked = command("ldd", str(node)) + match = re.search(r"libbrotlidec\.so\.1\s+=>\s+(/\S+)", linked) + if not match or Path(match[1]).resolve() != decoder: + raise ValueError(f"{node} does not select the checked Debian Brotli decoder") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--node", type=Path, help="also verify this distro Node executable's decoder") + args = parser.parse_args(argv) + try: + version, owned = check_package() + library = ctypes.CDLL(DECODER_SONAME) + decoder = loaded_path(library) + if decoder not in owned: + raise ValueError(f"selected decoder is outside libbrotli1: {decoder}") + check_decoder(library) + if args.node: + check_node(args.node, decoder) + except (ValueError, OSError, AttributeError, subprocess.CalledProcessError) as exc: + print(f"Brotli acceptance failed: {exc}", file=sys.stderr) + return 1 + print(f"Brotli accepted: Debian libbrotli1 {version}, decoder {decoder}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_ca_trust.py b/scripts/check_ca_trust.py new file mode 100644 index 000000000..d34fb53df --- /dev/null +++ b/scripts/check_ca_trust.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Reject the eMudhra incident's known CA certificates in PEM trust stores. + +This checks certificate contents, including OpenSSL's lazily loaded CA directory. +It does not model native OS certificate revocation or arbitrary custom clients. +Incident fingerprints: https://emudhra.we-have-a-problem.com/roots.json +and https://emudhra.we-have-a-problem.com/impacted-intermediates.json +Cross-signed issuer: https://emudhra.we-have-a-problem.com/chains.zip +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.util +import os +from pathlib import Path +import re +import ssl +import sys + + +BLOCKED_CERTIFICATES = { + "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b": "emSign ECC Root CA - G3", + "d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4": "AAA Certificate Services", + "7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d": "emSign Root TLS CA - G3", + "f2edaaff3d9b701fb4d14c9f285f8c0bad90fc3bcc1fa07d47f4718edaba34a5": "emSign Root CS CA - G3", + "3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd": "emSign Root TLS CA - G3 (cross-signed)", + # Intermediates can themselves be operator-supplied trust anchors. Include + # both published signatures for each key, without parsing SPKI in stdlib. + "42c4accd12c4f59597925e1a08a7a90102608543809308d50ec427e118ba1493": "EM DV TLS CA - G3B (eMudhra)", + "45a883f349aedfd5f3a649b5498ffb121712ba8b448f3f2a91dbc5a766fcddbe": "EM DV TLS CA - G3B (Sectigo)", + "98b896625c0c230adeec3eb2cf6aad5001ecf9156a68644e82374ce5833bc25b": "EM OV TLS CA - G3B (eMudhra)", + "5f1ded4dd590af74631290cf073b122a2aac6a1fa479778a0bd43579bd832699": "EM OV TLS CA - G3B (Sectigo)", + "e7568fe6e12331f4fd7de61da4c7559d0f8eb192a218d4741a3ba75184333838": "EM EV TLS CA - G3B (eMudhra)", + "579480aea166290bd3c643589e795fc0d0e54e86505d180f1dff99efd3199512": "EM EV TLS CA - G3B (Sectigo)", + "0164d7ceae5bb909c272af151ff44b1177ae9b47d9424b94f1d9558c30934d19": "EM DV TLS CA - G3C (eMudhra)", + "62b7f9868263f72f6911fc78ceaf72c1979158b7368bf648683bb2aa39b02e72": "EM DV TLS CA - G3C (Sectigo)", + "cb6e15165ac802e9b3f993f16c25d41f4e81c21ebde71ae1b012ac00bf9c7c3f": "EM OV TLS CA - G3C (eMudhra)", + "7485b51c0ed192878fa1d2f948fb529ae1842cc5086c4b67f7be5b2039de9489": "EM OV TLS CA - G3C (Sectigo)", + "432ec25f0d28de9006d97bf752f17bbb2cd41f669933fcb072d9a1148d0261f3": "EM EV TLS CA - G3C (eMudhra)", + "fd9d5c2dbbd4995ab69b8d50c439d9527b00529199c2fd01b2174eb58f07021f": "EM EV TLS CA - G3C (Sectigo)", + "9e39de2881273acc225a149c72ba3e0f51424963669ab633ee1dc7df5db067b4": "EM OV CS CA - G3B (eMudhra)", + "a496e23ac0cab8155fc9b117b97eae368efcbb9041aec8804d5a308d2d01c2d8": "EM OV CS CA - G3B (Sectigo)", + "4b2460a466f95fd40113bcdcca2ceb29b3b093921de45474da7df48874c630b1": "EM OV CS CA - G3C (eMudhra)", + "84112972b8e59d14560ae7e7119654474aad119990cf3722bfc1e3b91615e1fd": "EM OV CS CA - G3C (Sectigo)", +} +OVERRIDE_ENV_VARS = ( + "LLM_CA_BUNDLE", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", + "PIP_CERT", "NODE_EXTRA_CA_CERTS", "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH", +) + + +def check_bundle(path: Path) -> None: + """Raise on an unreadable, empty, malformed, or affected CA bundle.""" + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(cafile=str(path)) + # A hashed OpenSSL directory can include valid non-CA certificates (for + # example Ubuntu's snakeoil server certificate). get_ca_certs() omits them; + # use the total X.509 count to distinguish these from an empty store. + if not context.cert_store_stats()["x509"]: + raise ValueError("no certificates") + certificates = context.get_ca_certs(binary_form=True) + blocked = { + BLOCKED_CERTIFICATES[fingerprint] + for certificate in certificates + if (fingerprint := hashlib.sha256(certificate).hexdigest()) in BLOCKED_CERTIFICATES + } + if blocked: + raise ValueError("affected CA certificates: " + ", ".join(sorted(blocked))) + + +def ca_directory_paths(directory: Path) -> list[Path]: + """Return the hashed certificate files consumed by OpenSSL's CApath.""" + return [ + path for path in directory.iterdir() + if re.fullmatch(r"[0-9a-fA-F]{8}\.[0-9]+", path.name) and path.is_file() + ] + + +def system_paths() -> list[Path]: + """Inspect both the effective OpenSSL CA file and hashed directory entries.""" + defaults = ssl.get_default_verify_paths() + paths = [Path(defaults.cafile)] if defaults.cafile else [] + # get_default_verify_paths() can report None for a colon-separated + # SSL_CERT_DIR even though OpenSSL consumes each directory in that list. + capath = os.environ.get(defaults.openssl_capath_env, defaults.capath) + if capath: + for directory in filter(None, capath.split(os.pathsep)): + paths.extend(ca_directory_paths(Path(directory))) + if not paths: + raise ValueError("no effective OpenSSL PEM CA store found") + return paths + + +def override_paths() -> list[Path]: + """Inspect every configured override, including client fallback settings.""" + paths = [] + for name in OVERRIDE_ENV_VARS: + value = os.environ.get(name, "") + if name == "LLM_CA_BUNDLE": + value = value.strip() # Match src/tls_overrides.py. + if not value: + continue + path = Path(value) + # Requests (also used by pip) accepts either a file or a hashed directory. + if name in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "PIP_CERT") and path.is_dir(): + entries = ca_directory_paths(path) + if not entries: + raise ValueError(f"{name}: no hashed certificates in {path}") + paths.extend(entries) + else: + paths.append(path) + return paths + + +def grpc_roots_path() -> Path: + """Select gRPC's override or its independent Python-wheel default bundle.""" + override = os.environ.get("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH") + if override: + # gRPC can fall back to its bundled roots when this file is unreadable. + # Check the configured file strictly instead of blessing that fallback. + return Path(override) + spec = importlib.util.find_spec("grpc") + if spec is None or spec.origin is None: + raise ImportError("grpcio is not installed; cannot check its default roots") + return Path(spec.origin).parent / "_cython" / "_credentials" / "roots.pem" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundles", nargs="*", type=Path) + parser.add_argument("--system", action="store_true", help="check effective OpenSSL PEM stores") + parser.add_argument("--python", action="store_true", help="check certifi and pip's separate vendored bundle") + parser.add_argument("--grpc", action="store_true", help="check gRPC's selected default CA file") + parser.add_argument("--overrides", action="store_true", help="check configured LLM/HTTPX/Requests/pip/Node/gRPC CA overrides") + parser.add_argument("--verbose", action="store_true", help="show the runtime identity, Python executable and selected CA paths") + args = parser.parse_args(argv) + if not (args.bundles or args.system or args.python or args.grpc or args.overrides): + parser.error("select --system, --python, --grpc, --overrides, or explicit PEM bundle paths") + if args.verbose: + print(f"Python executable: {sys.executable}") + print(f"Python prefix: {sys.prefix}") + print(f"User home: {Path.home()}") + if hasattr(os, "geteuid"): + print(f"Effective user: uid={os.geteuid()} gid={os.getegid()}") + try: + paths = list(args.bundles) + if args.system: + paths.extend(system_paths()) + if args.python: + for name in ("certifi", "pip._vendor.certifi"): + paths.append(Path(importlib.import_module(name).where())) + if args.grpc: + paths.append(grpc_roots_path()) + if args.overrides: + paths.extend(override_paths()) + paths = list(dict.fromkeys(path.resolve() for path in paths)) + if not paths: + raise ValueError("no CA bundle files selected; also select --system, --python, or an explicit path") + errors = [] + for path in paths: + if args.verbose: + print(f"CA bundle: {path}") + try: + check_bundle(path) + except (OSError, ValueError, ssl.SSLError) as exc: + errors.append(f"{path}: {exc}") + if errors: + raise ValueError("\n".join(errors)) + except (ImportError, OSError, ValueError, ssl.SSLError) as exc: + print(str(exc), file=sys.stderr) + return 1 + print(f"Validated {len(paths)} CA bundle file(s): no known eMudhra incident CA certificates") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_env_reference.py b/scripts/generate_env_reference.py index 06a8a4c64..c83c992f4 100644 --- a/scripts/generate_env_reference.py +++ b/scripts/generate_env_reference.py @@ -914,6 +914,78 @@ VARIABLE_NOTES: dict[str, tuple[str, str, str]] = { "Overrides the source commit reported for runtime provenance, for builds " "that ship without a git directory.", ), + + # -- Photo editor and image operations ----------------------------------- + "ODYSSEUS_EDITOR_AGENT_BASES": ( + "Model routing and providers", USER, + "Comma-separated base URLs for the image-editor agent backends; a single " + "base URL in the model endpoint config wins over this list.", + ), + "ODYSSEUS_ALLOW_LOCAL_IMAGE_MODELS": ( + "Model routing and providers", USER, + "Set to 1, true, yes or on to allow local image-generation and editing " + "backends to be reachable as image providers.", + ), + "ODYSSEUS_IMAGE_OPS_BASE": ( + "Model routing and providers", USER, + "Base URL of the image-operations backend (inpaint, outpainting and style " + "transfer); a trailing slash is stripped before use.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_SIDE": ( + "Upload and media limits", USER, + "Maximum pixel side (width or height) an edited image is downscaled to " + "before it is sent to the image backend.", + ), + "ODYSSEUS_IMAGE_EDIT_MAX_STEPS": ( + "Upload and media limits", USER, + "Maximum number of iterative edit steps the image backend may perform in " + "a single request.", + ), + "ODYSSEUS_IMAGE_OUTPAINT_TIMEOUT": ( + "Upload and media limits", USER, + "Seconds the outpainting proxy waits for the image backend before giving " + "up on a single request.", + ), + "ODYSSEUS_JEFF_": ("Agent loop and tool execution", INTERNAL, "Prefix for per-job decision adapter opt-ins; each job is disabled by default."), + "ODYSSEUS_JEFF_TIMEOUT": ("Agent loop and tool execution", INTERNAL, "Timeout in seconds for a decision adapter request."), + "ODYSSEUS_JEFF_URL": ("Agent loop and tool execution", INTERNAL, "Base URL of the optional decision adapter service."), + "ODYSSEUS_PRIVATE_BROWSER_LEGACY": ("Agent loop and tool execution", INTERNAL, "Explicit opt-in to the legacy browser compatibility transport; disabled by default."), + "ODYSSEUS_SPEC_BROWSER_FALLBACK": ("Agent loop and tool execution", INTERNAL, "Optional browser routing fallback: mcp, web, or 0 to disable."), + "ODYSSEUS_SPEC_MODE": ("Agent loop and tool execution", INTERNAL, "Spec routing mode for explicitly configured models, including strict and driver."), + "ODYSSEUS_SPEC_ROUTER_MIN_CONF": ("Agent loop and tool execution", INTERNAL, "Minimum tool-selection confidence accepted from the router."), + "ODYSSEUS_SPEC_ROUTER_MODEL": ("Agent loop and tool execution", INTERNAL, "Decision adapter model used for tool routing."), + "ODYSSEUS_SPEC_ROUTER_SECOND": ("Agent loop and tool execution", INTERNAL, "Minimum confidence for adding a secondary tool candidate."), + "ODYSSEUS_SPEC_ROUTER_TIMEOUT": ("Agent loop and tool execution", INTERNAL, "Time budget in seconds for a router decision."), + "ODYSSEUS_SPEC_ROUTER_URL": ("Agent loop and tool execution", INTERNAL, "Base URL of the optional Spec router; an empty URL disables requests."), + "ODYSSEUS_SPEC_THINK_MIN_P": ("Agent loop and tool execution", INTERNAL, "Minimum router confidence for enabling thinking."), + "ODYSSEUS_SPEC_THINK_MODEL": ("Agent loop and tool execution", INTERNAL, "Optional thinking decision adapter model, falling back to the tool router model."), + "ODYSSEUS_SPEC_THINK_QUESTION": ("Agent loop and tool execution", INTERNAL, "Thinking classifier question format: binary or kind5."), + "ODYSSEUS_SPEC_THINK_ROUTER": ("Agent loop and tool execution", INTERNAL, "Enable router thinking decisions for explicitly configured Spec models."), + "ODYSSEUS_SPEC_TOOL_FORMAT": ("Agent loop and tool execution", INTERNAL, "Use names for compact router tool signatures, otherwise include descriptions."), + "ODYSSEUS_SPEC_TRIM_PROMPT": ("Agent loop and tool execution", INTERNAL, "Enable prompt trimming for the selected Spec tool menu."), + "ODYSSEUS_VISION_MAX_SIDE": ("Upload and media limits", USER, "Maximum pixel side for image attachments sent directly to a vision model."), + "ODYSSEUS_VISION_MODELS": ("Model routing and providers", USER, "Comma-separated model-name fragments that explicitly support inline images."), + "ODYSSEUS_VISION_PROBE": ("Model routing and providers", USER, "Allow a cached vision-capability probe against local model endpoints."), + "ODYSSEUS_SPEC_DIRECT_MODE": ( + "Agent loop and tool execution", INTERNAL, + "Controls how the spec router answers high-confidence direct-answer turns; " + "either tool_choice_none (default) or empty_menu.", + ), + "ODYSSEUS_SPEC_DIRECT_NOTOOLS": ( + "Agent loop and tool execution", INTERNAL, + "Set to 1, true, yes or on to enable the spec router's direct-answer " + "no-tools path for high-confidence answer_directly decisions.", + ), + "ODYSSEUS_SPEC_DIRECT_MIN_P": ( + "Agent loop and tool execution", INTERNAL, + "Minimum confidence probability a spec-router direct-answer decision must " + "reach before the no-tools path is taken.", + ), + "ODYSSEUS_GRACEFUL_SHUTDOWN_TIMEOUT": ( + "Deployment and first run", INTERNAL, + "Seconds the server waits for in-flight requests and background loops to " + "finish during a graceful shutdown.", + ), } diff --git a/scripts/sanitize_trace_artifacts.py b/scripts/sanitize_trace_artifacts.py new file mode 100644 index 000000000..ea975b3b5 --- /dev/null +++ b/scripts/sanitize_trace_artifacts.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +"""Sanitize Playwright trace archives to strip sensitive credentials and tokens. + +Structured, format-aware fail-closed security: +Replaces naive regex-based replacement with structured JSON / HAR parsing, +recursive key/value redaction, URL credential scrubbing, DOM snapshot sanitization, +safe source code masking, and strict fail-closed purge on corruption or error. +""" +from __future__ import annotations + +import json +import logging +import os +import re +import sys +import urllib.parse +import zipfile +from typing import Any, Dict, List, Set + +logger = logging.getLogger(__name__) + +BINARY_IMAGE_EXTENSIONS = ('.png', '.jpg', '.jpeg', '.webp', '.gif', '.ico') + +SENSITIVE_KEY_PATTERN = re.compile( + r'(?i)(password|passwd|secret|token|api_key|apikey|access_token|refresh_token|private_key|auth_token|client_secret|session|session_id|sid|csrf|xsrf|jwt|credential|authorization)' +) + +SENSITIVE_PARAM_NAMES = frozenset({ + "token", "session", "auth", "password", "passwd", "key", "secret", "api_key", + "apikey", "access_token", "refresh_token", "jwt", "sid", "code", "sig", + "signature", "client_secret", "credential" +}) + +SENSITIVE_HEADER_NAMES = frozenset({ + "authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", + "api-key", "x-auth-token", "x-access-token" +}) + + +def sanitize_url(url: str) -> str: + """Sanitize query parameters and authority credentials in URLs.""" + if not isinstance(url, str) or not url.strip(): + return url + try: + parts = urllib.parse.urlsplit(url) + except Exception: + return url + + netloc = parts.netloc + if "@" in netloc: + userinfo, host = netloc.split("@", 1) + if ":" in userinfo: + user, _ = userinfo.split(":", 1) + netloc = f"{user}:[REDACTED]@{host}" + else: + netloc = f"[REDACTED]@{host}" + + query = parts.query + if query: + q_pairs = urllib.parse.parse_qsl(query, keep_blank_values=True) + sanitized_pairs = [] + for k, v in q_pairs: + k_lower = k.lower().replace("-", "_") + if k_lower in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k_lower): + sanitized_pairs.append((k, "[REDACTED]")) + else: + sanitized_pairs.append((k, v)) + query = urllib.parse.urlencode(sanitized_pairs) + + return urllib.parse.urlunsplit((parts.scheme, netloc, parts.path, query, parts.fragment)) + + +def sanitize_body_text(text: str) -> str: + """Sanitize request bodies formatted as JSON or URL-encoded form data.""" + if not isinstance(text, str) or not text.strip(): + return text + + # 1. Try JSON + trimmed = text.strip() + if (trimmed.startswith("{") and trimmed.endswith("}")) or (trimmed.startswith("[") and trimmed.endswith("]")): + try: + parsed = json.loads(trimmed) + sanitized = sanitize_structured_obj(parsed) + return json.dumps(sanitized, ensure_ascii=False) + except Exception: + pass + + # 2. Try URL-encoded form data + if "=" in trimmed and not trimmed.startswith("<"): + try: + pairs = urllib.parse.parse_qsl(trimmed, keep_blank_values=True) + if pairs and any(k.lower() in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k) for k, _ in pairs): + out_pairs = [] + for k, v in pairs: + k_lower = k.lower().replace("-", "_") + if k_lower in SENSITIVE_PARAM_NAMES or SENSITIVE_KEY_PATTERN.search(k_lower): + out_pairs.append((k, "[REDACTED]")) + else: + out_pairs.append((k, v)) + return urllib.parse.urlencode(out_pairs) + except Exception: + pass + + return text + + +def sanitize_structured_obj(obj: Any) -> Any: + """Recursively sanitize Python objects decoded from JSON or HAR payloads.""" + if isinstance(obj, dict): + new_dict: Dict[str, Any] = {} + + # Check for header entry with inverted or standard keys: {"name": ..., "value": ...} + name_val = obj.get("name") + val_val = obj.get("value") + if isinstance(name_val, str) and isinstance(val_val, str): + name_lower = name_val.lower().strip() + if name_lower in SENSITIVE_HEADER_NAMES or SENSITIVE_KEY_PATTERN.search(name_lower): + for k, v in obj.items(): + if k == "value": + new_dict[k] = "[REDACTED]" + else: + new_dict[k] = sanitize_structured_obj(v) + return new_dict + + # Process standard dictionary keys + for k, v in obj.items(): + k_lower = str(k).lower().replace("-", "_") + + # Action parameters in Playwright traces (e.g. page.fill, page.type) + if k == "params" and isinstance(v, dict): + sanitized_params = sanitize_structured_obj(v) + # If this action involves fill, type, or sensitive inputs, redact value/text + if "value" in sanitized_params: + sanitized_params["value"] = "[REDACTED]" + if "text" in sanitized_params: + sanitized_params["text"] = "[REDACTED]" + new_dict[k] = sanitized_params + continue + + # DOM snapshot nodes + if k == "attributes" and isinstance(v, (dict, list)): + if isinstance(v, dict): + attr_dict = dict(v) + type_val = str(attr_dict.get("type") or "").lower() + if type_val == "password" or "password" in str(attr_dict.get("name") or "").lower(): + if "value" in attr_dict: + attr_dict["value"] = "[REDACTED]" + new_dict[k] = attr_dict + continue + elif isinstance(v, list): + # List of attribute strings or pairs + new_list = [] + is_pw = any("password" in str(x).lower() for x in v) + for x in v: + if is_pw and isinstance(x, str) and not x.startswith("type="): + new_list.append("[REDACTED]") + else: + new_list.append(sanitize_structured_obj(x)) + new_dict[k] = new_list + continue + + # Request postData / text + if k in ("postData", "postDataText") and isinstance(v, str): + new_dict[k] = sanitize_body_text(v) + continue + + # URL fields + if k == "url" and isinstance(v, str): + new_dict[k] = sanitize_url(v) + continue + + # Cookie arrays + if k == "cookies" and isinstance(v, list): + sanitized_cookies = [] + for c in v: + if isinstance(c, dict): + c_copy = dict(c) + if "value" in c_copy: + c_copy["value"] = "[REDACTED]" + sanitized_cookies.append(c_copy) + else: + sanitized_cookies.append(sanitize_structured_obj(c)) + new_dict[k] = sanitized_cookies + continue + + # Direct sensitive key matches + if SENSITIVE_KEY_PATTERN.search(k_lower) and isinstance(v, str): + new_dict[k] = "[REDACTED]" + continue + + new_dict[k] = sanitize_structured_obj(v) + + return new_dict + + elif isinstance(obj, list): + return [sanitize_structured_obj(item) for item in obj] + + elif isinstance(obj, str): + # Check for inline credentials/tokens + if obj.startswith("Bearer ") or obj.startswith("bearer "): + return "Bearer [REDACTED]" + if obj.startswith("Basic ") or obj.startswith("basic "): + return "Basic [REDACTED]" + return obj + + return obj + + +def sanitize_jsonl_content(data: bytes) -> bytes: + """Sanitize newline-delimited JSON Lines (.trace, .network, .stacks).""" + lines = data.split(b"\n") + out_lines = [] + for line in lines: + if not line.strip(): + out_lines.append(line) + continue + try: + parsed = json.loads(line.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + # Not valid JSON, keep as-is + out_lines.append(line) + continue + sanitized = sanitize_structured_obj(parsed) + out_lines.append(json.dumps(sanitized, ensure_ascii=False).encode("utf-8")) + return b"\n".join(out_lines) + + +def sanitize_trace_archive(zp: str) -> bool: + """Sanitize a single zip archive in a strict fail-closed manner. + + If the archive is malformed, partial, unreadable, or encounters an exception + during processing, the archive is permanently deleted so that raw traces + cannot leak. + Returns True if successfully sanitized, False if deleted due to error. + """ + if not os.path.exists(zp): + return False + + temp_zp = zp + ".tmp" + try: + if os.path.getsize(zp) == 0: + raise zipfile.BadZipFile("Empty archive") + + with zipfile.ZipFile(zp, "r") as zin: + infolist = zin.infolist() + if not infolist: + raise zipfile.BadZipFile("Archive has no entries") + + corrupt_file = zin.testzip() + if corrupt_file is not None: + raise zipfile.BadZipFile(f"Corrupt entry detected: {corrupt_file}") + + with zipfile.ZipFile(temp_zp, "w", compression=zipfile.ZIP_DEFLATED) as zout: + for item in infolist: + filename_lower = item.filename.lower() + + # 1. Binary image screenshots/diagnostics: preserve as-is + if filename_lower.endswith(BINARY_IMAGE_EXTENSIONS): + content = zin.read(item.filename) + zout.writestr(item, content) + continue + + # 2. Embedded test source code (e.g. resources/src@... or .js/.ts): + # Mask safely without syntax corruption! + if "src@" in filename_lower or filename_lower.endswith((".js", ".ts", ".mjs", ".py")): + safe_placeholder = b"// [SOURCE CODE REDACTED FOR TRACE CONFIDENTIALITY]\n" + zout.writestr(item, safe_placeholder) + continue + + # 3. JSON Lines and structured JSON/HAR traces + content = zin.read(item.filename) + if filename_lower.endswith((".trace", ".network", ".stacks")): + sanitized_content = sanitize_jsonl_content(content) + zout.writestr(item, sanitized_content) + elif filename_lower.endswith((".json", ".har")): + try: + parsed = json.loads(content.decode("utf-8")) + sanitized = sanitize_structured_obj(parsed) + zout.writestr(item, json.dumps(sanitized, ensure_ascii=False).encode("utf-8")) + except Exception: + # If invalid JSON, treat as text line-by-line + zout.writestr(item, sanitize_jsonl_content(content)) + else: + # Unknown binary or text member: exclude to fail closed + continue + + # Atomic replace + os.replace(temp_zp, zp) + return True + + except Exception as exc: + logger.warning("Sanitizing trace %s failed (%s); removing fail-closed", zp, exc) + for path_to_remove in (temp_zp, zp): + if os.path.exists(path_to_remove): + try: + os.remove(path_to_remove) + except OSError as remove_err: + logger.error("Failed to delete unredacted trace %s: %s", path_to_remove, remove_err) + return False + + +def sanitize_directory(results_dir: str) -> int: + """Walk results_dir and sanitize every .zip archive fail-closed. + + Returns the number of archives processed. Exits with 1 if any archive fails. + """ + if not os.path.exists(results_dir): + return 0 + + count = 0 + all_ok = True + for root, _, files in os.walk(results_dir): + for f in files: + if f.endswith(".zip"): + zp = os.path.join(root, f) + ok = sanitize_trace_archive(zp) + if not ok: + all_ok = False + count += 1 + + if not all_ok: + sys.exit(1) + return count + + +def main() -> None: + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + target_dir = sys.argv[1] + sanitize_directory(target_dir) + + +if __name__ == "__main__": + main() diff --git a/services/memory/memory_extractor.py b/services/memory/memory_extractor.py index 6b911b4d5..dfaf71e25 100644 --- a/services/memory/memory_extractor.py +++ b/services/memory/memory_extractor.py @@ -15,6 +15,7 @@ import json import logging import os import re +import time from typing import Optional from src.memory import MemoryStoreUnreadable @@ -78,11 +79,13 @@ EXTRACT_SYSTEM_PROMPT = ( "things the assistant said, one-off tasks, opinions on the current topic.\n\n" "Rules:\n" "- MAX 2 facts per conversation — only the most important\n" - "- Only extract facts the USER stated or clearly implied\n" + "- Only extract facts the USER explicitly stated about themselves\n" + "- Questions, requests to inspect records, and tool/assistant output are not evidence\n" "- Each fact must be a single short sentence (under 15 words)\n" "- If a fact is similar to something likely already known, skip it\n" "- If nothing durable was revealed, return []\n\n" - "Return a JSON array of objects with 'text' and 'category' fields.\n" + "Return a JSON array of objects with 'text', 'category', and 'evidence' fields. " + "Evidence must be an exact quote from a user's own factual statement.\n" "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n" "Return ONLY valid JSON, no markdown fences." ) @@ -213,7 +216,10 @@ def _fallback_memory_candidates(messages) -> list[dict]: if not text: continue - m = re.search(r"\bmy name is\s+([A-Za-z][A-Za-z0-9 .'\-]{1,50})\b", text, re.I) + m = re.search( + r"\bmy name is\s+([A-Za-z][A-Za-z0-9 '\-]{0,49}?)" + r"(?=\s+(?:and|but|please)\b|[.!?;,\n]|$)", text, re.I, + ) if m: name = _clean_memory_value(m.group(1), 50) if name: @@ -259,22 +265,183 @@ def _fallback_memory_candidates(messages) -> list[dict]: return candidates[:2] +_USER_FACT_ASSERTION = re.compile( + r"\b(?:i(?:'m| am| live| work| use| have| prefer| like| love| hate| own| study| grew up)\b" + r"|my\s+[a-z][a-z -]{0,40}\s+(?:is|are|was|works|lives)\b" + r"|we\s+(?:are|live|work|have|own)\b" + r"|our\s+[a-z][a-z -]{0,40}\s+(?:is|are|was)\b)", + re.IGNORECASE, +) + + +def _has_user_fact_evidence(fact: dict, messages) -> bool: + """Require an exact user assertion before persisting an LLM-inferred memory.""" + evidence = fact.get("evidence") + if not isinstance(evidence, str): + return False + evidence = re.sub(r"\s+", " ", evidence).strip() + if len(evidence.split()) < 3 or not _USER_FACT_ASSERTION.match(evidence): + return False + for message in messages: + if _message_role(message) != "user": + continue + utterance = re.sub(r"\s+", " ", _message_text(message)).strip() + position = utterance.casefold().find(evidence.casefold()) + if position < 0: + continue + # Judge the clause containing the quote, so a later question does not + # erase an earlier factual statement in the same user turn. + clause_start = max( + utterance.rfind(mark, 0, position) for mark in (".", "?", "!", ";") + ) + 1 + clause_end = min( + [end for mark in (".", "?", "!", ";") + if (end := utterance.find(mark, position)) >= 0] or [len(utterance)] + ) + clause = utterance[clause_start:clause_end].strip() + # A question about one's records is not a statement of the answer. + if utterance[clause_end:clause_end + 1] == "?" or re.match( + r"^(?:who|what|where|when|why|how|do|does|did|can|could|would|" + r"show|list|find|search|tell|summarize|open|read)\b", + clause, re.IGNORECASE, + ): + continue + return True + return False + + +_TOKEN_RE = re.compile(r"[a-z0-9]+") + +# Function words carry no distinguishing content. Dropping them before +# comparing two facts keeps punctuation and harmless rephrasing from defeating +# the duplicate check. +_STOPWORDS = frozenset( + "a an the and or but if then of to in on at for with by from as is are was " + "were be been being do does did has have had it its this that these those " + "there here user users someone somebody they them their he she his her him i " + "we you your my our me us who whom whose what when where why how also " + "currently now still about into over under again more most very can could " + "may might must will would should".split() +) + +# Words that frame a name/username statement; excluded when reading the value +# so the actual name tokens can be compared. +_NAME_TEMPLATE_WORDS = frozenset( + "name names named called call appears appear seems seem the user users his " + "her their it is are was were be been being username usernames login " + "account handle".split() +) + +_NAME_KEYWORDS = frozenset({"name", "named", "called"}) +_USERNAME_KEYWORDS = frozenset({"username", "login"}) + + +def _content_tokens(text: str) -> set: + """Lowercased content tokens with punctuation and stopwords removed.""" + toks = _TOKEN_RE.findall((text or "").lower()) + return {t for t in toks if t not in _STOPWORDS and len(t) >= 2} + + +def _name_slot(text: str): + """Return (slot, value_tokens) when text states a name/username, else None. + + Tokens are scanned so "username" is never mistaken for "name", and the + first keyword decides the slot (a sentence about a server *named* X that + also mentions a *username* Y is a 'name' fact, not a username fact).""" + words = _TOKEN_RE.findall((text or "").lower()) + slot = None + for w in words: + if w in _NAME_KEYWORDS: + slot = "name" + break + if w in _USERNAME_KEYWORDS: + slot = "username" + break + if not slot: + return None + value = { + w for w in words + if w not in _NAME_TEMPLATE_WORDS and w not in _STOPWORDS and len(w) >= 2 + } + return (slot, value) if value else None + + +def _name_slot_tokens(text: str): + slot = _name_slot(text) + return slot[1] if slot else None + + +def _name_vals_close(a: set, b: set) -> bool: + """True when two name values state the same name: identical, or one is a + strict refinement of the other where the larger value is a clean name + (<=4 alphabetic tokens). Guards against merging a username into a fact + like 'a server named Odysseus ... with username pewds'.""" + if a == b: + return True + if not a or not b: + return False + bigger = a if len(a) >= len(b) else b + clean = len(bigger) <= 4 and all(t.isalpha() for t in bigger) + return clean and (a <= b or b <= a) + + def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) -> bool: - """Check if new_text is too similar to any existing memory (Jaccard similarity).""" - new_tokens = set(new_text.lower().split()) + """Check if new_text states the same fact as an existing memory. + + Three tests, cheapest first: symmetric Jaccard over content tokens, + containment (one fact is a strict refinement of the other), and a + name/username-slot match. The naive whole-string comparison this replaces + kept punctuation (so "Alex." != "alex") and missed short-vs-full names + ("Felix" vs "FELIX ARVID ULF KJELLBERG" scored 0.43). + """ + new_tokens = _content_tokens(new_text) if not new_tokens: return False + new_slot = _name_slot(new_text) for entry in _memory_dicts(existing): - old_tokens = set(entry.get("text", "").lower().split()) + old_text = entry.get("text", "") + old_tokens = _content_tokens(old_text) if not old_tokens: continue intersection = new_tokens & old_tokens + if not intersection: + continue union = new_tokens | old_tokens if len(intersection) / len(union) >= threshold: return True + # Containment: one fact is a strict refinement of the other. Guard on + # the shorter side so two-token facts ("Likes Python") are not merged + # into longer sentences that merely start the same way. + if len(intersection) >= 3 and len(intersection) / min(len(new_tokens), len(old_tokens)) >= 0.85: + return True + # Name slot: both state the same kind of name and the values match + # (short vs full name, or the same name behind different phrasing). + old_slot = _name_slot(old_text) + if new_slot and old_slot and new_slot[0] == old_slot[0]: + if _name_vals_close(new_slot[1], old_slot[1]): + return True return False +def _name_enrichment_target(new_text: str, existing: list): + """Return an existing entry whose name value is a strict subset of the + new fact's, so the fuller name replaces it in place; else None.""" + ns = _name_slot(new_text) + if not ns: + return None + slot, new_val = ns + best = None + for entry in _memory_dicts(existing): + old_slot = _name_slot(entry.get("text", "")) + if not old_slot or old_slot[0] != slot: + continue + old_val = old_slot[1] + if old_val < new_val and _name_vals_close(old_val, new_val) and len(new_val) <= 4 and all(t.isalpha() for t in new_val): + if best is None or len(entry.get("text", "")) < len(best.get("text", "")): + best = entry + return best + + def _parse_extraction_json(raw: str) -> list: """Parse the extraction LLM's reply into a list of facts, tolerating reasoning-model noise. @@ -420,8 +587,13 @@ async def extract_and_store( if not isinstance(facts, list): facts = [] + # The extraction model sees assistant/tool text and can mistake a + # failed lookup or a one-off request for a durable personal fact. + # Only its claims with a quoted user assertion may enter memory. + facts = [fact for fact in facts if isinstance(fact, dict) + and _has_user_fact_evidence(fact, stripped_recent)] if fallback_facts: - facts = list(facts) + fallback_facts + facts.extend(fallback_facts) if not facts: logger.info("Auto memory extraction ran: 0 candidates") @@ -456,6 +628,14 @@ async def extract_and_store( if not fact_text or len(fact_text) < 5: continue + # A fuller owner-specific name must not rewrite a shared legacy + # entry, but that less-specific entry must not suppress it either. + legacy_refinements = { + id(entry) for entry in existing + if _owner and entry.get("owner") is None + and _name_enrichment_target(fact_text, [entry]) is entry + } + # Dedup: check vector similarity first (fast), then exact text match. # A runtime embedding/ChromaDB failure (backend OOM, model evicted, # remote endpoint down) must not abort the whole batch — fall through @@ -477,12 +657,38 @@ async def extract_and_store( # silently dropped. Mirror the owner predicate used by the # text dedup below; cross-tenant/stale matches fall through. _match = next((e for e in existing if e.get("id") == existing_id), None) - if _match is not None and (_match.get("owner") == _owner or _match.get("owner") is None): + if (_match is not None and id(_match) not in legacy_refinements + and (_match.get("owner") == _owner or _match.get("owner") is None)): logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}") continue # Text dedup fallback: exact match + fuzzy similarity - user_existing = [e for e in existing if e.get("owner") == _owner or e.get("owner") is None] if _owner else existing + user_existing = [e for e in existing + if (e.get("owner") == _owner or e.get("owner") is None) + and id(e) not in legacy_refinements] if _owner else existing + + # Name enrichment: if an existing name fact is a strict prefix of + # this more complete one, upgrade it in place instead of adding a + # near-duplicate ("name is Felix" -> "name is Felix Arvid Ulf + # Kjellberg"). The vector store upserts by id, so re-adding is safe. + # Legacy unowned entries may be visible to multiple owners; never + # rewrite one with a private fact from the current owner. + _enrich = _name_enrichment_target( + fact_text, [e for e in user_existing if e.get("owner") == _owner] + ) + if _enrich is not None: + _enrich["text"] = fact_text + _enrich["timestamp"] = int(time.time()) + if category: + _enrich["category"] = category + if memory_vector and memory_vector.healthy: + try: + memory_vector.add(_enrich["id"], fact_text) + except Exception as e: + logger.warning(f"Memory vector update failed for {_enrich['id']}: {e}") + added += 1 + continue + if memory_manager.find_duplicates(fact_text, user_existing): continue # Fuzzy text similarity check (catches rephrased duplicates when vector index is unavailable) diff --git a/src/activity.py b/src/activity.py new file mode 100644 index 000000000..92c4098e5 --- /dev/null +++ b/src/activity.py @@ -0,0 +1,61 @@ +"""In-flight activity counters (heavy image ops) for safe restarts/observability. + +A deploy restart kills any in-flight long image op (a diffusion edit is 60-120 s), +so the operator needs to know whether the process is busy before issuing +``systemctl --user restart``. These counters are process-local and cheap. +""" +from __future__ import annotations + +import threading +import time +from contextlib import contextmanager +from typing import Dict + +_lock = threading.Lock() +_image_ops = 0 +_image_ops_by_name: Dict[str, int] = {} +_started_at = time.time() + + +def begin_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops += 1 + _image_ops_by_name[name] = _image_ops_by_name.get(name, 0) + 1 + + +def end_image_op(name: str = "image") -> None: + global _image_ops + with _lock: + _image_ops = max(0, _image_ops - 1) + if name in _image_ops_by_name: + _image_ops_by_name[name] -= 1 + if _image_ops_by_name[name] <= 0: + _image_ops_by_name.pop(name, None) + + +@contextmanager +def image_op(name: str = "image"): + begin_image_op(name) + try: + yield + finally: + end_image_op(name) + + +def activity() -> Dict[str, object]: + with _lock: + return { + "image_ops": _image_ops, + "image_ops_by_name": dict(_image_ops_by_name), + "uptime_s": round(time.time() - _started_at, 1), + } + + +def heavy_image_ops_active() -> int: + with _lock: + return _image_ops + + +def busy() -> bool: + return heavy_image_ops_active() > 0 diff --git a/src/agent_evidence.py b/src/agent_evidence.py index b3c7e2b22..e012ebe66 100644 --- a/src/agent_evidence.py +++ b/src/agent_evidence.py @@ -240,11 +240,13 @@ def _artifact_match_is_callable(instruction: str, match: re.Match[str], path: st def _artifact_match_is_email_host(instruction: str, match: re.Match[str]) -> bool: - """Reject the domain portion of an email address as an output path.""" + """Reject either dotted portion of an email address as an output path.""" start = match.start("path") prefix = instruction[max(0, start - 80):start] - return bool(re.search(r"[A-Za-z0-9_.+-]+@$", prefix)) + return instruction[match.end("path"):].startswith("@") or bool( + re.search(r"[A-Za-z0-9_.+-]+@$", prefix) + ) def _workspace_path_identity(value: str) -> str: diff --git a/src/agent_loop.py b/src/agent_loop.py index c452c6c22..3b45c77d5 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -162,7 +162,7 @@ def _resolved_tool_call_id( logger = logging.getLogger(__name__) -_MODEL_TOOL_SURFACES = {"none", "compact", "full"} +_MODEL_TOOL_SURFACES = {"none", "compact", "full", "spec"} _ROUTE_THINKING_MODES = {"auto", "on", "off"} _NO_THINKING_COMPACT_DOMAINS = { "email", @@ -2834,7 +2834,7 @@ def _parse_qwen_explicit_admin_request(text: str) -> Optional[tuple[str, str]]: if re.search(r"\b(?:disabled\s+tools?|tools?\s+(?:are\s+)?(?:currently\s+)?disabled)\b", q): return "manage_settings", json.dumps({"action": "list_tools"}) if re.search(r"\b(?:turn|enable|re-enable|reenable)\b.{0,40}\b(?:image\s+generation|images?|generate_image)\b", q): - return "manage_settings", json.dumps({"action": "enable_tool", "tool": "images"}) + return "manage_settings", json.dumps({"action": "list_tools"}) if re.search(r"\b(?:disable|turn\s+off)\b.{0,40}\b(?:image\s+generation|images?|generate_image)\b", q): return "manage_settings", json.dumps({"action": "disable_tool", "tool": "images"}) if listish and re.search(r"\bapi\s+tokens?\b|\btokens?\b", q): @@ -8057,7 +8057,7 @@ Generate an image. Line 1 = description, line 2 = model name, line 3 = WxH (e.g. "manage_tokens": "- ```manage_tokens``` — Generate or revoke API access tokens for external integrations. Args (JSON): {\"action\": \"list|create|delete\", ...}", "manage_documents": "- ```manage_documents``` — List, read/open, delete, or tidy documents in the editor panel. Args (JSON): {\"action\": \"list|read|delete|tidy\", ...}. `list` returns rows like `[Title](#document-) — lang, size, updated 5m ago` sorted MOST-RECENT FIRST; the user clicks the anchor to open. `read` (aliases: view/open/get) takes `document_id` and returns the content. When the user asks \"open/show/read my notes\" or \"what documents do I have\", use this — do NOT shell out, do NOT curl.", "manage_research": "- ```manage_research``` — List, read/open, or delete saved DEEP RESEARCH results from the Library. Args (JSON): {\"action\": \"list|read|delete\", \"id\": \"\", \"search\": \"...\"}. `list` returns rows like `[query](#research-) — N sources` MOST-RECENT FIRST; the user clicks to open. `read` (aliases: open/view/get) takes `id` and returns the report text + sources. Use when the user says \"open/read/find/delete my research\" or \"that report\". This IS how you read a finished report: when the user refers to a just-completed deep-research job (\"check it out\", \"read that report\", \"summarize the research\") WITHOUT giving an id, call `manage_research` with `action:list` to get the most-recent id, then `action:read` with that id, and answer from the returned text. Do NOT `web_fetch`/`app_api` the `/api/research/report/{id}` URL — that endpoint renders HTML for the browser, not clean text — and do NOT start a fresh `web_search`/`trigger_research` just to read an existing report. To START new research, use trigger_research instead.", - "manage_settings": "- ```manage_settings``` — View/change the REAL app settings (same ones the Settings panel writes) AND turn tools on/off. Change a setting: `{\"action\":\"set\",\"key\":\"...\",\"value\":\"...\"}` — keys accept friendly aliases, e.g. voice→tts_voice, \"search engine\"→search_provider, \"default model\"→default_model, \"teacher model\"→teacher_model, \"task/background model\"→task_model, \"image quality\"→image_quality, \"reminder channel\"→reminder_channel (browser|email|ntfy), \"agent timeout\"/\"max tool calls\"/\"token budget\". Read: `{\"action\":\"get\",\"key\":\"...\"}`; see all: `{\"action\":\"list\"}`; reset one: `{\"action\":\"reset\",\"key\":\"...\"}`. Use this when the user asks to change ANY preference instead of making them open Settings. Secrets/API keys are read-only (tell them to set those in the panel). Tool toggles: `{\"action\":\"disable_tool|enable_tool\",\"tool\":\"shell\"}` (aliases: shell/search/browser/documents/memory/skills/images/tasks/notes/calendar/email), list disabled: `{\"action\":\"list_tools\"}`.", + "manage_settings": "- ```manage_settings``` — View/change the REAL app settings (same ones the Settings panel writes) AND turn tools on/off. Change a setting: `{\"action\":\"set\",\"key\":\"...\",\"value\":\"...\"}` — keys accept friendly aliases, e.g. voice→tts_voice, \"search engine\"→search_provider, \"default model\"→default_model, \"teacher model\"→teacher_model, \"task/background model\"→task_model, \"image quality\"→image_quality, \"reminder channel\"→reminder_channel (browser|email|ntfy), \"agent timeout\"/\"max tool calls\"/\"token budget\". Read: `{\"action\":\"get\",\"key\":\"...\"}`; see all: `{\"action\":\"list\"}`; reset one: `{\"action\":\"reset\",\"key\":\"...\"}`. Use this when the user asks to change ANY preference instead of making them open Settings. Secrets/API keys are read-only (tell them to set those in the panel). Tool toggles: `{\"action\":\"disable_tool\",\"tool\":\"shell\"}` (aliases: shell/search/browser/documents/memory/skills/images/tasks/notes/calendar/email), list disabled: `{\"action\":\"list_tools\"}`. You cannot re-enable a disabled tool: that is done by the user in Settings > Agent Tools.", "manage_notes": """\ ```manage_notes {"action": "add", "title": "", "due_date": ""} @@ -8472,7 +8472,7 @@ def _agent_route_tool_mode( return is_api_model, is_ollama_native, ollama_openai_compat -def _configured_model_tool_surface( +def _configured_model_tool_surface_raw( endpoint_url: str, model: str, owner: Optional[str] = None, @@ -8561,6 +8561,19 @@ _ADMIN_KEYWORDS = [ "note", "notes", "todo", "todos", "reminder", "reminders", ] + +def _configured_model_tool_surface( + endpoint_url: str, + model: str, + owner: Optional[str] = None, + headers: Optional[Dict] = None, + endpoint_id: Optional[str] = None, +) -> str: + """Tool surface as the harness sees it: Ajax Spec runs the compact runtime + (same dispatch, prompts, completions); only Spec hooks read the raw mode.""" + value = _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers, endpoint_id=endpoint_id) + return "compact" if value == "spec" else value + def _detect_admin_intent(messages: List[Dict]) -> bool: """Check if the last user message suggests admin/management tool usage.""" for msg in reversed(messages): @@ -13565,6 +13578,20 @@ def _ody_qwen_terminal_tool_summary(tool_event: dict[str, Any], user_text: str = """Return a deterministic user-facing answer for tools we can render safely.""" tool_name = _resolved_tool_event_name(tool_event) output = str(tool_event.get("output") or "") + if tool_name in { + "mcp__email__list_email_accounts", "mcp__email__list_emails", + "mcp__email__search_emails", "mcp__email__read_email", + } and output.lstrip().startswith("{"): + # The external email bridge wraps stdout in a JSON result. Render its + # successful read payload, never the wrapper or a failed result. + try: + email_result = json.loads(output) + except (TypeError, ValueError): + email_result = None + if isinstance(email_result, dict) and isinstance(email_result.get("stdout"), str): + if email_result.get("exit_code") not in (None, 0) or email_result.get("error"): + return "" + output = email_result["stdout"] command = str(tool_event.get("command") or "") action = "" try: @@ -15204,7 +15231,7 @@ _ADMIN_TOOLS = { "manage_documents", "manage_settings", "create_session", "list_sessions", "send_to_session", "search_chats", "pipeline", "ask_teacher", "list_models", "list_cached_models", "list_downloads", "list_cookbook_servers", - "list_serve_presets", "list_served_models", + "list_serve_presets", "list_served_models", "tail_serve_output", "search_hf_models", } def _build_base_prompt( @@ -21019,9 +21046,179 @@ def _blocks_before_inference(turn_contract) -> bool: ) +def _spec_unavailable_reply(unavailable) -> str: + blocked = set(unavailable or ()) + if blocked & (set(WEB_TOOL_NAMES) | {"private_browser", "youtube_tool"}): + return ("Web or browser access is unavailable for this chat. Enable Web search " + "in the chat controls, if available, and ask me to check again. " + "I have not opened the page or verified its contents.") + return ("The tool needed for this request is unavailable in this chat. " + "Enable the required capability, if available, and try again. " + "I have not performed the requested lookup or action.") + + from src.agent_runtime.authority import MISSING_AUTHORITY, active_request_authority, with_request_authority + +def _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy): + from src.turn_contract import _ACTIVE_CONTRACT as _spec_active_contract + disabled = set(disabled_tools or ()) + from src.spec_router import latest_decision + _spec_decision = latest_decision() or {} + _spec_choice = _spec_decision.get("choice") + _spec_direct = (os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" + and decided and _spec_choice == "answer_directly") + try: + _mgr = get_mcp_manager() + mcp_schemas = _mgr.get_all_openai_schemas(_load_mcp_disabled_map()) if _mgr else [] + except Exception: + mcp_schemas = [] + mcp_names = {(sc.get("function") or {}).get("name") for sc in mcp_schemas} + if _spec_choice == "ask_user": + # ask_user = a tool is needed but a detail may be missing: offer Spec's top tool too, so the model + # can act when the detail is there or ask when it is not. + _top = _spec_decision.get("top") or [] + if _top: + picks = set(picks) | {_top[0][0]} + picks = {(f"mcp__email__{n}" if f"mcp__email__{n}" in mcp_names else n) for n in picks} + strict = os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" + base_offered = set(turn_contract.offered) + # The router sees the full catalog, while this request's policy may + # deny a tool (for example, a browser turn with Web access switched + # off). Preserve that conflict as an unavailable contract. Keeping the + # old empty menu instead lets the model invent a tool result in prose. + _top = _spec_decision.get("top") or [] + _primary = str(_top[0][0]) if _spec_choice and str(_spec_choice).startswith("t") and _top else "" + if _primary and f"mcp__email__{_primary}" in mcp_names: + _primary = f"mcp__email__{_primary}" + # A policy denial is surfaced even if the router chose outside the + # base inventory; it never grants access to that tool. + def _denied(name): + return (name in disabled + or (tool_policy is not None and ( + tool_policy.blocks(name) + or (tool_policy.disable_mcp and name.startswith("mcp__"))))) + _unavailable_pick = "" + if _primary in picks and _denied(_primary): + _unavailable_pick = _primary + elif picks and all(_denied(name) for name in picks): + _unavailable_pick = sorted(picks)[0] + if _unavailable_pick: + blocked = replace( + turn_contract, required=frozenset(), offered=frozenset(), + unavailable=frozenset({_unavailable_pick}), schema_json=(), + required_read_operation=None, routing_experiment="spec-policy-denied", + ) + _spec_active_contract.set(blocked) + logger.info("[spec-harness] router pick %s unavailable under request policy", _unavailable_pick) + return blocked + want = set(picks) - disabled + # Always-on basics only when the base turn offers them, so the Spec + # arm never offers a tool the base arm would not. + if not _spec_direct: + want |= {"ask_user", "update_plan"} & base_offered + if not (strict and decided): + want |= base_offered + want &= base_offered + if not want: + logger.info("[spec-harness] router picks %s outside base offered %s; base contract kept", + sorted(picks), sorted(base_offered)) + return turn_contract + offer = want & base_offered & set(turn_contract.executable) + # A router may narrow the menu only when every existing obligation + # remains satisfiable. It cannot redefine the user's requested work. + required_read = getattr(turn_contract, "required_read_operation", None) + if (not set(turn_contract.required) <= offer or turn_contract.unavailable + or (required_read is not None and required_read.tool not in offer)): + return turn_contract + by_name = {} + for sc in list(FUNCTION_TOOL_SCHEMAS) + list(mcp_schemas): + nm = (sc.get("function") or {}).get("name") + if nm in offer and nm not in by_name: + by_name[nm] = sc + for sc in turn_contract.schemas(): + nm = (sc.get("function") or {}).get("name") + if nm in offer: + by_name[nm] = sc + if not by_name and not _spec_direct: + logger.info("[spec-harness] router picks %s not executable this turn; contract kept", sorted(picks)) + return turn_contract + # Required tools and the required read carry over unchanged: harness + # completion logic keys on them (e.g. a calendar list is a read, not an + # effect that may end with "Done."). If one lost its schema, contract + # validation fails and the original contract is retained below. + new = replace( + turn_contract, offered=frozenset(by_name), + schema_json=tuple(json.dumps(by_name[n], ensure_ascii=False, sort_keys=True) for n in sorted(by_name)), + routing_experiment="spec-router") + _spec_active_contract.set(new) + logger.info("[spec-harness] contract replaced by router: offered=%s (was %s, decided=%s)", + sorted(by_name), sorted(turn_contract.offered), decided) + return new + + +def _spec_router_contract(turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=None): + """Ajax Spec: return a copy of the turn contract whose offered tools are the + router's picks (plus ask_user/update_plan), or the contract unchanged. + + Runs before any runtime dispatch so the compact preview runtime and the + main loop both serve the router's set; only the offered tools change. + """ + # Contexts can be inherited by a child task; a turn must never inherit a + # previous turn's decision when routing is disabled or unavailable. + from src import spec_router as _router_context + if hasattr(_router_context, "clear_decision"): + _router_context.clear_decision() + if (turn_contract is None + or getattr(turn_contract, "routing_experiment", "") in {"spec-router", "spec-policy-denied"} + or (tool_policy is not None and tool_policy.block_all_tool_calls)): + return turn_contract + if os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() not in {"strict", "additive"}: + return turn_contract + try: + if _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) != "spec": + return turn_contract + except Exception: + return turn_contract + try: + from src.spec_router import spec_route_tools as _spec_route_tools_early + picks, decided = _spec_route_tools_early( + _extract_last_user_message(messages), FUNCTION_TOOL_SCHEMAS, set(), messages, return_decision=True) + return _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy) + except Exception as exc: + logger.warning("[spec-harness] router failed; retaining original contract: %s", exc) + return turn_contract + + +async def _spec_router_contract_async(turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=None): + """Asynchronous non-blocking turn contract routing.""" + from src import spec_router as _router_context + if hasattr(_router_context, "clear_decision"): + _router_context.clear_decision() + if (turn_contract is None + or getattr(turn_contract, "routing_experiment", "") in {"spec-router", "spec-policy-denied"} + or (tool_policy is not None and tool_policy.block_all_tool_calls)): + return turn_contract + if os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() not in {"strict", "additive"}: + return turn_contract + try: + if _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) != "spec": + return turn_contract + except Exception: + return turn_contract + try: + from src.spec_router import spec_route_tools_async as _spec_route_tools_early_async + picks, decided = await _spec_route_tools_early_async( + _extract_last_user_message(messages), FUNCTION_TOOL_SCHEMAS, set(), messages, return_decision=True) + return _apply_spec_decision_to_contract(turn_contract, picks, decided, disabled_tools, tool_policy) + except Exception as exc: + logger.warning("[spec-harness] router failed; retaining original contract: %s", exc) + return turn_contract + + @with_request_authority @with_turn_contract @with_teacher_takeover @@ -21099,6 +21296,41 @@ async def stream_agent_loop( set(tool_policy.all_disabled_names()) if tool_policy else set() )) + # Ajax Spec driver: the Spec router picks every step (src/spec_driver.py). + if (os.getenv("ODYSSEUS_SPEC_MODE", "").strip().lower() == "driver" + and not (tool_policy and tool_policy.block_all_tool_calls) + and _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) == "spec"): + yield 'data: ' + json.dumps({"delta": "Spec driver mode is unavailable because its authority and completion checks are not supported. No action was performed."}) + '\n\n' + yield 'data: [DONE]\n\n' + return + + turn_contract = await _spec_router_contract_async( + turn_contract, messages, disabled_tools, endpoint_url, model, owner, headers, + tool_policy=tool_policy) + if (turn_contract is not None + and getattr(turn_contract, "routing_experiment", "") == "spec-policy-denied"): + # A selected but disallowed tool is a policy outcome, not an invitation + # to generate an ungrounded answer with an empty tool menu. + reply = _spec_unavailable_reply(turn_contract.unavailable) + yield 'data: ' + json.dumps({"type": "turn_contract", **turn_contract.audit()}) + '\n\n' + yield 'data: ' + json.dumps({"delta": reply}) + '\n\n' + yield 'data: [DONE]\n\n' + return + + try: + _spec_turn = _configured_model_tool_surface_raw(endpoint_url, model, owner, headers=headers) == "spec" + except Exception: + _spec_turn = False + # Without a turn contract (scheduled, skill and monitor runs) no Spec + # routing applies; the turn runs on the compact surface as usual. + if _spec_turn and _blocks_before_inference(turn_contract): + unavailable = ", ".join(sorted(turn_contract.unavailable)) + yield 'data: ' + json.dumps({"type": "turn_contract", **turn_contract.audit()}) + '\n\n' + yield 'data: ' + json.dumps({"delta": "I can’t perform this request with the currently permitted tools " + f"(unavailable: {unavailable}). I haven’t substituted another tool."}) + '\n\n' + yield 'data: [DONE]\n\n' + return + if is_compact_preview_contract(turn_contract): from src.clean_agent_preview import stream_preview async for chunk in stream_preview( @@ -24126,6 +24358,15 @@ async def stream_agent_loop( headers=candidate_headers, endpoint_id=(route_descriptor or {}).get("endpoint_id"), ) + # "spec" = compact tools plus a small-model router; the harness-facing + # surface already reads "compact", so detect spec from the raw mode. + try: + _spec_surface = getattr(turn_contract, "routing_experiment", "") == "spec-router" and _configured_model_tool_surface_raw( + candidate_url, candidate_model, owner, headers=candidate_headers, + endpoint_id=(route_descriptor or {}).get("endpoint_id")) == "spec" + except Exception: + _spec_surface = False + _spec_offered = None textual_tools = ( force_textual_tool_transport or force_textual_tools @@ -24159,6 +24400,57 @@ async def stream_agent_loop( _last_user, _intent_domains, ) + if _spec_surface: + from src.spec_router import spec_route_tools as _spec_route_tools + + if turn_contract is not None and getattr(turn_contract, "routing_experiment", "") == "spec-router": + _spec_added, _spec_decided = set(turn_contract.offered) - {"ask_user", "update_plan"}, True + else: + _spec_added, _spec_decided = _spec_route_tools( + _last_user, + FUNCTION_TOOL_SCHEMAS, + disabled_tools, + compacted_source, + return_decision=True, + ) + _spec_added = _spec_added - set(disabled_tools) + # The router catalog uses the legacy native email names; the + # live tools are the MCP ones, and the legacy alias schemas are + # dropped when the MCP server is connected. Map picks over. + try: + from src.tool_utils import get_mcp_manager as _spec_mcp_mgr + _mgr = _spec_mcp_mgr() + _mcp_names = { + (schema.get("function") or {}).get("name") + for schema in (_mgr.get_all_openai_schemas(_load_mcp_disabled_map()) if _mgr else []) + } + except Exception: + _mcp_names = set() + _spec_added = { + (f"mcp__email__{name}" if f"mcp__email__{name}" in _mcp_names else name) + for name in _spec_added + } + _spec_mode = os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() + _compact_before = set(route_tools) + if _spec_mode == "strict" and _spec_decided: + # The router decides which tools are injected: its pick(s), + # tools already used in this chat, and the always-on basics. + _spec_strict = (_spec_added | {"ask_user", "update_plan"}) & set(turn_contract.offered) - set(disabled_tools) + if _spec_strict: + route_tools = _spec_strict + _spec_offered = frozenset(route_tools) + else: + logger.info("[spec-harness] strict set empty after disabled-tool filter; keeping compact set") + route_tools = set(route_tools) | _spec_added + else: + route_tools = set(route_tools) | _spec_added + logger.info("[spec-debug] disabled_tools (%d): %s | policy block_all=%s disable_mcp=%s", + len(set(disabled_tools)), sorted(set(disabled_tools))[:80], + getattr(tool_policy, "block_all_tool_calls", None), getattr(tool_policy, "disable_mcp", None)) + logger.info( + "[spec-harness] mode=%s decided=%s compact=%s final=%s", + _spec_mode, _spec_decided, sorted(_compact_before), sorted(route_tools), + ) # The compact router only receives user text, while native task # inputs may be declared out-of-band by the runner. Reapply that # concrete input contract after compaction so an implicit video @@ -24199,7 +24491,7 @@ async def stream_agent_loop( _irrelevant_local_media_web_tools.add("private_browser") route_tools.difference_update(_irrelevant_local_media_web_tools) if turn_contract is not None and tool_surface != "full": - route_tools = set(turn_contract.offered) + route_tools = set(_spec_offered) if _spec_offered else set(turn_contract.offered) if prompt_compact: route_tools.update( _blocked_network_recovery_tools(tool_events) @@ -24299,7 +24591,7 @@ async def stream_agent_loop( disabled_tools, needs_admin=_needs_admin, relevant_tools=prompt_route_tools, - preserve_conversation=turn_contract is not None, + preserve_conversation=(turn_contract is not None) or _spec_surface, mcp_disabled_map=_mcp_disabled_map, compact=prompt_compact, owner=owner, @@ -24511,6 +24803,13 @@ async def stream_agent_loop( _prepend_agent_directive(route_messages, build_active_plan_note(approved_plan)) if guide_only: _prepend_agent_directive(route_messages, GUIDE_ONLY_DIRECTIVE) + if _spec_offered and os.getenv("ODYSSEUS_SPEC_TRIM_PROMPT", "1") != "0": + from src.spec_router import trim_system_prompt as _spec_trim + + route_messages = _spec_trim(route_messages, _spec_offered) + from src.spec_router import drop_skills_index as _spec_drop_skills + + route_messages = _spec_drop_skills(route_messages, _spec_offered) return { "messages": route_messages, "mcp_schemas": route_mcp_schemas, @@ -24519,6 +24818,8 @@ async def stream_agent_loop( "is_ollama_native": is_native_ollama, "ollama_openai_compat": is_ollama_compat, "tool_surface": tool_surface, + "spec_offered": _spec_offered, + "spec_surface": _spec_surface, "ody_qwen_finetune_model": is_ody, "qwen38_tool_router": _is_qwen38_tool_router(candidate_model) and not _full_inventory_mode, "ody_doc_finetune_mode": doc_mode, @@ -25076,6 +25377,15 @@ async def stream_agent_loop( if (schema.get("function", {}).get("name") or schema.get("name")) not in disabled_tools ] + _spec_names = route_state.get("spec_offered") + if _spec_names: + # Ajax Spec: the router, not the turn contract, decides which + # tools are injected for this turn. + contract_schemas = [ + schema for schema in contract_schemas + if (schema.get("function", {}).get("name") or schema.get("name")) + in _spec_names + ] return _filter_route_tool_schemas(_apply_tool_surface_to_schemas(contract_schemas, tool_surface)) if route_state["is_api_model"]: if tool_surface == "full": @@ -25137,15 +25447,18 @@ async def stream_agent_loop( and schema.get("name") not in disabled_tools ] if _pure_web_turn and tool_surface != "full": - allowed = _web_only_route_tools(_last_user, disabled_tools) - schemas = [ - schema for schema in schemas - if ( - schema.get("function", {}).get("name") - or schema.get("name") - ) in allowed - or schema.get("function", {}).get("name") in external_names - ] + strict_spec = (route_state.get("spec_surface") and + os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict") + if not strict_spec: + allowed = _web_only_route_tools(_last_user, disabled_tools) + schemas = [ + schema for schema in schemas + if ( + schema.get("function", {}).get("name") + or schema.get("name") + ) in allowed + or schema.get("function", {}).get("name") in external_names + ] schemas = _drop_legacy_email_alias_schemas_when_mcp_available(schemas) schemas = _apply_tool_surface_to_schemas(schemas, tool_surface) if ( @@ -25997,6 +26310,8 @@ async def stream_agent_loop( "is_ollama_native": _is_ollama_native, "ollama_openai_compat": _ollama_openai_compat, "tool_surface": _route_state.get("tool_surface", ""), + "spec_offered": _route_state.get("spec_offered"), + "spec_surface": _route_state.get("spec_surface"), "ody_qwen_finetune_model": _ody_qwen_finetune_model, "qwen38_tool_router": _qwen38_tool_router, "ody_doc_finetune_mode": _ody_doc_finetune_mode, @@ -26012,10 +26327,19 @@ async def stream_agent_loop( _active_route_state["request_messages"] = _initial_route_request_messages all_tool_schemas = _tool_schemas_for_route(_active_route_state) if turn_contract is not None: - _relevant_tools = set(turn_contract.offered) + _relevant_tools = ( + set(_route_state.get("spec_offered")) + if _route_state.get("spec_offered") + else set(turn_contract.offered) + ) agent_stream_timeout = int(get_setting("agent_stream_timeout_seconds", 300) or 300) _tool_names_sent = [t.get("function", {}).get("name") for t in (all_tool_schemas or []) if t.get("function")] + if round_num == 1: + try: + logger.info("[spec-debug] prompt composition (chars): %s", [(m.get("role"), len(str(m.get("content") or ""))) for m in messages]) + except Exception: + pass logger.info(f"[agent-debug] round={round_num} model={model} _is_api_model={_is_api_model} tools_sent={len(_tool_names_sent)} tool_names={_tool_names_sent} relevant_tools={sorted(_relevant_tools)[:50] if _relevant_tools else 'ALL'}") _routing_intentional_exclusions = set(disabled_tools) if ( @@ -27605,7 +27929,11 @@ async def stream_agent_loop( ), active_document=active_document, last_user=_last_user, - offered_tool_names=(set(turn_contract.offered) if turn_contract is not None else set(_tool_names_sent)), + offered_tool_names=( + set(turn_contract.offered) + if turn_contract is not None and not _active_route_state.get("spec_offered") + else set(_tool_names_sent) + ), recover_unoffered_tool_names=_recover_unoffered_local_tools, passthrough_tool_names={ schema["function"]["name"] @@ -32169,6 +32497,29 @@ async def stream_agent_loop( budget_hit = False local_network_budget_hit = False local_inspection_budget_hit = False + if _active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and tool_blocks and not used_native and _is_api_model: + # Ajax Spec with a native-API model: a round without a native call + # carries only harness-synthesized recovery blocks. Drop them all. + logger.info("[spec-harness] dropped %d synthesized tool block(s) (no native call): %s", + len(tool_blocks), [b.tool_type for b in tool_blocks]) + tool_blocks = []; converted_calls = [] + if ( + _active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and used_native and tool_blocks + and len(converted_calls) == len(tool_blocks) + ): + # Ajax Spec: only the model's own native calls run; harness- + # synthesized recovery blocks (no native call) are dropped. When + # converted_calls was reset (singleton collapse) keep every block. + _spec_keep = [ + i for i in range(len(tool_blocks)) + if i < len(converted_calls) and converted_calls[i] is not None + ] + if len(_spec_keep) != len(tool_blocks): + logger.info("[spec-harness] dropped %d synthesized tool block(s): %s", + len(tool_blocks) - len(_spec_keep), + [tool_blocks[i].tool_type for i in range(len(tool_blocks)) if i not in _spec_keep]) + tool_blocks = [tool_blocks[i] for i in _spec_keep] + converted_calls = [converted_calls[i] for i in _spec_keep] for i, block in enumerate(tool_blocks): native_call = converted_calls[i] if i < len(converted_calls) else None tool_call_id = _resolved_tool_call_id( @@ -33463,7 +33814,8 @@ async def stream_agent_loop( ), } if ( - (_qwen38_tool_router or _full_inventory_mode) and (_pure_web_turn or _contextual_public_web_followup) + not (_active_route_state.get("spec_offered") and os.getenv("ODYSSEUS_SPEC_MODE", "strict").strip().lower() == "strict" and block.tool_type == "private_browser") + and (_qwen38_tool_router or _full_inventory_mode) and (_pure_web_turn or _contextual_public_web_followup) and not _artifact_creation_requested and (block.tool_type == "web_search" or _web_execution_budget.searches) and not _web_execution_budget.admit( diff --git a/src/agent_runtime/authority.py b/src/agent_runtime/authority.py index cd5197300..541fac52f 100644 --- a/src/agent_runtime/authority.py +++ b/src/agent_runtime/authority.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, replace from functools import wraps from inspect import signature import json +import os from pathlib import Path import re from uuid import uuid4 @@ -140,11 +141,20 @@ class RequestAuthority: raise ValueError("Malformed request authority") if self.resource_roots is None: roots = () - if self.workspace: - try: - roots = (FilesystemRoot.seal(self.workspace, owner=self.owner),) - except (OSError, ValueError, RuntimeError): - pass # An unresolved workspace grants no filesystem root. + # With no workspace, bash/python run in the agent workspace + # (agent_cwd), so the file tools get that same root (#6679). + # Imported here: authority -> execution is a new edge, and + # tool_execution imports this module at load time. + from src import tool_execution + try: + root = self.workspace + if not root: + # Vetted like agent_cwd(): a symlinked agent workspace + # raises and so grants no root. + root = tool_execution.default_agent_workspace() + roots = (FilesystemRoot.seal(root, owner=self.owner),) + except (OSError, ValueError, RuntimeError): + pass # An unresolved workspace grants no filesystem root. object.__setattr__(self, "resource_roots", roots) if (not isinstance(self.resource_roots, tuple) or any(not isinstance(r, FilesystemRoot) or (r.owner and r.owner != self.owner) diff --git a/src/agent_runtime/effect_adapters.py b/src/agent_runtime/effect_adapters.py index 0e1defd2b..34c3772cb 100644 --- a/src/agent_runtime/effect_adapters.py +++ b/src/agent_runtime/effect_adapters.py @@ -230,6 +230,18 @@ def classify(capture: DispatchCapture) -> dict[str, Any] | None: if capture.backend is not None: from src.agent_runtime.resources import ExternalResource if isinstance(capture.backend.resource, ExternalResource): + # A read through an external backend is still a read. The tool + # capability registry is sealed by the dispatcher, and unknown or + # mixed-effect calls continue to receive an external effect claim. + if all(bound is None for bound in (capture.browser, capture.filesystem, + capture.process, capture.owned)): + from src.tool_capabilities import ToolEffect, capabilities_for_action + capabilities = capabilities_for_action(capture.backend.transport_tool, + capture.backend.exact_input) + reads = frozenset({ToolEffect.READ_PUBLIC, ToolEffect.READ_WORKSPACE, + ToolEffect.READ_PRIVATE, ToolEffect.BROKERED_NETWORK_READ}) + if capabilities.known and capabilities.effects and capabilities.effects <= reads: + return None external = True impact = (*impact, resource_ref(capture.backend.resource, "backend")) return {"impact_scope": impact, "dependencies": dependencies, "obligations": obligations, "external": external} diff --git a/src/agent_tools/__init__.py b/src/agent_tools/__init__.py index 7db8cffbb..7689f6ead 100644 --- a/src/agent_tools/__init__.py +++ b/src/agent_tools/__init__.py @@ -29,6 +29,7 @@ from .interaction_tools import AskUserTool, UpdatePlanTool from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool from .bg_job_tools import ManageBgJobsTool from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool +from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool from .admin_tools import ( ADMIN_TOOL_HANDLERS, do_manage_endpoints, do_manage_mcp, do_manage_webhooks, @@ -72,6 +73,10 @@ TOOL_HANDLERS = { "list_sessions": ListSessionsTool().execute, "send_to_session": SendToSessionTool().execute, "manage_session": ManageSessionTool().execute, + "plan_edit": PlanEditTool().execute, + "run_editor_command": RunEditorCommandTool().execute, + "get_editor_screenshot": GetEditorScreenshotTool().execute, + "ai_edit_image": AiEditImageTool().execute, } # Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings). TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS) diff --git a/src/agent_tools/admin_tools.py b/src/agent_tools/admin_tools.py index 6b1349cff..419b7f742 100644 --- a/src/agent_tools/admin_tools.py +++ b/src/agent_tools/admin_tools.py @@ -38,11 +38,25 @@ async def do_manage_endpoints(content: str, owner: Optional[str] = None) -> Dict elif action == "add": import uuid as _uuid + from src.url_safety import check_outbound_url name = args.get("name", "") base_url = args.get("base_url", "") api_key = args.get("api_key", "") if not base_url: return {"error": "base_url is required", "exit_code": 1} + # SSRF hardening: validate the URL before storing it. Every entry + # point that accepts an external base_url must run this guard. + # The HTTP route (POST /api/model-endpoints) gets the same guard + # in #6316; this tool bypasses that route entirely, so it needs + # its own validation. A prompt-injection payload smuggled into a + # skill, note, fetched page, or email could instruct the model to + # register a cloud-metadata or internal-network endpoint. + ok, reason = check_outbound_url( + base_url, + block_private=os.getenv("MODELENDPOINT_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + return {"error": f"Rejected endpoint URL: {reason}", "exit_code": 1} eid = str(_uuid.uuid4())[:8] from datetime import datetime ep = ModelEndpoint(id=eid, name=name or base_url, base_url=base_url, @@ -75,7 +89,7 @@ async def do_manage_endpoints(content: str, owner: Optional[str] = None) -> Dict return {"error": f"Unknown action: {action}", "exit_code": 1} except Exception as e: logger.error(f"manage_endpoints error: {e}") - return {"error": str(e), "exit_code": 1} + return {"error": "An internal error occurred while managing endpoints", "exit_code": 1} finally: db.close() @@ -571,7 +585,9 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: return _ALIASES_SET.get(k2, (k or "").strip()) def _is_managed_key(key): - return key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS + # Tool policy is not a general setting, even if a future default + # is added: resetting it must never re-enable disabled tools. + return key != "disabled_tools" and key in DEFAULT_SETTINGS and key not in RETIRED_SETTING_KEYS _ENUMS = { "image_quality": ["low", "medium", "high"], @@ -699,13 +715,35 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: save_settings(s) return {"response": f"Reset {key} to default ({DEFAULT_SETTINGS[key]}).", "exit_code": 0} - elif action in ("disable_tool", "enable_tool", "list_tools"): + elif action == "enable_tool": + # The global disabled_tools denylist is the one hard permission + # boundary the Settings -> Agent Tools panel offers admins. It + # must only ever be *narrowed* from inside an agent turn: a + # model that can call enable_tool can undo the boundary itself, + # and any web page, email, or document the agent reads can ask + # it to (prompt injection re-enabling bash/email/etc.). Widening + # the tool set stays a human action in the Settings panel. + tool_name = (args.get("tool") or args.get("name") or "").strip().lower() + return { + "error": ( + f"Re-enabling tools is not available from chat" + f"{f' (requested: {tool_name})' if tool_name else ''}. " + "Disabled tools are an admin policy: ask the user to turn the tool " + "back on in Settings > Agent Tools. You can still disable tools " + "(disable_tool) or list the current state (list_tools)." + ), + "exit_code": 1, + } + + elif action in ("disable_tool", "list_tools"): # Tool-toggle actions. These edit settings.json:disabled_tools # (the global list read on every chat request) rather than # prefs.json. Friendly aliases accepted: "shell" -> "bash", # "search" -> "web_search", "browser" -> browser tools, # "documents" -> the document tool set, "memory" -> - # manage_memory, etc. + # manage_memory, etc. Only disable_tool mutates: see the + # enable_tool branch above for why the agent can never widen + # the denylist. from src.settings import get_setting, save_settings, load_settings _ALIASES = { "shell": ["bash"], @@ -739,7 +777,8 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: "response": ( f"Currently disabled: {', '.join(current) if current else '(none)'}.\n" "Common toggles: shell (bash), search (web_search), browser, documents, " - "memory, skills, images, tasks, notes, calendar, email." + "memory, skills, images, tasks, notes, calendar, email. " + "To re-enable a tool, open Settings > Agent Tools." ), "disabled": list(current), "exit_code": 0, @@ -753,22 +792,19 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict: settings = load_settings() current = list(settings.get("disabled_tools") or []) before = set(current) - if action == "disable_tool": - for t in targets: - if t not in current: - current.append(t) - else: # enable_tool - current = [t for t in current if t not in targets] + for t in targets: + if t not in current: + current.append(t) after = set(current) settings["disabled_tools"] = current save_settings(settings) - verb = "Disabled" if action == "disable_tool" else "Enabled" changed = sorted(after.symmetric_difference(before)) return { "response": ( - f"{verb} {tool_name} ({', '.join(targets)}). " - f"Now disabled: {', '.join(current) if current else '(none)'}." + f"Disabled {tool_name} ({', '.join(targets)}). " + f"Now disabled: {', '.join(current) if current else '(none)'}. " + "Re-enabling is done by the user in Settings > Agent Tools." ), "changed": changed, "disabled": list(current), diff --git a/src/agent_tools/editor_tools.py b/src/agent_tools/editor_tools.py new file mode 100644 index 000000000..aec99a56a --- /dev/null +++ b/src/agent_tools/editor_tools.py @@ -0,0 +1,697 @@ +"""In-process agent tools for the scoped ``photo_editor`` capability family. + +These handlers run as the logged-in owner and drive the image editor over the +owner-scoped in-process bridge (``/api/editor/bridge/*``) plus the app's +existing image transforms (``/api/image/*``). They are only offered to the +model when the ``photo_editor`` family is active (the Quick Edit popup or an +explicit photo-edit chat turn), never in the general tool list. + +Loopback calls carry the internal token and impersonate ``owner`` (middleware +app.py), so the editor bridge resolves the same ``owner:session`` scope the +browser subscriber uses. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import json +import logging +import re +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx + +from src.tools._common import _INTERNAL_BASE, _internal_headers + +logger = logging.getLogger(__name__) + + +def _provider_message() -> str: + try: + from src.ai_interaction import IMAGE_PROVIDER_REQUIRED_MESSAGE + + return IMAGE_PROVIDER_REQUIRED_MESSAGE + except Exception: + return ( + "Removing/adding objects needs an image generation provider; enable one " + "in Settings > Models." + ) + + +def _provider_available(owner: Optional[str]) -> bool: + try: + from src.ai_interaction import image_editing_provider_available + + return bool(image_editing_provider_available(owner)) + except Exception: + return True + +# Local diffusion/image servers can take 90-110s per image (8 steps) and +# longer on a cold start; the previous 120s default timed out mid-generation. +_EDITOR_TIMEOUT = 300.0 +# A mask covering less than this fraction of the image cannot drive a useful +# regional inpaint (tiny figures, empty SAM output): fall back to grounding or +# whole-image instruction mode instead. +_MIN_MASK_FRACTION = 0.002 +_AI_OPS = { + "remove-bg": "/api/image/remove-bg", + "upscale": "/api/image/upscale-local", + "denoise": "/api/image/denoise", + "enhance-face": "/api/image/enhance-face", + "inpaint": "/api/image/inpaint", + "generative": "/api/image/inpaint", +} +_PLACE_OP = "layers.placeImage" + + +def _parse(content: Any) -> Dict[str, Any]: + raw = content if isinstance(content, str) else json.dumps(content or {}) + try: + parsed = json.loads(raw) if raw.strip() else {} + except (ValueError, TypeError): + parsed = {} + return parsed if isinstance(parsed, dict) else {} + + +def _session(args: Dict[str, Any], ctx: Dict[str, Any]) -> str: + # Explicit editor session bound in context is authoritative; + # otherwise fallback to caller/tool args, then default. + # Note: ctx.get("session_id") is the chat/turn UUID, NOT the editor canvas session. + return str(ctx.get("editor_session") or args.get("session") or args.get("editor_session") or "default") + + +def _incognito(ctx: Dict[str, Any]) -> bool: + runtime = ctx.get("client_runtime_context") + state = runtime.get("web_ui_state") if isinstance(runtime, dict) else None + return ctx.get("incognito") is True or (isinstance(state, dict) and state.get("incognito") is True) + + +async def _post(path: str, payload: Dict[str, Any], owner: Optional[str], timeout: float = _EDITOR_TIMEOUT) -> Dict[str, Any]: + headers = {**_internal_headers(owner), "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(f"{_INTERNAL_BASE}{path}", json=payload, headers=headers) + resp.raise_for_status() + try: + return resp.json() + except ValueError: + return {"raw": resp.text} + + +def _as_data_url(value: Any) -> Optional[str]: + if not isinstance(value, str) or not value: + return None + if value.startswith("data:"): + return value + if value.startswith("/"): + return None + return f"data:image/png;base64,{value}" + + +def _data_url_bytes(data_url: str) -> bytes: + _, _, b64 = data_url.partition(",") + return base64.b64decode(b64) + + +def _image_reference_path(value: str) -> bool: + # Never use the internal tool credential to fetch an arbitrary app route. + generated = re.fullmatch(r"/api/generated-image/[a-f0-9]{8,64}\.(?:png|jpg|jpeg|webp|gif)", value) + upload = re.fullmatch(r"/api/upload/[a-fA-F0-9]{32}(?:\.[A-Za-z0-9]+)?", value) + return bool(generated or upload) + + +def _persist_png(image_bytes: bytes, owner: Optional[str], session_id: Optional[str], prompt: str, model: str) -> Dict[str, Any]: + """Save an edited/generated PNG and return {image_id, image_url, image_size}.""" + from core.database import GalleryImage, SessionLocal + from src.constants import GENERATED_IMAGES_DIR + + db = SessionLocal() + image_path = None + try: + root = Path(GENERATED_IMAGES_DIR).resolve() + root.mkdir(parents=True, exist_ok=True) + filename = f"{uuid.uuid4().hex[:12]}.png" + image_path = root / filename + image_path.write_bytes(image_bytes) + new_id = str(uuid.uuid4()) + width = height = 0 + try: + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as img: + width, height = img.size + except Exception: + pass + size = f"{width}x{height}" if width and height else "" + # The editor bridge uses an owner-scoped session id (e.g. "pewds:3") + # that is not a chat-sessions row. GalleryImage.session_id is a FK to + # sessions.id, so drop an unknown id instead of raising IntegrityError. + if session_id: + from core.database import Session as _ChatSession + + try: + known = db.query(_ChatSession.id).filter(_ChatSession.id == session_id).first() + except Exception: + known = None + if known is None: + session_id = None + derived = GalleryImage( + id=new_id, + filename=filename, + prompt=prompt or None, + caption=None, + model=model, + size=size or None, + quality=None, + tags=None, + ai_tags=None, + session_id=session_id, + album_id=None, + owner=owner, + file_hash=hashlib.sha256(image_bytes).hexdigest(), + file_size=len(image_bytes), + width=width or None, + height=height or None, + ) + db.add(derived) + db.commit() + return {"image_id": new_id, "image_url": f"/api/generated-image/{filename}", "image_size": size or None} + except Exception: + db.rollback() + if image_path is not None: + try: + image_path.unlink(missing_ok=True) + except OSError: + logger.warning("Failed to remove uncommitted editor image", exc_info=True) + raise + finally: + db.close() + + +async def _place_layer(data_url: str, name: str, owner: Optional[str], session: str, *, masked: bool = False, blend: str = "normal", opacity: float = 100.0, mode: Optional[str] = None) -> bool: + params: Dict[str, Any] = { + "dataUrl": data_url, + "name": name, + "masked": bool(masked), + "blend": blend or "normal", + "opacity": opacity, + } + # Forward the edit mode so the client can build a difference mask for + # instruction-mode results: the whole image was regenerated under an + # instruction prompt, so untouched pixels must stay pixel-identical (the + # client keeps only the changed region) instead of the new layer overwriting + # the entire canvas. Omit the keys entirely when no mode is given so the + # client keeps its current behavior (e.g. generate_image must not diff-mask). + if mode: + params["mode"] = mode + params["differenceMask"] = mode == "instruction" + if mode == "instruction": + # Exclude a thin edge band from the client difference mask: full-frame + # regenerations can shift pixels along the border. + params["edgeExcludePx"] = 3 + payload = { + "session": session, + "op": _PLACE_OP, + "params": params, + "wait": True, + "timeout": 60, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=90) + return bool(data.get("ok") is True and data.get("completed", True) is not False) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor layer placement failed: %s", exc) + return False + + +async def _fetch_selection_mask(owner: Optional[str], session: str) -> Optional[str]: + """Return the editor's current selection as a data-URL mask, or None. + + Generative ops (inpaint/generative) need a mask; the planner/judge may not + have supplied one, so fall back to whatever is selected in the editor. + """ + payload = { + "session": session, + "op": "selection.exportMask", + "params": {}, + "wait": True, + "timeout": 30, + } + try: + data = await _post("/api/editor/bridge/command", payload, owner, timeout=45) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor selection mask fetch failed: %s", exc) + return None + result = data.get("result") if isinstance(data, dict) else None + if isinstance(result, dict): + return _as_data_url(result.get("mask")) + return None + + +def _mask_white_fraction(data_url: Optional[str]) -> float: + """Fraction of a grayscale mask that is selected (white, >=128).""" + if not data_url: + return 0.0 + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))).convert("L") as im: + total = im.width * im.height + if not total: + return 0.0 + hist = im.histogram() + return sum(hist[128:]) / total + except Exception: + return 0.0 + + +def _image_is_blank(data_url: str) -> bool: + """True when a result has an alpha channel that is essentially transparent.""" + try: + from PIL import Image as _PILImage + + with _PILImage.open(io.BytesIO(_data_url_bytes(data_url))) as im: + if im.mode not in ("RGBA", "LA") and "transparency" not in im.info: + return False + alpha = im.convert("RGBA").getchannel("A") + hist = alpha.histogram() + total = sum(hist) + return total > 0 and (sum(hist[1:]) / total) < 0.001 + except Exception: + return False + + +def _rect_mask_data_url(image_data_url: str, box: List[float], *, pad_frac: float = 0.15) -> Optional[str]: + """Build a rectangular mask data URL from a [x1,y1,x2,y2] box, dilated. + + Used when the grounding detector finds an object but SAM returns an empty + mask (tiny figures): a dilated box guarantees a usable region. + """ + try: + from PIL import Image as _PILImage, ImageDraw + + with _PILImage.open(io.BytesIO(_data_url_bytes(image_data_url))) as im: + w, h = im.size + x1, y1, x2, y2 = (float(v) for v in box) + bw, bh = x2 - x1, y2 - y1 + x1, y1 = max(0.0, x1 - bw * pad_frac), max(0.0, y1 - bh * pad_frac) + x2, y2 = min(float(w), x2 + bw * pad_frac), min(float(h), y2 + bh * pad_frac) + mask = _PILImage.new("L", (w, h), 0) + ImageDraw.Draw(mask).rectangle([x1, y1, x2, y2], fill=255) + buf = io.BytesIO() + mask.save(buf, "PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + except Exception: + return None + + +async def _ground_box(owner: Optional[str], image_data_url: str, label: str) -> Optional[List[float]]: + """Use the in-app grounding detector (OWL-ViT via /api/image/mask) for a box.""" + try: + data = await _post("/api/image/mask", {"image": image_data_url, "text": label}, owner, timeout=90) + except Exception as exc: # pragma: no cover - defensive + logger.warning("editor grounding box failed: %s", exc) + return None + box = data.get("bbox") if isinstance(data, dict) else None + if isinstance(box, (list, tuple)) and len(box) == 4: + try: + return [float(v) for v in box] + except Exception: + return None + return None + + +def _ensure_keep_clause(text: str) -> str: + """Append a keep-the-subject clause when the prompt lacks one.""" + body = str(text or "").strip() + if not body: + return body + if re.search(r"\b(keep|preserve|leave|unchanged|identical|same)\b", body, re.I): + return body + return body.rstrip(". ") + ". Keep the main subject and everything else in the image exactly the same." + + +def _mask_label(args: Dict[str, Any], prompt: str) -> str: + """Best-effort object noun for grounding a regional removal.""" + label = str(args.get("label") or args.get("object") or "").strip() + if label: + return label + m = re.search( + r"\b(?:remove|delete|erase|eliminate|get\s+rid\s+of|take\s+out)\s+(?:the\s+|a\s+|an\s+)?" + r"([a-z][a-z \-]{1,30}?)(?:\s+(?:on|in|at|from|to|near|behind|beside|by)\b|[.,;!?]|$)", + prompt or "", + re.I, + ) + if m: + noun = m.group(1).strip().lower() + if noun and noun not in {"background", "bg"}: + return noun + for noun in ("person", "people", "man", "woman", "car", "dog", "sign", "logo"): + if re.search(rf"\b{re.escape(noun)}\b", prompt or "", re.I): + return "person" if noun in {"person", "people", "man", "woman"} else noun + return "object" + + +class PlanEditTool: + """Plan an image edit as ordered editor operations (no side effects).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "plan_edit: invalid", {"error": "plan_edit needs a non-empty `prompt`.", "exit_code": 1} + state = args.get("state") if isinstance(args.get("state"), dict) else {} + try: + data = await _post( + "/api/editor/plan", + {"prompt": prompt, "state": state, "mode": "plan", "session": _session(args, ctx)}, + ctx.get("owner"), + ) + except Exception as exc: + return "plan_edit: error", {"error": f"plan_edit failed: {exc}", "exit_code": 1} + steps = data.get("steps") or [] + reply = str(data.get("reply") or "") + return f"plan_edit: {len(steps)} step(s)", { + "output": reply or f"Planned {len(steps)} editor step(s).", + "reply": reply, + "planner": data.get("planner"), + "steps": steps, + "exit_code": 0, + } + + +class RunEditorCommandTool: + """Run a single validated editor op in the browser via the owner bridge.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + op = str(args.get("op") or "").strip() + if not op: + return "run_editor_command: invalid", {"error": "run_editor_command needs `op`.", "exit_code": 1, "completed": False} + params = args.get("params") if isinstance(args.get("params"), dict) else {} + wait = bool(args.get("wait", True)) + payload = { + "session": _session(args, ctx), + "op": op, + "params": params, + "wait": wait, + "timeout": float(args.get("timeout") or 60), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner")) + except Exception as exc: + return f"run_editor_command: {op}", {"error": f"run_editor_command failed: {exc}", "exit_code": 1, "completed": False, "status": "failed"} + + is_queued = bool(data.get("queued")) or data.get("status") == "queued" + if is_queued and (not wait or not data.get("ok")): + summary = json.dumps({"status": "queued", "queued": data.get("queued"), "session": data.get("session")}, ensure_ascii=False) + return f"run_editor_command: {op}", { + "output": summary, + "queued": data.get("queued"), + "status": "queued", + "pending": True, + "completed": False, + "ok": False, + "exit_code": 1, + } + + ok = bool(data.get("ok") is True and data.get("completed", True) is not False) + status = data.get("status") or ("completed" if ok else "failed") + result = data.get("result") + summary = json.dumps({"ok": ok, "status": status, "error": data.get("error"), "result": result}, ensure_ascii=False) + return f"run_editor_command: {op}", { + "output": summary[:4000], + "result": result, + "screenshot": data.get("screenshot"), + "status": status, + "completed": ok, + "ok": ok, + "exit_code": 0 if ok else 1, + } + + +class GetEditorScreenshotTool: + """Capture the current editor canvas as a data URL.""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + payload = { + "session": _session(args, ctx), + "op": "screenshot", + "params": {}, + "wait": True, + "timeout": float(args.get("timeout") or 30), + } + try: + data = await _post("/api/editor/bridge/command", payload, ctx.get("owner"), timeout=45) + except Exception as exc: + return "get_editor_screenshot: error", {"error": f"get_editor_screenshot failed: {exc}", "exit_code": 1} + result = data.get("result") if isinstance(data.get("result"), dict) else {} + shot = data.get("screenshot") or result.get("dataUrl") + if not shot: + return "get_editor_screenshot: unavailable", { + "error": "No editor screenshot was returned (is the editor open?).", + "exit_code": 1, + } + return "get_editor_screenshot", {"output": "Captured the editor canvas.", "screenshot": shot, "exit_code": 0} + + +class AiEditImageTool: + """Apply an AI image transform (remove-bg/upscale/denoise/enhance/inpaint).""" + + async def execute(self, content: Any, ctx: Dict[str, Any]): + args = _parse(content) + _t = {"start": time.perf_counter()} + op = str(args.get("op") or "").strip().lower() + if op not in _AI_OPS: + return "ai_edit_image: invalid", { + "error": f"ai_edit_image `op` must be one of {sorted(_AI_OPS)}.", + "exit_code": 1, + } + owner = ctx.get("owner") + if op in ("inpaint", "generative") and not _provider_available(owner): + return "ai_edit_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + image = _as_data_url(args.get("image")) + if image is None and isinstance(args.get("image"), str) and args["image"].startswith("/"): + # A same-origin gallery/attachment URL: fetch it (owner-scoped) as bytes. + if not _image_reference_path(args["image"]): + return "ai_edit_image: invalid image", {"error": "Invalid image reference.", "exit_code": 1} + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.get(f"{_INTERNAL_BASE}{args['image']}", headers=_internal_headers(owner)) + resp.raise_for_status() + if not resp.headers.get("content-type", "").lower().startswith("image/"): + return "ai_edit_image: invalid image", {"error": "Image reference is not an image.", "exit_code": 1} + image = _as_data_url("data:image/png;base64," + base64.b64encode(resp.content).decode()) + except Exception: + image = None + used_canvas = False + if not image: + shot = await GetEditorScreenshotTool().execute(json.dumps({"session": session}), ctx) + image = (shot[1] or {}).get("screenshot") if isinstance(shot, tuple) else None + used_canvas = bool(image) + if not image: + return "ai_edit_image: no image", { + "error": "ai_edit_image needs an `image` (data URL or same-origin URL) or an open editor.", + "exit_code": 1, + } + _t["image"] = time.perf_counter() + payload: Dict[str, Any] = {"image": image} + model_choice = str(args.get("model") or "").strip() + mask_used = False + place_mode: Optional[str] = None + if op == "upscale": + payload["scale"] = int(args.get("scale") or 2) + elif op == "denoise": + payload["strength"] = float(args.get("strength") or 0.5) + elif op in ("inpaint", "generative"): + payload["prompt"] = str(args.get("prompt") or "").strip() + if model_choice: + # inpaint_proxy reads `_model` to route to a specific model. + payload["_model"] = model_choice + mode = str(args.get("mode") or "instruction").strip().lower() + _steps = int(args.get("steps") or 0) + if _steps > 0: + payload["steps"] = _steps + mask_used = False + mask = None + if mode == "masked": + mask = _as_data_url(args.get("mask")) + if not mask: + mask = await _fetch_selection_mask(owner, session) + # An empty/tiny SAM mask on small figures is useless: drop it. + if mask and _mask_white_fraction(mask) < _MIN_MASK_FRACTION: + mask = None + if not mask: + # Fall back to the grounding detector (OWL-ViT): a dilated + # box mask around the matching object. + box = await _ground_box(owner, image, _mask_label(args, payload["prompt"])) + if box: + candidate = _rect_mask_data_url(image, box, pad_frac=0.15) + if candidate and _mask_white_fraction(candidate) >= _MIN_MASK_FRACTION: + mask = candidate + if mask: + payload["mask"] = mask + payload["mode"] = "masked" + mask_used = True + else: + # Nothing usable to mask -> whole-image instruction fallback + # rather than placing a transparent/empty layer. + mode = "instruction" + if mode != "masked": + # Instruction mode: whole-image edit with no tight mask. Send a + # full-white mask so the edit endpoint accepts the request and + # regenerates the whole frame under the instruction prompt, which + # names what must stay unchanged. This avoids the mask-compositing + # halos that loose selections caused. + payload["mode"] = "instruction" + payload.pop("mask", None) + payload["prompt"] = _ensure_keep_clause(payload["prompt"]) + try: + from PIL import Image as _PILImage + + _raw = _data_url_bytes(image) + _w, _h = _PILImage.open(io.BytesIO(_raw)).size + _mbuf = io.BytesIO() + _PILImage.new("L", (_w, _h), 255).save(_mbuf, "PNG") + payload["mask"] = "data:image/png;base64," + base64.b64encode(_mbuf.getvalue()).decode() + except Exception: + pass + place_mode = "masked" if mask_used else "instruction" + _t["prep"] = time.perf_counter() + try: + data = await _post(_AI_OPS[op], payload, owner) + except Exception as exc: + logger.info( + "image-op ai_edit_image op=%s stage=provider FAILED image_ms=%.1f prep_ms=%.1f total_ms=%.1f err=%s", + op, (_t["image"] - _t["start"]) * 1000.0, (_t["prep"] - _t["image"]) * 1000.0, + (time.perf_counter() - _t["start"]) * 1000.0, exc, + ) + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) failed: {exc}", "exit_code": 1} + _t["provider"] = time.perf_counter() + result = data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return f"ai_edit_image: {op}", {"error": f"ai_edit_image ({op}) returned no image.", "exit_code": 1} + if _image_is_blank(data_url): + return f"ai_edit_image: {op}", { + "error": f"ai_edit_image ({op}) produced an empty layer.", + "failure_kind": "empty_result", + "exit_code": 1, + } + + info: Dict[str, Any] = { + "output": f"Applied AI edit '{op}'" + (" to the current canvas." if used_canvas else "."), + "op": op, + "model": data.get("model") or model_choice or "", + "exit_code": 0, + } + if _incognito(ctx): + info["image_url"] = data_url + else: + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), f"ai_edit_image:{op}", f"ai_edit_image:{op}")) + except Exception as exc: + logger.warning("ai_edit_image persist failed: %s", exc) + _t["persist"] = time.perf_counter() + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + f"AI {op}", + owner, + session, + masked=mask_used, + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + mode=place_mode, + ) + info["placed"] = placed + if placed: + info["output"] += " Placed the result as a new layer." + _t["place"] = time.perf_counter() + logger.info( + "image-op ai_edit_image op=%s mode=%s canvas=%s image_ms=%.1f mask_ms=%.1f " + "model_ms=%.1f persist_ms=%.1f place_ms=%.1f total_ms=%.1f", + op, place_mode or "instruction", used_canvas, + (_t["image"] - _t["start"]) * 1000.0, + (_t["prep"] - _t["image"]) * 1000.0, + (_t["provider"] - _t["prep"]) * 1000.0, + (_t["persist"] - _t["provider"]) * 1000.0, + (_t["place"] - _t["persist"]) * 1000.0, + (_t["place"] - _t["start"]) * 1000.0, + ) + return f"ai_edit_image: {op}", info + + +class GenerateImageLayerTool: + """Generate an image with the configured image model and place it as a layer.""" + + async def execute(self, content: str, ctx: dict): + args = _parse(content) + prompt = str(args.get("prompt") or "").strip() + if not prompt: + return "generate_image: invalid", {"error": "generate_image needs a non-empty `prompt`.", "exit_code": 1} + owner = ctx.get("owner") + if not _provider_available(owner): + return "generate_image: provider required", { + "error": _provider_message(), + "failure_kind": "image_provider_unavailable", + "needs_provider": True, + "exit_code": 1, + } + session = _session(args, ctx) + payload: Dict[str, Any] = {"prompt": prompt} + for key in ("model", "size", "quality"): + if args.get(key): + payload[key] = args[key] + try: + from src.ai_interaction import do_generate_image + + data = await do_generate_image( + json.dumps(payload, ensure_ascii=False), + session_id=session, + owner=owner, + incognito=True, + ) + except Exception as exc: + return "generate_image: error", {"error": f"generate_image failed: {exc}", "exit_code": 1} + result = data.get("image_url") or data.get("image") or data.get("data") + data_url = _as_data_url(result) + if not data_url: + return "generate_image", {"error": "generate_image returned no image.", "exit_code": 1} + info: Dict[str, Any] = { + "output": f"Generated an image for: {prompt[:120]}", + "image_size": data.get("image_size"), + "model": data.get("image_model") or str(args.get("model") or ""), + "exit_code": 0, + } + if _incognito(ctx): + info["image_url"] = data_url + else: + try: + info.update(_persist_png(_data_url_bytes(data_url), owner, ctx.get("session_id"), prompt, "generate_image")) + except Exception as exc: + logger.warning("generate_image persist failed: %s", exc) + if bool(args.get("place", True)): + placed = await _place_layer( + data_url, + str(args.get("name") or "Generated image"), + owner, + session, + masked=bool(args.get("mask")), + blend=str(args.get("blend") or "normal"), + opacity=float(args.get("opacity") or 100), + ) + info["placed"] = placed + if placed: + info["output"] += " Placed it as a new layer." + return "generate_image", info diff --git a/src/agent_tools/web_tools_prewave3.py b/src/agent_tools/web_tools_prewave3.py new file mode 100644 index 000000000..700ab4a85 --- /dev/null +++ b/src/agent_tools/web_tools_prewave3.py @@ -0,0 +1,3863 @@ +import asyncio +import base64 +import contextlib +import contextvars +import inspect +import io +import json +import os +import re +import signal +import shutil +import sys +import tempfile +import time +import html +import hashlib +import pwd +import urllib.parse +import urllib.request +import uuid +from pathlib import Path +from typing import Dict, Any + +from core import platform_compat +from src import browser_lifecycle, process_lifecycle +from src.constants import MAX_OUTPUT_CHARS + +PDF_EXTRACT_MAX_BYTES = 80_000_000 +_ACTIVE_BROWSER_SESSIONS: set[str] = set() + + +def _service_home() -> Path: + """Return the account home even when a task overrides ``HOME``.""" + + try: + return Path(pwd.getpwuid(os.getuid()).pw_dir) + except (KeyError, OSError): + return Path.home() + + +def _host_npm_roots() -> list[Path]: + """Return user npm roots visible to a runtime using an isolated HOME.""" + + roots = [_service_home() / ".npm"] + roots.extend(Path("/home").glob("*/.npm")) + roots.append(Path("/root/.npm")) + return list(dict.fromkeys(roots)) + + +def _accessible_glob(roots: list[Path], pattern: str): + """Yield matches while ignoring roots unreadable by the service account.""" + + for root in dict.fromkeys(roots): + try: + yield from root.glob(pattern) + except (OSError, PermissionError): + continue + + +def _browser_executable_candidates() -> list[Path]: + """Return executable Chromium builds from standard host cache layouts.""" + + homes = [_service_home()] + homes.extend(path for path in _accessible_glob([Path("/home")], "*") if path.is_dir()) + homes.append(Path("/root")) + patterns = ( + ".cache/ms-playwright/chromium-*/chrome-linux64/chrome", + ".cache/ms-playwright/chromium_headless_shell-*/" + "chrome-headless-shell-linux64/chrome-headless-shell", + ".chromium-browser-snapshots/chromium/linux-*/chrome-linux/chrome", + ) + candidates = [ + path + for pattern in patterns + for path in _accessible_glob(list(dict.fromkeys(homes)), pattern) + if path.is_file() and os.access(path, os.X_OK) + ] + return sorted(dict.fromkeys(candidates), reverse=True) + + +def _bounded_browser_identity(value: str, *, max_length: int = 20) -> str: + """Return an agent-browser identity safe for Unix socket paths.""" + + safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value or "")).strip("-._") + if safe == value and 0 < len(safe) <= max_length: + return safe + return hashlib.sha256(str(value).encode("utf-8")).hexdigest()[:max_length] + + +def _scoped_browser_session(namespace: str, session_id: str) -> str: + """Build one upstream-compatible session key for runtime isolation.""" + + scope = f"{str(namespace or 'odysseus-ui')}\0{str(session_id or '')}" + return f"ody-{_bounded_browser_identity(scope)}" + + +def _browser_namespace(env: dict[str, str] | None) -> str: + return str( + (env or {}).get("ODYSSEUS_BROWSER_NAMESPACE") + or os.getenv("ODYSSEUS_BROWSER_NAMESPACE", "odysseus-ui") + ).strip() or "odysseus-ui" + + +# Browser CLI processes started by the current private_browser call, so a +# cancellation can stop every client it spawned, not only the main command. +_BROWSER_CALL_PROCS: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "_BROWSER_CALL_PROCS", default=None +) + + +async def _spawn_browser_cli(*command, **kwargs): + proc = await asyncio.create_subprocess_exec(*command, **kwargs) + # Identity taken while we hold the unreaped child: teardown later signals + # its group only while this identity still verifies, never on the pid + # alone (the event loop may reap it before returncode is observed). + pid = getattr(proc, "pid", None) + if isinstance(pid, int) and pid > 0: + with contextlib.suppress(Exception): + proc._ody_identity = process_lifecycle.ProcessIdentity.capture(pid, pgid=pid) + tracked = _BROWSER_CALL_PROCS.get() + if tracked is not None: + tracked.append(proc) + return proc + + +def _browser_pid_file_candidates( + runtime_dir: Path, namespace: str, session_id: str | None +) -> list[Path]: + """Return only the daemon pid files owned by one browser runtime. + + Current agent-browser stores ``--session`` state directly below its + runtime directory. Older builds used a namespace/run subdirectory, so + retain that layout as a compatibility fallback. When no session is + supplied, only the legacy namespace directory is eligible; never sweep + all upstream sessions from the shared root. + """ + + browser_root = runtime_dir / "agent-browser" + legacy_run = ( + browser_root / "namespaces" / _bounded_browser_identity(namespace) / "run" + ) + if not session_id: + return list(legacy_run.glob("ody-*.pid")) if legacy_run.is_dir() else [] + + scoped = _scoped_browser_session(namespace, session_id) + candidates = [browser_root / f"{scoped}.pid"] + if legacy_run.is_dir(): + candidates.extend( + [ + legacy_run / f"ody-{_bounded_browser_identity(session_id)}.pid", + legacy_run / f"{scoped}.pid", + ] + ) + return list(dict.fromkeys(candidates)) + + +# Linux exposes one command line per pid under /proc; macOS and Windows do not. +# Kept as a module attribute so the procfs-dependent paths stay testable on a +# host that has no procfs, and on one that does. + + +def _process_command_line(pid: int) -> str | None: + """Command line of a running process, or ``None`` when it cannot be read. + + ``None`` means "this host cannot tell", not "the process is gone". Off + Linux there is no procfs to read a command line from, so callers must not + treat it as proof that the process exited. + """ + + try: + return (platform_compat.PROC_ROOT / str(pid) / "cmdline").read_bytes().replace( + b"\0", b" " + ).decode("utf-8", errors="replace") + except (OSError, UnicodeError): + return None + + +def _process_is_alive(pid: int) -> bool: + """Whether a pid currently exists. + + Delegates to ``core.platform_compat.pid_alive`` rather than probing with + ``os.kill(pid, 0)`` directly. That probe is POSIX-only: CPython's Windows + ``os.kill`` calls ``TerminateProcess(handle, sig)`` for any signal other + than CTRL_C / CTRL_BREAK, so it would *kill* the daemon it is asked about. + Windows is also where there is no procfs, which is precisely when this + function gets called at all. + + ``pid_alive`` reads False for a pid that ``os.kill`` reports with + ``PermissionError`` — a live process owned by another user. Neither caller + here wants a different answer: the sweep only unlinks a pid file it wrote + itself, and treating somebody else's pid as "not our daemon" is the safe + reading in both. + """ + + return platform_compat.pid_alive(pid) + +_SCHOLARLY_METADATA_CUE_RE = re.compile( + r"\b(?:accept(?:ed|ance)?|publish(?:ed|ing|cation)?|venue|conference|" + r"journal|proceedings|doi)\b", + re.IGNORECASE, +) +_EXPLICIT_ARXIV_ID_RE = re.compile( + r"\barxiv(?:\.org)?\b.{0,24}\b\d{4}\.\d{4,5}(?:v\d+)?\b", + re.IGNORECASE, +) +_SCHOLARLY_SUBJECT_CUE_RE = re.compile( + r"\b(?:paper|preprint|arxiv|benchmark|language model|vision-language|" + r"ICLR|ICML|CVPR|NeurIPS|ACL|EMNLP|AAAI|IEEE)\b", + re.IGNORECASE, +) +_DISTINCTIVE_SCHOLARLY_NAME_RE = re.compile( + r"\b(?:[A-Za-z][A-Za-z-]*\d[A-Za-z0-9.-]*|" + r"[A-Z][A-Za-z0-9]*-[A-Z][A-Za-z0-9]*)\b" +) + + +def _is_scholarly_metadata_query(query: str) -> bool: + text = str(query or "") + cues = { + match.group(0).casefold() + for match in _SCHOLARLY_METADATA_CUE_RE.finditer(text) + } + # Source discovery for a named paper should return ranked URLs/snippets, + # not download every result page. Full-page fetching can spend the entire + # tool timeout on one blocked publisher before the model ever sees the + # arXiv/official result it needs for pdf_extract. + if ( + re.search(r"\b(?:paper|preprint|arxiv)\b", text, re.IGNORECASE) + and re.search( + r"\b(?:table|figure|benchmark|results?|pdf|source|url)\b", + text, + re.IGNORECASE, + ) + ): + return True + if ( + _DISTINCTIVE_SCHOLARLY_NAME_RE.search(text) + and re.search(r"\b(?:table|figure|benchmark|scores?|results?)\b", text, re.IGNORECASE) + ): + return True + if not cues: + return False + return bool( + _EXPLICIT_ARXIV_ID_RE.search(text) + or _SCHOLARLY_SUBJECT_CUE_RE.search(text) + or len(cues) >= 3 + ) + + +def _is_official_site_metadata_query(query: str) -> bool: + """Canonical-homepage lookup needs ranked URLs, not downloaded pages.""" + + value = re.sub(r"\s+", " ", str(query or "")).strip() + return bool(re.fullmatch( + r"(?:the\s+)?official\s+.+?\s+(?:website|web\s*site|site|homepage)" + r"|.+?\s+official\s+(?:website|web\s*site|site|homepage)", + value, + re.IGNORECASE, + )) + + +def _format_search_metadata(query: str, results: list[dict]) -> tuple[str, list[dict]]: + sources = [ + {"url": str(item.get("url") or ""), "title": str(item.get("title") or "")} + for item in results + if item.get("url") + ] + parts = [f"WEB SEARCH METADATA RESULTS\nQuery: {query}"] + for index, item in enumerate(results, 1): + parts.extend([ + "", + f"[{index}] {item.get('title') or 'Untitled result'}", + f"URL: {item.get('url') or ''}", + f"Snippet: {str(item.get('snippet') or '')[:1200]}", + ]) + if not results: + parts.append("\nNo search results found.") + return "\n".join(parts), sources + + +def _looks_like_youtube_video_id(value: str) -> bool: + return bool(re.fullmatch(r"[A-Za-z0-9_-]{10,16}", str(value or "").strip())) + + +def _arxiv_listing_api_hint(url: str, error: str) -> str: + """Return an explicit recovery path for a blocked arXiv date listing.""" + if "406" not in str(error or ""): + return "" + parsed = urllib.parse.urlsplit(str(url or "")) + if parsed.hostname not in {"arxiv.org", "www.arxiv.org"}: + return "" + match = re.fullmatch(r"/list/([A-Za-z0-9.-]+)", parsed.path.rstrip("/")) + date = urllib.parse.parse_qs(parsed.query).get("date", [""])[0] + if not match or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date): + return "" + day = date.replace("-", "") + query = f"cat:{match.group(1)} AND submittedDate:[{day}0000 TO {day}2359]" + api_url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode({ + "search_query": query, "start": "0", "max_results": "100", + }) + return ( + " arXiv rejected its static listing (HTTP 406). The public Atom API is " + f"available for this category/day; call web_fetch on {api_url} to read " + "the dated feed, then inspect individual paper sources for details." + ) + + +class WebSearchTool: + async def execute(self, content: str, ctx: dict) -> dict: + from src.search import comprehensive_web_search, searxng_search_results + progress_cb = ctx.get("progress_cb") if isinstance(ctx, dict) else None + raw = content.strip() + query = raw + time_filter = None + max_pages = 5 + if raw.startswith("{"): + try: + parsed = json.loads(raw) + if isinstance(parsed, dict) and "query" in parsed: + query = str(parsed.get("query", "")).strip() + tf = parsed.get("time_filter") or parsed.get("freshness") + if isinstance(tf, str) and tf.lower() in ("day", "week", "month", "year"): + time_filter = tf.lower() + mp = parsed.get("max_pages") + if isinstance(mp, int) and 1 <= mp <= 10: + max_pages = mp + except json.JSONDecodeError: + pass + if not query: + query = raw.split("\n")[0].strip() + if time_filter is None: + from src.search_intent import inferred_search_publication_window + time_filter = inferred_search_publication_window(query) + loop = asyncio.get_running_loop() + if progress_cb: + await progress_cb({ + "elapsed_s": 0, + "tail": f"Searching web for: {query[:160]}", + }) + try: + if _is_scholarly_metadata_query(query) or _is_official_site_metadata_query(query): + results = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: searxng_search_results(query, max_pages, **({'time_filter': time_filter} if time_filter else {})), + ), + timeout=30, + ) + text, sources = _format_search_metadata(query, results) + else: + text, sources = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: comprehensive_web_search( + query, + max_pages=max_pages, + time_filter=time_filter, + return_sources=True, + ), + ), + timeout=30, + ) + except asyncio.TimeoutError: + # Comprehensive search also downloads several result pages. A + # slow or hostile publisher must not erase the ranked search + # evidence that was already available. Fall back to the metadata + # path so the agent can choose a source and continue with + # web_fetch/private_browser. Keep this bounded independently: the + # abandoned executor thread may still be winding down. + try: + results = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: searxng_search_results(query, max_pages, **({'time_filter': time_filter} if time_filter else {})), + ), + timeout=12, + ) + text, sources = _format_search_metadata(query, results) + if sources: + output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text + output += "\n\n" + return { + "output": output, + "exit_code": 0, + "evidence_status": "available", + "degraded_mode": "metadata_after_content_timeout", + } + except Exception: + pass + return { + "error": f"web_search timed out after 30s: {query[:200]}", + "exit_code": 1, + } + except Exception as e: + return { + "error": f"web_search failed: {type(e).__name__}: {str(e) or 'no details'}", + "exit_code": 1, + "untrusted_content": True, + } + from .weather_tools import WeatherTool, weather_location_from_query + weather_location = weather_location_from_query(query) + if not sources and time_filter and weather_location: + # A forecast or current fact need not live on a newly published page. + try: + text, sources = await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: comprehensive_web_search( + query, max_pages=max_pages, time_filter=None, + return_sources=True, + ), + ), + timeout=20, + ) + except Exception: + pass + if not sources: + from src.turn_contract import active_turn_contract + contract = active_turn_contract() + policy = ctx.get("tool_policy") if isinstance(ctx, dict) else None + weather_allowed = ( + weather_location + and "get_weather" not in (ctx.get("disabled_tools") or ()) + and not (policy and policy.blocks("get_weather")) + and not (contract and not contract.permits("get_weather")) + ) + if weather_allowed: + weather = await WeatherTool().execute(json.dumps({"location": weather_location}), ctx) + if weather.get("exit_code") == 0: + return weather + if progress_cb: + await progress_cb({ + "elapsed_s": 30, + "tail": "Search completed; preparing sources.", + }) + # Compact the complete report before the transport cap. Otherwise + # repeated summaries from early sources permanently erase later pages. + from src.search_passages import bounded_search_observation + text = bounded_search_observation(text, MAX_OUTPUT_CHARS) + output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text + if sources: + output += "\n\n" + return {"output": output, "exit_code": 0, + "evidence_status": "available" if sources else "empty"} + +class WebFetchTool: + _MAX_BATCH_URLS = 12 + _MAX_BATCH_CONCURRENCY = 4 + + @staticmethod + def _query_from_request(request_text: str) -> str: + """Extract distinctive document/table terms from the active request.""" + candidates = re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", request_text or "") + generic = { + "http", "https", "arxiv.org", "pdf", "pdfs", "www", "com", "org", + "please", "download", "read", "create", "save", "write", "workspace", + "table", "tables", "report", "benchmark", "benchmarks", "score", "scores", + "model", "models", "file", "chart", "data", "using", "from", "with", + } + selected: list[str] = [] + for token in candidates: + folded = token.casefold() + distinctive = ( + any(ch.isdigit() for ch in token) + or any(ch.isupper() for ch in token[1:]) + or "-" in token + or folded.endswith("qa") + ) + if distinctive and folded not in generic and token not in selected: + selected.append(token) + return ", ".join(selected[:20]) + + @staticmethod + def _focused_passages(text: str, query: str, max_chars: int) -> str: + """Select bounded page/line passages matching long-document terms.""" + segments = [term.strip() for term in re.split(r"[,|;\n]+", query) if term.strip()] + stopwords = { + "benchmark", "table", "evaluation", "scores", "score", "model", + "paper", "document", "accuracy", "results", "result", + } + terms: list[str] = [] + for segment in segments: + if len(segments) > 1 and len(segment) >= 2: + terms.append(segment) + for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", segment): + if token.casefold() not in stopwords and token not in terms: + terms.append(token) + terms = terms[:16] + if not terms: + return text + lines = text.splitlines() + anchors: list[int] = [] + for index, line in enumerate(lines): + low = line.casefold() + if any(term.casefold() in low for term in terms): + anchors.append(index) + if not anchors: + return f"[No passages matched query: {query}]" + # Keep independent windows around each hit. Merging nearby hits can + # accidentally turn a dense PDF table/page into one giant passage and + # clip the target row at the output boundary. + windows: list[tuple[int, int]] = [] + for anchor in anchors: + table_start = None + if len(re.findall(r"\b\d+(?:\.\d+)?\b", lines[anchor])) >= 4: + for candidate in range(anchor, max(-1, anchor - 81), -1): + if re.match(r"\s*Table\s+\d+", lines[candidate], re.IGNORECASE): + table_start = candidate + break + interval = ( + (table_start, min(len(lines), anchor + 9)) + if table_start is not None + else (max(0, anchor - 28), min(len(lines), anchor + 15)) + ) + if interval not in windows: + windows.append(interval) + ranked_passages = [] + for start, end in windows: + passage = "\n".join(lines[start:end]) + low = passage.casefold() + coverage = sum(1 for term in terms if term.casefold() in low) + occurrences = sum(min(low.count(term.casefold()), 8) for term in terms) + numeric_density = min(len(re.findall(r"\b\d+(?:\.\d+)?\b", passage)), 40) + table_rows = 0 + for line in passage.splitlines(): + line_low = line.casefold() + if ( + any(term.casefold() in line_low for term in terms) + and len(re.findall(r"\b\d+(?:\.\d+)?\b", line)) >= 4 + ): + table_rows += 1 + score = coverage * 100 + occurrences * 4 + numeric_density + min(table_rows, 4) * 200 + # PDF table extraction can collapse a full page into one enormous + # line. Bound each candidate so an earlier broad table cannot + # consume the entire result before the exact matching table. + passage_cap = max(2000, min(8000, max_chars // 3)) + if len(passage) > passage_cap: + # HTML-to-text can collapse an entire arXiv page into one line. + # Choose the densest term/numeric window instead of blindly + # retaining the abstract at the head of that line. + candidates: list[tuple[int, int, str]] = [] + passage_low = passage.casefold() + for term in terms: + term_low = term.casefold() + cursor = 0 + for _ in range(8): + position = passage_low.find(term_low, cursor) + if position < 0: + break + window_start = max(0, min( + len(passage) - passage_cap, + position - passage_cap // 2, + )) + window = passage[window_start:window_start + passage_cap] + window_low = window.casefold() + window_coverage = sum( + 1 for candidate in terms + if candidate.casefold() in window_low + ) + window_numbers = min( + len(re.findall(r"\b\d+(?:\.\d+)?\b", window)), 300 + ) + candidates.append(( + window_coverage * 1000 + + window_numbers + + len(term) * 10 + - window_start // 100, + -window_start, + window, + )) + cursor = position + max(1, len(term_low)) + if candidates: + passage = max(candidates, key=lambda item: (item[0], item[1]))[2] + else: + passage = passage[:passage_cap] + passage += "\n[...passage windowed around query terms]" + ranked_passages.append((score, start, passage)) + ranked_passages.sort(key=lambda item: (-item[0], item[1])) + passages: list[str] = [] + for _score, _index, passage in ranked_passages: + # Collapse duplicate/near-identical windows without merging them. + normalized = re.sub(r"\s+", " ", passage).strip() + if any(normalized in re.sub(r"\s+", " ", prior) for prior in passages): + continue + passages.append(passage) + if len(passages) >= 1: + break + selected = "\n\n--- matching passage ---\n\n".join(passages) + if len(selected) > max_chars: + selected = selected[:max_chars] + "\n\n[...focused passages truncated]" + return f"[Focused passages for: {query}]\n\n{selected}" + + async def _execute_batch(self, payload: dict, ctx: dict) -> dict: + """Fetch several known URLs concurrently with bounded combined output.""" + raw_urls = payload.get("urls") + if not isinstance(raw_urls, list) or not raw_urls: + return { + "error": "web_fetch: urls must be a non-empty array of at most 12 URLs", + "exit_code": 1, + } + if len(raw_urls) > self._MAX_BATCH_URLS: + return { + "error": "web_fetch: urls must contain at most 12 URLs", + "exit_code": 1, + } + + shared_query = str(payload.get("query") or "").strip() + shared_full = payload.get("full") is True + requests: list[dict[str, Any]] = [] + for index, item in enumerate(raw_urls): + if isinstance(item, str): + request = {"url": item, "query": shared_query, "full": shared_full} + elif isinstance(item, dict): + request = { + "url": str(item.get("url") or "").strip(), + "query": str(item.get("query") or shared_query).strip(), + "full": item.get("full") is True or shared_full, + } + else: + return { + "error": f"web_fetch: urls item {index + 1} must be a URL string or object", + "exit_code": 1, + } + if not request["url"]: + return { + "error": f"web_fetch: urls item {index + 1} is missing url", + "exit_code": 1, + } + requests.append(request) + + semaphore = asyncio.Semaphore(self._MAX_BATCH_CONCURRENCY) + child_ctx = dict(ctx) if isinstance(ctx, dict) else {} + child_ctx.pop("progress_cb", None) + + async def fetch_one(request: dict[str, Any]) -> dict: + async with semaphore: + return await self.execute(json.dumps(request), child_ctx) + + results = await asyncio.gather(*(fetch_one(request) for request in requests)) + per_item_cap = max(800, min(6000, (MAX_OUTPUT_CHARS - 1000) // len(results))) + sections: list[str] = [] + successful = 0 + summaries: list[dict[str, Any]] = [] + for index, (request, result) in enumerate(zip(requests, results), start=1): + ok = result.get("exit_code") == 0 + if ok: + successful += 1 + body = str(result.get("output") or "") + else: + body = "ERROR: " + str(result.get("error") or "fetch failed") + if len(body) > per_item_cap: + body = body[:per_item_cap] + "\n[...batch item truncated]" + sections.append(f"## URL {index}: {request['url']}\n{body}") + summaries.append({ + "url": request["url"], + "exit_code": int(result.get("exit_code", 1)), + }) + output = "\n\n".join(sections) + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + "\n\n[...batch output truncated]" + return { + "output": output, + "batch_results": summaries, + "successful": successful, + "requested": len(requests), + "exit_code": 0 if successful else 1, + } + + async def execute(self, content: str, ctx: dict) -> dict: + from src.search.content import fetch_webpage_content + from src.constants import WEB_FETCH_HARD_MAX_BYTES + raw = content.strip() + if raw.startswith("{"): + try: + batch_payload = json.loads(raw) + except json.JSONDecodeError: + batch_payload = None + if isinstance(batch_payload, dict) and "urls" in batch_payload: + return await self._execute_batch(batch_payload, ctx) + url = "" + max_bytes = None + query = "" + if raw.startswith("{"): + try: + parsed = json.loads(raw) + if isinstance(parsed, dict): + url = str(parsed.get("url") or "").strip() + query = str(parsed.get("query") or "").strip() + # Download-budget override (#3812): "full": true raises the + # budget to the hard cap; an explicit max_bytes is clamped + # to the hard cap downstream. Default stays the soft cap. + if parsed.get("full") is True: + max_bytes = WEB_FETCH_HARD_MAX_BYTES + mb = parsed.get("max_bytes") + if isinstance(mb, int) and mb > 0: + max_bytes = mb + except json.JSONDecodeError: + url = "" + if not url: + url = raw.split("\n")[0].strip() + local_path = url + if local_path.lower().startswith("file:///workspace/"): + local_path = local_path[7:] + if local_path.startswith("/workspace/"): + suffix = Path(local_path.split("?", 1)[0].split("#", 1)[0]).suffix.lower() + if suffix in {".html", ".htm"}: + return { + "error": ( + "web_fetch: local HTML requires private_browser so rendered DOM and " + "page errors are inspected" + ), + "exit_code": 1, + } + if suffix in { + ".avi", ".bmp", ".gif", ".jpeg", ".jpg", ".m4v", ".mkv", + ".mov", ".mp4", ".mpeg", ".mpg", ".png", ".webm", ".webp", + }: + return { + "error": "web_fetch: local visual media requires inspect_media", + "exit_code": 1, + } + from src.agent_tools.filesystem_tools import ReadFileTool + + result = await ReadFileTool().execute(local_path, ctx) + if result.get("exit_code") == 0: + result = dict(result) + result["output"] = ( + "Argument note: Read the local workspace path with read_file; " + "use web_fetch only for HTTP(S) URLs.\n" + str(result.get("output") or "") + ) + return result + if not url or url.startswith("{") or any(c in url for c in (" ", "\t", "\n")): + return {"error": "web_fetch: provide a single URL or domain, e.g. example.com", "exit_code": 1} + low = url.lower() + if "://" in low and not low.startswith(("http://", "https://")): + return {"error": f"web_fetch: unsupported URL scheme (only http/https): {url[:80]}", "exit_code": 1} + if not low.startswith(("http://", "https://")): + url = "https://" + url + if re.search(r"(?:\.pdf(?:[?#]|$)|arxiv\.org/pdf/)", url, re.IGNORECASE) and not query: + runtime_context = ctx.get("client_runtime_context") if isinstance(ctx, dict) else None + request_text = ( + str(runtime_context.get("request_text") or "") + if isinstance(runtime_context, dict) else "" + ) + query = self._query_from_request(request_text) + if not query: + return { + "error": ( + "web_fetch: this is a PDF. Use pdf_extract with this URL and a query " + "that names the target model, metrics, or table; do not download it " + "with Python/curl and do not estimate missing values." + ), + "exit_code": 1, + } + loop = asyncio.get_running_loop() + try: + def _fetch(): + kwargs = {"timeout": 10} + try: + sig = inspect.signature(fetch_webpage_content) + if "max_bytes" in sig.parameters: + kwargs["max_bytes"] = max_bytes + except (TypeError, ValueError): + # Some deployed/test shims may not expose a signature. + # Prefer compatibility over failing the whole fetch. + pass + return fetch_webpage_content(url, **kwargs) + + result = await asyncio.wait_for( + loop.run_in_executor(None, _fetch), + timeout=30, + ) + except asyncio.TimeoutError: + return {"error": f"web_fetch: timed out fetching {url}", "exit_code": 1} + except Exception as e: + return {"error": f"web_fetch: {url}: {e}", "exit_code": 1} + err = result.get("error") + text = (result.get("linked_content") or result.get("content") or "").strip() + title = result.get("title") or "" + + if not text: + if err: + arxiv_hint = _arxiv_listing_api_hint(url, str(err)) + return { + "error": f"web_fetch: {url}: {err}{arxiv_hint}", + "exit_code": 1, + "untrusted_content": True, + } + return {"error": f"web_fetch: {url}: no readable text content (not HTML, or the page needs JS/login)", "exit_code": 1} + + # Tell the model when the download budget cut the body short and how + # to get the rest, instead of silently presenting a partial page as + # the whole thing. + size_note = "" + if result.get("truncated"): + fetched = result.get("fetched_bytes") or 0 + total = result.get("total_bytes") + total_txt = f" of {total:,} bytes" if total else "" + size_note = ( + f"[partial content: download stopped at {fetched:,} bytes{total_txt}. " + f'Re-call with {{"url": "{url}", "full": true}} to fetch up to ' + f"{WEB_FETCH_HARD_MAX_BYTES:,} bytes.]\n\n" + ) + + # The notice must lead the output so the MAX_OUTPUT_CHARS trim below can + # never drop it. The title is untrusted, uncapped page content, so a + # giant title ahead of the notice could push it out of range; keep the + # notice first and cap the title as a second guard. + if len(title) > 300: + title = title[:300] + "..." + header = (f"# {title}\n" if title else "") + f"Source: {url}\n\n" + if query: + text = self._focused_passages(text, query, MAX_OUTPUT_CHARS - len(header) - 500) + output = size_note + header + text + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + ( + "\n\n[...truncated; re-call web_fetch with query terms to retrieve matching passages]" + if not query else "\n\n[...truncated]" + ) + return {"output": output, "exit_code": 0, "page_entries": result.get("page_entries") or []} + + +class PdfExtractTool: + """Extract focused, source-attributed passages from a PDF URL or workspace file. + + This deliberately builds on Odysseus' native web reader instead of adding + a benchmark transport. The separate semantic affordance keeps models from + downloading PDFs with ad-hoc Python and then guessing when extraction fails. + Local task PDFs use the same positioned reader, but are resolved through the + active workspace path policy rather than through an unrestricted filesystem + path. + """ + + @staticmethod + def _local_pdf_path(source: str) -> Path | None: + """Resolve a task-local PDF through the active workspace policy.""" + raw = str(source or "").strip() + if raw.lower().startswith("file://"): + parsed = urllib.parse.urlsplit(raw) + if parsed.netloc not in {"", "localhost"}: + return None + raw = urllib.parse.unquote(parsed.path) + if not raw.startswith("/workspace/") and not Path(raw).is_absolute(): + return None + try: + from src.tool_execution import _resolve_tool_path + + path = Path(_resolve_tool_path(raw)) + except (OSError, ValueError): + return None + return path if path.suffix.casefold() == ".pdf" else None + + @staticmethod + def _arxiv_html_url(url: str) -> str: + """Return the official HTML companion for an arXiv document URL.""" + match = re.search( + r"https?://(?:www\.)?arxiv\.org/(?:pdf|abs|html)/(?P\d{4}\.\d{4,5}(?:v\d+)?)", + str(url or ""), + re.IGNORECASE, + ) + return f"https://arxiv.org/html/{match.group('id')}" if match else "" + + @staticmethod + def _pdf_row_contains_term(row_text: str, term: str) -> bool: + """Match model names even when PDF glyph extraction splits punctuation.""" + row_folded = str(row_text or "").casefold() + term_folded = str(term or "").casefold() + if term_folded in row_folded: + return True + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", value.casefold()) + normalized_term = normalize(term_folded) + return bool(normalized_term) and normalized_term in normalize(row_folded) + + @staticmethod + def _pdf_row_exact_term_match(row_text: str, term: str) -> bool: + """Match a complete model label without accepting a named variant. + + PDF text extraction may split punctuation (``LLaVA- Onevision``), so a + literal equality check is too strict. The broad table matcher above is + intentionally useful for locating candidate pages, but it also makes + ``DeepSeek-VL2`` match ``DeepSeek-VL2-Tiny``. Row resolution needs the + narrower relation: tolerate extracted punctuation while rejecting an + attached hyphen/slash variant suffix. + """ + row = str(row_text or "") + chunks = re.findall(r"[A-Za-z0-9]+", str(term or "")) + if not chunks: + return False + pattern = re.compile( + r"(? list[str]: + """Return compact digit-bearing aliases used by shared-family headers. + + Comparison tables often put the vendor/family in the caption and use + headers such as ``R1-Zero`` or ``o1``. Restrict aliases to suffixes + containing a digit so ordinary trailing words cannot become matches. + """ + parts = [part for part in re.split(r"-+", str(term or "")) if part] + aliases: list[str] = [] + for index in range(1, len(parts)): + alias = "-".join(parts[index:]) + if any(character.isdigit() for character in alias) and len(alias) >= 2: + aliases.append(alias) + return aliases + + @staticmethod + def _pdf_row_contains_model_alias(row_text: str, term: str) -> bool: + """Match an exact full model label or a bounded table-header alias.""" + if PdfExtractTool._pdf_row_exact_term_match(row_text, term): + return True + return any( + PdfExtractTool._pdf_row_exact_term_match(row_text, alias) + for alias in PdfExtractTool._pdf_model_suffix_aliases(term) + ) + + @staticmethod + def _bibliography_evidence( + page_columns: list[tuple[int, str, str]], query: str + ) -> str: + """Return ordered reference entries for bibliography-focused queries. + + Academic PDFs commonly use two columns. Reading the whole page in + layout order interleaves those columns, while generic query scoring can + rank citation-heavy body pages above the actual reference section. + ``page_columns`` keeps each column independent so references can be + reconstructed in normal reading order without knowing task page + numbers or paper titles. + """ + bibliography_intent = re.search( + r"\b(?:references?|bibliograph(?:y|ies)|bibtex|citation\s+list)\b", + str(query or ""), + re.IGNORECASE, + ) + if not bibliography_intent: + return "" + + heading_pattern = re.compile( + r"(?im)^\s*(?:references|bibliography)(?:\s+\d+)?\s*$" + ) + start_index = -1 + start_offset = 0 + for index, (_page_number, left, _right) in enumerate(page_columns): + heading = heading_pattern.search(left) + if heading: + start_index = index + start_offset = heading.start() + break + if start_index < 0: + return "" + + ordered_pages: list[tuple[int, str]] = [] + for index, (page_number, left, right) in enumerate( + page_columns[start_index:], start=start_index + ): + if index == start_index: + left = left[start_offset:] + text = "\n".join(part.strip() for part in (left, right) if part.strip()) + if text: + ordered_pages.append((page_number, text)) + if not ordered_pages: + return "" + + joined = "\n".join(text for _page_number, text in ordered_pages) + starts = list(re.finditer(r"(?m)^\s*\[(\d+)\]\s*", joined)) + numbered_entries: list[tuple[int, str]] = [] + for position, match in enumerate(starts): + end = starts[position + 1].start() if position + 1 < len(starts) else len(joined) + numbered_entries.append(( + int(match.group(1)), + joined[match.start():end].strip(), + )) + requested_range = re.search( + r"\b(?:references?|refs?|citations?)\s*(?:numbers?\s*)?" + r"\[?(\d{1,4})\]?\s*(?:through|to|[-–—])\s*\[?(\d{1,4})\]?", + str(query or ""), + re.IGNORECASE, + ) + if requested_range and numbered_entries: + first, last = sorted(( + int(requested_range.group(1)), + int(requested_range.group(2)), + )) + selected_entries = [ + entry for number, entry in numbered_entries + if first <= number <= last + ] + if selected_entries: + pages = f"{ordered_pages[0][0]}-{ordered_pages[-1][0]}" + body = ( + f"[Local PDF bibliography evidence, pages {pages}; " + f"reference entries {first}-{last}]\n" + + "\n\n".join(selected_entries) + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography entries truncated]" + return body + + wants_arxiv = bool( + re.search(r"\b(?:arxiv|preprints?)\b", str(query or ""), re.IGNORECASE) + ) + if wants_arxiv: + # Keep complete numbered entries instead of returning every page. + # This is both more useful and prevents the bounded output cap from + # dropping late matching references. + entries: list[str] = [] + for _number, entry in numbered_entries: + if re.search(r"arxiv", entry, re.IGNORECASE): + entries.append(entry) + if entries: + pages = f"{ordered_pages[0][0]}-{ordered_pages[-1][0]}" + body = ( + f"[Local PDF bibliography evidence, pages {pages}; " + f"{len(entries)} arXiv/preprint entries]\n" + + "\n\n".join(entries) + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography entries truncated]" + return body + + body = "\n\n".join( + f"[Local PDF bibliography evidence, page {page_number}]\n{text}" + for page_number, text in ordered_pages + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...bibliography text truncated]" + return body + + @staticmethod + def _select_positioned_target_model( + model_terms: list[str], + rows: list[tuple[float, list[dict]]], + ) -> str: + """Choose the requested model that is actually present in selected PDF rows. + + Broad benchmark prompts often ask for several models in every PDF + request. Picking the longest/first requested model contaminates a GLM + or Seed table with the Qwen column. Score each requested model against + the selected positioned rows and prefer explicit in-table matches. + """ + if not model_terms: + return "" + table_text = " ".join( + str(word["text"]) + for _top, words in rows + for word in words + ) + table_folded = table_text.casefold() + table_normalized = re.sub(r"[^a-z0-9]+", "", table_folded) + scored: list[tuple[int, int, str]] = [] + for term in model_terms: + folded = str(term or "").casefold() + base = re.sub(r"-(?:\d+(?:\.\d+)?[BMK])$", "", term, flags=re.I) + score = 0 + for _top, words in rows: + row_text = " ".join(str(word["text"]) for word in words) + if PdfExtractTool._pdf_row_contains_term(row_text, term): + score += 100 + elif base and PdfExtractTool._pdf_row_contains_term(row_text, base): + score += 70 + if PdfExtractTool._pdf_row_contains_model_alias(row_text, term): + score += 60 + if "a22b" in folded: + row_folded = row_text.casefold() + normalized = re.sub(r"[^a-z0-9]+", "", row_folded) + if "a22b" in row_folded and ( + "qwen3" in row_folded + or "235b" in row_folded + or "qwen3vl" in normalized + ): + score += 80 + if "seed" in folded and "1.5" in folded: + if ( + "seed" in row_text.casefold() + and ("1.5-vl" in row_text.casefold() or "15vl" in re.sub(r"[^a-z0-9]+", "", row_text.casefold())) + ): + score += 80 + if "glm" in folded and "4.6" in folded: + if "glm" in row_text.casefold() and "4.6" in row_text.casefold(): + score += 80 + if "seed" in folded and "1.5" in folded: + if "seed" in table_folded and ( + "1.5-vl" in table_folded or "15vl" in table_normalized + ): + score += 90 + if "thinking" in folded and "thinking" in table_folded: + score += 25 + if "glm" in folded and "4.6" in folded: + if "glm" in table_folded and "4.6" in table_folded: + score += 90 + scored.append((score, len(term), term)) + best = max(scored) + return best[2] if best[0] > 0 else max(model_terms, key=len, default="") + + @staticmethod + def _looks_like_pdf_metric_term(token: str) -> bool: + """Identify benchmark metric labels, including compact table acronyms.""" + value = str(token or "") + folded = value.casefold() + if ( + folded.endswith("qa") + or "refcoco" in folded + or folded in { + "chartqa", + "docvqa", + "textvqa", + "ocrbench", + "countbench", + "blink", + "mmbench", + "mmstar", + "mmmu", + "mathvista", + } + ): + return True + # Papers frequently label table columns with short uppercase task + # acronyms (TR, AR, AO, AC, ...). Keep transport/document/model + # acronyms out so ordinary query prose does not become a fake header. + return ( + value.isupper() + and 2 <= len(value) <= 8 + and value not in {"PDF", "URL", "HTML", "HTTP", "HTTPS", "GPT", "LLM", "VLM"} + ) + + @staticmethod + def _pdf_query_tokens(query: str) -> list[str]: + """Tokenize table queries without dropping two-letter metric labels.""" + return re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{1,}", str(query or "")) + + @staticmethod + def _looks_like_pdf_model_term(token: str) -> bool: + """Separate compact model identifiers from ordinary hyphenated prose.""" + value = str(token or "") + if ( + any(character.isdigit() for character in value) + and any(character.isalpha() for character in value) + ): + return True + return "-" in value and sum(character.isupper() for character in value) >= 2 + + @staticmethod + def _pdf_words_contain_metric(words: list[dict], metric: str) -> bool: + """Match a metric as a table cell, not inside ordinary prose words.""" + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + target = normalize(metric) + return bool(target) and any( + normalize(word.get("text")) == target for word in words + ) + + @staticmethod + def _pdf_rows_contain_table_label( + rows: list[tuple[float, list[dict]]], table_number: int + ) -> bool: + """Return whether positioned rows contain an exact numbered caption.""" + label = re.compile( + rf"^\s*table\s+{int(table_number)}(?:\D|$)", + re.IGNORECASE, + ) + return any( + label.search(" ".join(str(word.get("text") or "") for word in words)) + for _top, words in rows + ) + + @staticmethod + def _select_positioned_table_region( + rows: list[tuple[float, list[dict]]], + metric_terms: list[str], + model_terms: list[str], + requested_table_number: int | None, + ) -> list[tuple[float, list[dict]]]: + """Keep one physical table when a page contains adjacent tables. + + Joining coordinates across two tables on the same page can map a + header from one table to values in another. Real ``Table N`` caption + rows provide a generic boundary. Prefer an explicitly requested + table, otherwise rank regions by exact requested-row and metric + coverage; named variants receive only weak fallback credit. + """ + caption_pattern = re.compile(r"^\s*table\s+(\d{1,3})(?:\D|$)", re.I) + starts: list[tuple[int, int]] = [] + for index, (_top, words) in enumerate(rows): + row_text = " ".join(str(word.get("text") or "") for word in words) + match = caption_pattern.search(row_text) + if match: + starts.append((index, int(match.group(1)))) + if len(starts) < 2: + return rows + + regions: list[tuple[int, list[tuple[float, list[dict]]]]] = [] + for position, (start, number) in enumerate(starts): + end = starts[position + 1][0] if position + 1 < len(starts) else len(rows) + regions.append((number, rows[start:end])) + if requested_table_number is not None: + requested = [region for number, region in regions if number == requested_table_number] + if requested: + return requested[0] + + def _score(region: list[tuple[float, list[dict]]]) -> tuple[int, int]: + row_texts = [ + " ".join(str(word.get("text") or "") for word in words) + for _top, words in region + ] + exact_models = sum( + any(PdfExtractTool._pdf_row_exact_term_match(text, term) for text in row_texts) + for term in set(model_terms) + ) + broad_models = sum( + any(PdfExtractTool._pdf_row_contains_term(text, term) for text in row_texts) + for term in set(model_terms) + ) + metrics = sum( + any(PdfExtractTool._pdf_words_contain_metric(words, term) for _top, words in region) + for term in set(metric_terms) + ) + numeric = sum( + len(re.findall(r"\b\d+(?:\.\d+)?\b", text)) for text in row_texts + ) + return ( + exact_models * 1000 + broad_models * 100 + metrics * 300 + min(numeric, 99), + -len(region), + ) + + return max((region for _number, region in regions), key=_score) + + @staticmethod + def _positioned_table_evidence(url: str, query: str) -> str: + """Return compact PDF rows with x coordinates for column-safe reading.""" + try: + import pdfplumber + import httpx + from services.search.content import _PinnedTransport, _resolve_public_ips + from src.constants import WEB_FETCH_HARD_MAX_BYTES, WEB_FETCH_USER_AGENT + except ImportError: + return "" + local_path = PdfExtractTool._local_pdf_path(url) + if local_path is not None: + if not local_path.is_file(): + return "" + pdf_bytes = local_path.read_bytes() + declared_size = len(pdf_bytes) + else: + ips = _resolve_public_ips(url) + headers = { + "User-Agent": WEB_FETCH_USER_AGENT, + "Accept": "application/pdf,*/*", + } + with httpx.Client( + headers=headers, + timeout=30, + follow_redirects=True, + transport=_PinnedTransport(ips[0]), + ) as client: + response = client.get(url) + response.raise_for_status() + pdf_bytes = response.content + declared = response.headers.get("content-length") + declared_size = int(declared) if declared and declared.isdigit() else 0 + if declared_size and declared_size > PDF_EXTRACT_MAX_BYTES: + return ( + f"[PDF too large for pdf_extract positioned table reader: " + f"{declared_size:,} bytes > {PDF_EXTRACT_MAX_BYTES:,} bytes]" + ) + if len(pdf_bytes) > PDF_EXTRACT_MAX_BYTES: + return ( + f"[PDF too large for pdf_extract positioned table reader: " + f"{len(pdf_bytes):,} bytes > {PDF_EXTRACT_MAX_BYTES:,} bytes]" + ) + tokens = PdfExtractTool._pdf_query_tokens(query) + requested_table_match = re.search( + r"\btable\s+(\d{1,3})\b", str(query or ""), re.IGNORECASE + ) + requested_table_number = ( + int(requested_table_match.group(1)) if requested_table_match else None + ) + metric_terms = [ + token for token in tokens + if PdfExtractTool._looks_like_pdf_metric_term(token) + ] + model_terms = [ + token for token in tokens + if token not in metric_terms + and PdfExtractTool._looks_like_pdf_model_term(token) + ] + normalize = lambda value: re.sub(r"[^a-z0-9]+", "", value.casefold()) + + def _row_contains_model_alias(row_text: str, term: str) -> bool: + row_folded = str(row_text or "").casefold() + term_folded = str(term or "").casefold() + if PdfExtractTool._pdf_row_contains_model_alias(row_text, term): + return True + if "a22b" in term_folded: + return "a22b" in row_folded and ( + "qwen3" in row_folded + or "235b" in row_folded + or "qwen3vl" in normalize(row_folded) + ) + return False + + candidates: list[tuple[int, int, list[tuple[float, list[dict]]]]] = [] + with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: + for page_number, page in enumerate(pdf.pages, 1): + grouped: dict[float, list[dict]] = {} + for word in page.extract_words(x_tolerance=2, y_tolerance=2): + grouped.setdefault(round(float(word["top"]) / 3) * 3, []).append(word) + rows = [(top, sorted(words, key=lambda item: float(item["x0"]))) for top, words in grouped.items()] + anchors: list[float] = [] + score = 0 + for top, words in rows: + row_text = " ".join(str(word["text"]) for word in words) + low = row_text.casefold() + metric_hits = sum( + PdfExtractTool._pdf_words_contain_metric(words, term) + for term in metric_terms + ) + model_hits = sum( + PdfExtractTool._pdf_row_exact_term_match(row_text, term) + for term in model_terms + ) + alias_model_hits = sum( + _row_contains_model_alias(row_text, term) + for term in model_terms + ) + numeric_count = len(re.findall(r"\b\d+(?:\.\d+)?\b", row_text)) + starts_with_metric = bool(words) and any( + re.sub(r"[^a-z0-9]+", "", str(words[0]["text"]).casefold()) + == re.sub(r"[^a-z0-9]+", "", term.casefold()) + for term in metric_terms + ) + table_header_hit = ( + "capability" in low + and "benchmark" in low + ) + model_table_header_hit = bool( + alias_model_hits + and ( + low.strip().startswith("table ") + or low.strip().startswith("benchmark") + ) + ) + if metric_hits >= 2 or (metric_hits and numeric_count >= 3) or (model_hits and numeric_count >= 3): + anchors.append(top) + score += ( + metric_hits * 10 + + model_hits * 5 + + alias_model_hits * 20 + + min(numeric_count, 10) + + (50 if starts_with_metric else 0) + ) + elif model_table_header_hit: + score += alias_model_hits * 40 + if table_header_hit: + score += 15 + if anchors: + # Preserve the continuous table body between matching + # anchors. Selecting disjoint +/-100 bands dropped middle + # model rows in tall benchmark tables when the queried + # model name was split across PDF words and therefore did + # not become its own anchor. + table_top = max(0.0, min(anchors) - 100) + header_tops = [ + top for top, words in rows + if top <= min(anchors) + and ( + ( + "capability" in " ".join(str(word["text"]) for word in words).casefold() + and "benchmark" in " ".join(str(word["text"]) for word in words).casefold() + ) + or ( + "benchmark" in " ".join(str(word["text"]) for word in words).casefold() + and any( + _row_contains_model_alias( + " ".join(str(word["text"]) for word in words), + term, + ) + for term in model_terms + ) + ) + or ( + " ".join(str(word["text"]) for word in words).casefold().strip().startswith("table ") + and any( + _row_contains_model_alias( + " ".join(str(word["text"]) for word in words), + term, + ) + for term in model_terms + ) + ) + ) + ] + if header_tops: + table_top = min(table_top, max(header_tops) - 20) + table_bottom = max(anchors) + 100 + selected = [ + row for row in rows + if table_top <= row[0] <= table_bottom + ] + candidates.append((score, page_number, selected)) + if not candidates: + return "" + labeled_candidates = ( + [ + candidate for candidate in candidates + if PdfExtractTool._pdf_rows_contain_table_label( + candidate[2], requested_table_number + ) + ] + if requested_table_number is not None + else [] + ) + if labeled_candidates: + # A paper can repeat a table number in an appendix. The first + # exact caption is the canonical table for an unqualified request. + _score, page_number, rows = min( + labeled_candidates, key=lambda item: item[1] + ) + else: + _score, page_number, rows = max(candidates, key=lambda item: item[0]) + rows = PdfExtractTool._select_positioned_table_region( + rows, + metric_terms, + model_terms, + requested_table_number, + ) + resolved: dict[str, str] = {} + metric_positions: dict[str, float] = {} + header_candidates = [] + for _top, words in rows: + positions = { + metric: float(word["x0"]) + for word in words for metric in metric_terms + if re.sub(r"[^a-z0-9]+", "", str(word["text"]).casefold()) + == re.sub(r"[^a-z0-9]+", "", metric.casefold()) + } + if positions: + header_candidates.append((len(positions), positions)) + if header_candidates: + metric_positions = max(header_candidates, key=lambda item: item[0])[1] + + def _numbers(words: list[dict]) -> list[tuple[float, str]]: + return [ + (float(word["x0"]), str(word["text"])) + for word in words + if re.fullmatch(r"\d+(?:\.\d+)?", str(word["text"])) + ] + + # Row-oriented tables: metric names are columns and the requested + # model is a row (for example DeepSeek-VL2). + target_model = PdfExtractTool._select_positioned_target_model( + model_terms, + rows, + ) + target_base = re.sub(r"-(?:\d+(?:\.\d+)?[BMK])$", "", target_model, flags=re.I) + exact_model_rows = [ + (top, words) + for top, words in rows + if target_base + and PdfExtractTool._pdf_row_exact_term_match( + " ".join(str(word["text"]) for word in words), + target_base, + ) + ] + candidate_model_rows = exact_model_rows or [ + (top, words) + for top, words in rows + if target_base + and PdfExtractTool._pdf_row_contains_term( + " ".join(str(word["text"]) for word in words), + target_base, + ) + ] + for _top, words in candidate_model_rows: + row_text = " ".join(str(word["text"]) for word in words) + nums = _numbers(words) + if len(nums) >= len(metric_terms): + for metric, metric_x in metric_positions.items(): + nearest = min(nums, key=lambda item: abs(item[0] - metric_x)) + if abs(nearest[0] - metric_x) <= 30: + resolved[metric] = nearest[1] + if resolved: + break + + # A single row-oriented table can contain several models requested in + # the same query. Return a coordinate-joined record for each exact row + # instead of forcing the model to decode the remaining dense HTML row. + additional_row_resolutions: dict[str, dict[str, str]] = {} + for requested_model in dict.fromkeys(model_terms): + if requested_model == target_model: + continue + requested_base = re.sub( + r"-(?:\d+(?:\.\d+)?[BMK])$", "", requested_model, flags=re.I + ) + requested_rows = [ + words + for _top, words in rows + if requested_base + and PdfExtractTool._pdf_row_exact_term_match( + " ".join(str(word["text"]) for word in words), + requested_base, + ) + ] + for words in requested_rows: + nums = _numbers(words) + if len(nums) < len(metric_terms): + continue + values: dict[str, str] = {} + for metric, metric_x in metric_positions.items(): + nearest = min(nums, key=lambda item: abs(item[0] - metric_x)) + if abs(nearest[0] - metric_x) <= 30: + values[metric] = nearest[1] + if values: + additional_row_resolutions[requested_model] = values + break + + # Column-oriented tables: models are columns and metrics are rows (for + # example Qwen2.5-VL variants). Resolve a split size suffix such as + # "72B" to its x coordinate, then join each metric row at that x. + if len(resolved) < len(metric_terms) and target_model: + size_match = re.search(r"(\d+(?:\.\d+)?[BMK])$", target_model, re.I) + target_x = None + if size_match: + suffix = size_match.group(1).casefold() + base_hits = [ + (top, float(word["x0"])) + for top, words in rows for word in words + if target_base and str(word["text"]).casefold() == target_base.casefold() + ] + suffix_hits = [ + (top, float(word["x0"])) + for top, words in rows for word in words + if str(word["text"]).casefold() == suffix + ] + pairs = [ + (abs(base_top - suffix_top) + abs(base_x - suffix_x), suffix_x) + for base_top, base_x in base_hits for suffix_top, suffix_x in suffix_hits + if abs(base_top - suffix_top) <= 15 and abs(base_x - suffix_x) <= 30 + ] + if pairs: + target_x = min(pairs)[1] + if target_x is None: + normalized_target = normalize(target_model) + metric_tops = [ + top for top, words in rows + if any( + PdfExtractTool._pdf_words_contain_metric(words, metric) + for metric in metric_terms + ) + ] + first_metric_top = min(metric_tops) if metric_tops else float("inf") + column_fragments: dict[int, list[tuple[float, str]]] = {} + for top, words in rows: + if top >= first_metric_top: + continue + for word in words: + text = str(word["text"]) + if not re.search(r"[A-Za-z]", text): + continue + x_key = round(float(word["x0"])) + column_fragments.setdefault(x_key, []).append((top, text)) + matches: list[tuple[int, int]] = [] + for x_key, fragments in column_fragments.items(): + combined = " ".join(text for _top, text in sorted(fragments)) + normalized_combined = normalize(combined) + if ( + normalized_combined + and normalized_target + and ( + normalized_target in normalized_combined + or normalized_combined in normalized_target + ) + ): + matches.append((len(normalized_combined), x_key)) + if matches: + target_x = float(max(matches)[1]) + if target_x is None and target_model: + target_folded = target_model.casefold() + mode_terms = [] + if "instruct" in target_folded: + mode_terms.extend(["instruct", "non-thinking"]) + if "thinking" in target_folded and "instruct" not in target_folded: + mode_terms.append("thinking") + if mode_terms: + metric_tops = [ + top for top, words in rows + if any( + PdfExtractTool._pdf_words_contain_metric(words, metric) + for metric in metric_terms + ) + ] + first_metric_top = min(metric_tops) if metric_tops else float("inf") + mode_hits = [ + float(word["x0"]) + for top, words in rows + if top < first_metric_top + for word in words + if str(word["text"]).casefold() in mode_terms + ] + if mode_hits: + target_x = mode_hits[0] + if target_x is not None: + for metric in metric_terms: + for row_index, (_top, words) in enumerate(rows): + if not PdfExtractTool._pdf_words_contain_metric(words, metric): + continue + nums = _numbers(words) + if not nums: + adjacent_numeric_rows = [ + ( + abs(float(rows[candidate_index][0]) - float(_top)), + rows[candidate_index][1], + ) + for candidate_index in (row_index - 1, row_index + 1) + if 0 <= candidate_index < len(rows) + and abs(float(rows[candidate_index][0]) - float(_top)) <= 9 + and _numbers(rows[candidate_index][1]) + ] + if adjacent_numeric_rows: + nums = _numbers( + min(adjacent_numeric_rows, key=lambda item: item[0])[1] + ) + if nums: + nearest = min(nums, key=lambda item: abs(item[0] - target_x)) + if abs(nearest[0] - target_x) <= 30: + resolved[metric] = nearest[1] + break + + rendered = [] + for top, words in sorted(rows, key=lambda item: item[0]): + cells = " | ".join( + f"x={float(word['x0']):.0f}:{word['text']}" for word in words + ) + rendered.append(f"y={top:.0f} :: {cells}") + body = "\n".join(rendered) + if len(body) > MAX_OUTPUT_CHARS - 1000: + body = body[:MAX_OUTPUT_CHARS - 1000] + "\n[...positioned rows truncated]" + resolved_text = "" + all_resolutions: list[tuple[str, dict[str, str]]] = [] + if resolved: + all_resolutions.append((target_model or "requested model", resolved)) + all_resolutions.extend(additional_row_resolutions.items()) + resolved_sections: list[str] = [] + for resolved_model, resolved_values in all_resolutions: + missing_metrics = [ + metric for metric in metric_terms if metric not in resolved_values + ] + missing_note = ( + " | requested metrics not found: " + ", ".join(missing_metrics) + if missing_metrics else "" + ) + lock_values = { + metric: resolved_values[metric] + for metric in metric_terms + if metric in resolved_values + } + for metric in missing_metrics: + lock_values[metric] = None + resolved_sections.append( + "Resolved requested values by coordinate join: " + + resolved_model + + " | " + + " | ".join( + f"{metric}={resolved_values[metric]}" + for metric in metric_terms + if metric in resolved_values + ) + + missing_note + + "\n" + + "Resolved values JSON: " + + json.dumps({ + "model": resolved_model, + "values": lock_values, + "source": "pdf_extract_positioned_table", + "page": page_number, + }, sort_keys=True) + ) + if resolved_sections: + resolved_text = "\n".join(resolved_sections) + "\n" + return ( + f"[Positioned PDF table evidence, page {page_number}. Values sharing the same " + "x coordinate belong to the same column; map the requested model header x to " + "metric-row values at that x.]\n" + resolved_text + body + ) + + @staticmethod + def _local_text_evidence(path: Path, query: str) -> str: + """Return focused text pages when a local PDF has no positioned rows.""" + try: + import pdfplumber + except ImportError: + return "" + tokens = { + token.casefold() + for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{2,}", query) + } + requested_labels = [ + re.sub(r"\s+", "", label.casefold()) + for label in re.findall(r"\b(?:table|figure|fig\.?)\s*\d+\b", query, re.I) + ] + pages: list[tuple[int, int, str]] = [] + page_columns: list[tuple[int, str, str]] = [] + try: + with pdfplumber.open(path) as pdf: + for page_number, page in enumerate(pdf.pages, 1): + text = str(page.extract_text(layout=True) or "").strip() + if not text: + continue + midpoint = float(page.width) / 2 + left = str( + page.crop((0, 0, midpoint, page.height)).extract_text() or "" + ).strip() + right = str( + page.crop((midpoint, 0, page.width, page.height)).extract_text() or "" + ).strip() + page_columns.append((page_number, left, right)) + folded = text.casefold() + compact = re.sub(r"\s+", "", folded) + label_hits = sum(label in compact for label in requested_labels) + numeric_count = len(re.findall(r"\b\d+(?:\.\d+)?\b", text)) + table_signal = bool( + re.search(r"\b(?:table|figure|fig\.?)\s*\d+\b", folded, re.I) + ) + result_table_signal = bool( + re.search( + r"(?:table\s*3|evaluation\s+results\s+on\s+groundingme|" + r"main\s+results|main\s+experimental\s+results)", + folded, + re.I, + ) + ) + appendix_table_signal = bool( + re.search( + r"(?:supplementary\s+material|detailed\s+subtask|" + r"table\s*[67])", + folded, + re.I, + ) + ) + requested_result_terms = { + "appendix", "average", "baseline", "experimental", + "leaderboard", "results", "score", "scores", "thinking", + } + result_table_boost = ( + 260 + if result_table_signal + and tokens & requested_result_terms + else 0 + ) + appendix_table_boost = ( + 220 + if appendix_table_signal + and ("appendix" in tokens or "thinking" in tokens) + else 0 + ) + score = ( + sum(token in folded for token in tokens) + + label_hits * 100 + + (8 if table_signal else 0) + + min(numeric_count, 20) // 4 + + result_table_boost + + appendix_table_boost + ) + pages.append((score, page_number, text)) + except Exception: + return "" + bibliography = PdfExtractTool._bibliography_evidence(page_columns, query) + if bibliography: + return bibliography + if not pages: + return "" + selected = sorted( + pages, key=lambda item: (item[0], -item[1]), reverse=True + )[:4] + # Put the most relevant pages first so the bounded output cap cannot + # truncate the requested table behind introductory pages. Page labels + # remain in each block, so callers can still cite the source page. + selected.sort(key=lambda item: (item[0], -item[1]), reverse=True) + body = "\n\n".join( + f"[Local PDF text evidence, page {page_number}]\n{text}" + for _score, page_number, text in selected + ) + if len(body) > MAX_OUTPUT_CHARS - 500: + body = body[:MAX_OUTPUT_CHARS - 500] + "\n[...local PDF text truncated]" + return body + + async def execute(self, content: str, ctx: dict) -> dict: + raw = content.strip() + try: + args = json.loads(raw) if raw.startswith("{") else {} + except json.JSONDecodeError: + args = {} + if not isinstance(args, dict): + args = {} + if not args: + lines = [line.strip() for line in raw.splitlines() if line.strip()] + args = { + "url": lines[0] if lines else "", + "query": ", ".join(lines[1:]), + } + url = str(args.get("url") or args.get("path") or "").strip() + query = str(args.get("query") or "").strip() + if not url: + return {"error": "pdf_extract: provide a PDF URL or local PDF path", "exit_code": 1} + if not query: + return { + "error": ( + "pdf_extract: provide query terms naming the target model, " + "metrics, or table so the returned evidence is precise" + ), + "exit_code": 1, + } + loop = asyncio.get_running_loop() + local_path = self._local_pdf_path(url) + if local_path is not None: + from src.tool_execution import _display_tool_path + + url = _display_tool_path(str(local_path)) + + async def _positioned_evidence() -> str: + try: + return await asyncio.wait_for( + loop.run_in_executor(None, self._positioned_table_evidence, url, query), + timeout=30, + ) + except Exception: + return "" + + # arXiv's PDF CDN can temporarily serve an older revision than its + # current HTML conversion. The HTML table is also substantially more + # legible than positioned PDF glyphs. Fetch both official forms in + # parallel and return complementary evidence, so a newly-added model + # row cannot disappear and trigger browser/shell retry loops. + html_url = self._arxiv_html_url(url) if local_path is None else "" + positioned_task = _positioned_evidence() + if html_url: + query_tokens = re.findall(r"[A-Za-z0-9][A-Za-z0-9_.+-]{1,}", query) + query_models = sorted( + { + token for token in query_tokens + if not self._looks_like_pdf_metric_term(token) + and self._looks_like_pdf_model_term(token) + }, + key=len, + reverse=True, + ) + # A broad query with two model names often locks onto the first of + # several identically numbered tables (for example dev Table 2 + # instead of test Table 2). Query each requested model separately; + # the rarer/longer row is emitted first and anchors the right table. + if len(query_models) > 1: + # Exact model-only focus keeps the matching row inside the + # passage budget. Repeating every metric acronym can rank the + # table introduction above a late model row and truncate the + # very values the caller requested. + html_queries = query_models + else: + html_queries = [query] + html_tasks = [ + WebFetchTool().execute( + json.dumps({"url": html_url, "query": focused, "full": True}), ctx + ) + for focused in html_queries + ] + gathered = await asyncio.gather(positioned_task, *html_tasks) + positioned = gathered[0] + html_outputs: list[str] = [] + for focused, html_result in zip(html_queries, gathered[1:]): + if html_result.get("exit_code") != 0: + continue + html_output = str(html_result.get("output") or "") + if html_output and html_output not in html_outputs: + html_outputs.append( + f"[Focused HTML evidence for: {focused}]\n{html_output}" + ) + evidence = [] + # Structured coordinate evidence is compact and column-safe. Put + # it first so a long prose extraction cannot consume the bounded + # response and truncate the exact table rows. + if positioned: + evidence.append(positioned) + if html_outputs: + evidence.append( + f"[Official arXiv HTML companion: {html_url}]\n" + + "\n\n--- additional requested model row ---\n\n".join(html_outputs) + ) + if evidence: + output = ( + f"Source: {url}\n\n" + "[Extraction guidance: align every value to the complete table header. " + "A row may contain extra unrequested generation metrics (for example VS or SSC); " + "skip those columns rather than treating the next contiguous number as the requested metric.]\n\n" + + "\n\n--- PDF/HTML corroboration ---\n\n".join(evidence) + ) + if len(output) > MAX_OUTPUT_CHARS: + output = output[:MAX_OUTPUT_CHARS] + "\n[...combined evidence truncated]" + return {"output": output, "exit_code": 0} + else: + positioned = await positioned_task + if positioned: + return {"output": f"Source: {url}\n\n{positioned}", "exit_code": 0} + if local_path is not None: + text_evidence = await asyncio.wait_for( + loop.run_in_executor( + None, self._local_text_evidence, local_path, query + ), + timeout=30, + ) + if text_evidence: + return { + "output": f"Source: {url}\n\n{text_evidence}", + "exit_code": 0, + } + return { + "error": ( + "pdf_extract: no selectable table/text evidence was found in the local PDF; " + "use inspect_media for scanned or visual pages" + ), + "exit_code": 1, + } + fetch_args = {"url": url, "query": query, "full": True} + result = await WebFetchTool().execute(json.dumps(fetch_args), ctx) + if result.get("exit_code") == 0: + result["output"] = ( + "[PDF extraction; the highest-ranked matching table window is below. " + "Align row values to the full column header, use only the requested model's " + "column, and do not re-query when the target row and header are present.]\n\n" + + str(result.get("output") or "") + ) + return result + + +class YouTubeTool: + """YouTube-specific read tool for videos, transcripts, comments, channels.""" + + _ACTIONS = {"comments", "transcript", "metadata", "latest_channel_video"} + + async def execute(self, content: str, ctx: dict) -> dict: + from services.youtube.youtube_handler import ( + extract_transcript_async, + fetch_youtube_comments, + init_youtube, + ) + + args, err = self._parse_args(content) + if err: + return {"error": err, "exit_code": 1} + action = str(args.get("action") or "metadata").strip().lower() + if action not in self._ACTIONS: + return { + "error": "youtube_tool: action must be one of " + + ", ".join(sorted(self._ACTIONS)), + "exit_code": 1, + } + + max_results = args.get("max_results") + if not isinstance(max_results, int) or max_results <= 0: + max_results = 20 + max_results = max(1, min(50, max_results)) + + progress_cb = ctx.get("progress_cb") if isinstance(ctx, dict) else None + if progress_cb: + await progress_cb({"elapsed_s": 0, "tail": f"youtube_tool: {action}"}) + + if action == "latest_channel_video": + channel = str(args.get("channel_url") or args.get("url") or args.get("handle") or "").strip() + if not channel: + return {"error": "youtube_tool latest_channel_video: provide channel_url or handle", "exit_code": 1} + return await self._latest_channel_video(channel, max_results=max_results) + + if action == "metadata" and not any(args.get(k) for k in ("url", "video_url", "video_id")): + channel = str(args.get("channel_url") or args.get("handle") or "").strip() + if channel: + return await self._latest_channel_video(channel, max_results=max_results) + + url_or_id = str(args.get("url") or args.get("video_url") or args.get("video_id") or "").strip() + # The shared extractor accepts ID prefixes inside text. At the tool + # boundary require the complete target, not a truncated invented ID. + if url_or_id.startswith(('http://', 'https://')): + parsed = urllib.parse.urlparse(url_or_id) + host = (parsed.hostname or '').lower() + if host in {'youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com'}: + parts = parsed.path.strip('/').split('/') + candidate = (urllib.parse.parse_qs(parsed.query).get('v', [''])[0] + if parsed.path == '/watch' else + parts[1] if len(parts) == 2 and parts[0] in {'shorts', 'embed', 'live'} else '') + elif host in {'youtu.be', 'www.youtu.be'}: + candidate = parsed.path.strip('/') + else: + candidate = '' + else: + candidate = url_or_id + if (not re.fullmatch(r'[A-Za-z0-9_-]{11}', candidate) + or (args.get('video_id') and str(args['video_id']) != candidate)): + return {"error": f"youtube_tool {action}: invalid video target. Resolve the actual video URL " + "from the user or an observed link. A title or channel page is not a video ID; " + "open the referenced video in the browser or use latest_channel_video first.", + "exit_code": 1, "failure_kind": "invalid_target"} + video_id = candidate + url = url_or_id if url_or_id.startswith(("http://", "https://")) else f"https://www.youtube.com/watch?v={video_id}" + + if action == "comments": + api_result = await self._comments_from_data_api(video_id, max_results) + if api_result.get("success"): + return {"output": self._format_comments(api_result, url), "exit_code": 0, "untrusted_content": True} + comments_data = await fetch_youtube_comments(video_id, max_comments=max_results, timeout=45) + if not comments_data.get("success"): + api_error = api_result.get("error") + fallback_error = comments_data.get("error") or "unknown error" + joined = f"{fallback_error}" + if api_error: + joined = f"YouTube Data API unavailable: {api_error}; yt-dlp fallback failed: {fallback_error}" + return {"error": f"youtube_tool comments: {joined}", "exit_code": 1, + "failure_kind": "comments_unavailable", "video_url": url, + "untrusted_content": True} + return {"output": self._format_comments(comments_data, url), "exit_code": 0, "untrusted_content": True} + + if action == "transcript": + init_youtube() + transcript_data = await extract_transcript_async(url, video_id) + if not transcript_data.get("success"): + return { + "error": f"youtube_tool transcript: {transcript_data.get('error') or 'transcript unavailable'}", + "exit_code": 1, + "untrusted_content": True, + } + return {"output": self._format_transcript(transcript_data, url), "exit_code": 0, "untrusted_content": True} + + return await self._metadata(url) + + def _parse_args(self, content: str) -> tuple[dict, str | None]: + raw = (content or "").strip() + if not raw: + return {}, "youtube_tool: provide JSON with action and url/video_id" + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {"action": "metadata", "url": raw}, None + if not isinstance(parsed, dict): + return {}, "youtube_tool: arguments must be a JSON object" + return parsed, None + + async def _comments_from_data_api(self, video_id: str, max_results: int) -> dict: + api_key = os.getenv("YOUTUBE_API_KEY", "").strip() + if not api_key: + return {"success": False, "error": "YOUTUBE_API_KEY not set", "comments": []} + params = urllib.parse.urlencode({ + "part": "snippet", + "videoId": video_id, + "maxResults": str(max_results), + "order": "relevance", + "textFormat": "plainText", + "key": api_key, + }) + url = f"https://www.googleapis.com/youtube/v3/commentThreads?{params}" + + def _fetch() -> dict: + req = urllib.request.Request(url, headers={"User-Agent": "odysseus-ui/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + return json.loads(resp.read().decode("utf-8", errors="replace")) + + try: + data = await asyncio.wait_for(asyncio.to_thread(_fetch), timeout=20) + except Exception as e: + return {"success": False, "error": f"{type(e).__name__}: {e}", "comments": []} + + comments = [] + for item in data.get("items") or []: + snippet = ( + (item.get("snippet") or {}) + .get("topLevelComment", {}) + .get("snippet", {}) + ) + text = html.unescape(str(snippet.get("textDisplay") or snippet.get("textOriginal") or "")).strip() + if not text: + continue + comments.append({ + "author": snippet.get("authorDisplayName") or "Unknown", + "text": text, + "likes": snippet.get("likeCount") or 0, + }) + return {"success": True, "comments": comments, "count": len(comments), "source": "youtube_data_api"} + + def _format_comments(self, data: dict, url: str) -> str: + comments = [c for c in (data.get("comments") or []) if isinstance(c, dict)] + if not comments: + return f"YouTube comments for {url}: no comments returned." + lines = [ + f"YouTube comments for {url}", + f"Source: {data.get('source') or 'yt-dlp'}", + f"Count: {len(comments)}", + ] + title = str(data.get("title") or "").strip() + channel = str(data.get("channel") or "").strip() + if title: + lines.append(f"Title: {title}") + if channel: + lines.append(f"Channel: {channel}") + lines.append("") + for idx, comment in enumerate(comments, 1): + likes = comment.get("likes") or 0 + like_text = f" [{likes} likes]" if likes else "" + author = str(comment.get("author") or "Unknown").strip().lstrip("@") + text = re.sub(r"\s+", " ", str(comment.get("text") or "")).strip() + lines.append(f"{idx}. @{author}{like_text}: {text}") + output = "\n".join(lines) + return output[:MAX_OUTPUT_CHARS] + ("\n\n[...truncated]" if len(output) > MAX_OUTPUT_CHARS else "") + + def _format_transcript(self, data: dict, url: str) -> str: + lines = [ + f"YouTube transcript for {url}", + f"Video ID: {data.get('video_id') or ''}", + f"Language: {data.get('language') or 'unknown'}", + "", + ] + segments = data.get("segments") or [] + if segments: + for seg in segments: + if not isinstance(seg, dict): + continue + lines.append(f"[{seg.get('timestamp') or '??:??'}] {seg.get('text') or ''}") + else: + lines.append(str(data.get("transcript") or "")) + output = "\n".join(lines) + return output[:MAX_OUTPUT_CHARS] + ("\n\n[...truncated]" if len(output) > MAX_OUTPUT_CHARS else "") + + async def _metadata(self, url: str) -> dict: + result = await self._ytdlp_json(url, timeout=30) + if not result.get("success"): + return {"error": f"youtube_tool metadata: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + data = result["data"] + lines = [ + f"Title: {data.get('title') or ''}", + f"Channel: {data.get('channel') or data.get('uploader') or ''}", + f"URL: {data.get('webpage_url') or url}", + f"Duration: {data.get('duration_string') or data.get('duration') or ''}", + f"View count: {data.get('view_count') or ''}", + f"Like count: {data.get('like_count') or ''}", + f"Upload date: {data.get('upload_date') or ''}", + ] + return {"output": "\n".join(lines), "exit_code": 0, "untrusted_content": True} + + async def _latest_channel_video(self, channel: str, *, max_results: int = 5) -> dict: + original_channel = channel + if channel.startswith("@"): + channel = f"https://www.youtube.com/{channel}/videos" + elif channel.startswith(("http://", "https://")): + if "/videos" not in urllib.parse.urlparse(channel).path: + channel = channel.rstrip("/") + "/videos" + else: + channel = f"https://www.youtube.com/{channel.strip('/')}/videos" + result = await self._ytdlp_json(channel, timeout=45, flat_playlist=True, playlist_end=max_results) + if not result.get("success"): + resolved_channel = await self._resolve_channel_from_search(original_channel) + if resolved_channel: + retry = await self._ytdlp_json( + resolved_channel, + timeout=45, + flat_playlist=True, + playlist_end=max_results, + ) + if retry.get("success"): + channel = resolved_channel + result = retry + else: + return { + "error": ( + "youtube_tool latest_channel_video: " + f"{result.get('error')}; resolved {resolved_channel} also failed: {retry.get('error')}" + ), + "exit_code": 1, + "untrusted_content": True, + } + else: + return {"error": f"youtube_tool latest_channel_video: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + if not result.get("success"): + return {"error": f"youtube_tool latest_channel_video: {result.get('error')}", "exit_code": 1, "untrusted_content": True} + data = result["data"] + entries = [e for e in (data.get("entries") or []) if isinstance(e, dict)] + if not entries: + return {"error": "youtube_tool latest_channel_video: no videos found", "exit_code": 1, "untrusted_content": True} + entries = entries[:max_results] + heading = "Latest channel video" if len(entries) == 1 else f"Latest {len(entries)} channel videos" + lines = [f"{heading} for {channel}", ""] + for idx, entry in enumerate(entries, 1): + video_id = entry.get("id") or "" + video_url = entry.get("url") or entry.get("webpage_url") or "" + if video_id and not str(video_url).startswith("http"): + video_url = f"https://www.youtube.com/watch?v={video_id}" + lines.extend([ + f"{idx}. {entry.get('title') or ''}", + f" URL: {video_url}", + f" Duration: {entry.get('duration_string') or entry.get('duration') or ''}", + f" Video ID: {video_id}", + ]) + return {"output": "\n".join(lines), "exit_code": 0, "untrusted_content": True} + + async def _resolve_channel_from_search(self, channel: str) -> str: + query = self._channel_search_query(channel) + if not query: + return "" + result = await self._ytdlp_json( + f"ytsearch10:{query} official YouTube channel", + timeout=30, + flat_playlist=True, + playlist_end=10, + ) + if not result.get("success"): + return "" + entries = [e for e in (result.get("data") or {}).get("entries") or [] if isinstance(e, dict)] + if not entries: + return "" + target = self._normalize_channel_name(query) + + def _score(entry: dict) -> tuple[int, int]: + channel_name = self._normalize_channel_name(entry.get("channel") or entry.get("uploader") or "") + uploader_id = self._normalize_channel_name(entry.get("uploader_id") or "") + title = self._normalize_channel_name(entry.get("title") or "") + score = 0 + if channel_name == target: + score += 100 + elif target and target in channel_name: + score += 60 + if uploader_id == target: + score += 45 + elif target and target in uploader_id: + score += 25 + if target and target in title: + score += 10 + if re.search(r"\b(?:clips?|shorts?|two|second|fan|archive)\b", channel_name): + score -= 35 + if re.search(r"\b(?:clips?|shorts?|compilation|reacts?)\b", title): + score -= 10 + return score, int(entry.get("view_count") or 0) + + best = max(entries, key=_score) + if _score(best)[0] <= 0: + return "" + channel_url = str(best.get("channel_url") or "").strip() + uploader_url = str(best.get("uploader_url") or "").strip() + resolved = channel_url or uploader_url + if not resolved: + uploader_id = str(best.get("uploader_id") or "").strip() + if uploader_id.startswith("@"): + resolved = f"https://www.youtube.com/{uploader_id}" + if not resolved: + return "" + parsed = urllib.parse.urlparse(resolved) + if parsed.netloc and "/videos" not in parsed.path: + resolved = resolved.rstrip("/") + "/videos" + return resolved + + @staticmethod + def _channel_search_query(channel: str) -> str: + value = str(channel or "").strip() + if not value: + return "" + if value.startswith(("http://", "https://")): + path = urllib.parse.urlparse(value).path.strip("/") + parts = [part for part in path.split("/") if part and part.lower() != "videos"] + value = parts[-1] if parts else value + value = value.strip().lstrip("@").strip("/") + value = re.sub(r"(?i)^(?:c|channel|user)/", "", value) + value = re.sub(r"[_-]+", " ", value) + return re.sub(r"\s+", " ", value).strip() + + @staticmethod + def _normalize_channel_name(value: str) -> str: + value = str(value or "").lower().lstrip("@") + value = re.sub(r"[^a-z0-9]+", "", value) + return value + + async def _ytdlp_json(self, url: str, *, timeout: int, flat_playlist: bool = False, playlist_end: int = 5) -> dict: + binary = shutil.which("yt-dlp") + if not binary: + venv_binary = os.path.join(os.path.dirname(sys.executable), "yt-dlp") + binary = venv_binary if os.path.exists(venv_binary) else "" + if not binary: + return {"success": False, "error": "yt-dlp not installed"} + cmd = [binary, "--skip-download", "--no-warnings", "--js-runtimes", "node"] + if flat_playlist: + playlist_end = max(1, min(50, int(playlist_end or 5))) + cmd.extend(["--flat-playlist", "--playlist-end", str(playlist_end), "--dump-single-json"]) + else: + cmd.append("--dump-json") + cmd.append(url) + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + with contextlib.suppress(Exception): + proc.kill() + await proc.wait() + return {"success": False, "error": f"yt-dlp timed out after {timeout}s"} + except Exception as e: + return {"success": False, "error": f"{type(e).__name__}: {e}"} + if proc.returncode != 0: + return {"success": False, "error": stderr.decode("utf-8", errors="replace")[:300]} + try: + data = self._parse_ytdlp_json_output(stdout.decode("utf-8", errors="replace")) + except json.JSONDecodeError as e: + return {"success": False, "error": f"invalid yt-dlp JSON: {e}"} + return {"success": True, "data": data} + + @staticmethod + def _parse_ytdlp_json_output(output: str) -> dict: + """Parse yt-dlp JSON from either a single object or JSON-lines output.""" + text = (output or "").strip() + try: + data = json.loads(text) + except json.JSONDecodeError: + entries = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + parsed = json.loads(line) + if isinstance(parsed, dict): + entries.append(parsed) + return {"entries": entries} + if isinstance(data, dict): + return data + if isinstance(data, list): + return {"entries": [item for item in data if isinstance(item, dict)]} + return {"entries": []} + + +class PrivateBrowserTool: + """Small deterministic wrapper around Vercel's agent-browser CLI. + + This is intentionally narrower than handing the model the raw browser MCP + schema. Use web_search/web_fetch first; this exists for JS-rendered pages, + forms, clicks, screenshots, and logged-in browser state. + """ + + _ACTIONS = { + "open", + "read", + "snapshot", + "find", + "evaluate", + "click", + "fill", + "press", + "scroll", + "wait", + "screenshot", + "close", + "batch", + } + _AUTO_SCREENSHOT_ACTIONS = { + "open", + "snapshot", + "batch", + "click", + "fill", + "press", + "scroll", + } + + @staticmethod + def _shopping_landing_hint(output: str) -> str: + """Expose the actionable store link on global retail landing pages.""" + text = str(output or "") + if not re.search(r"\b(?:global|choose another store|store selector)\b", text, re.IGNORECASE): + return "" + match = re.search( + r'link\s+"(?P`` as normal answer deltas. ``state`` is a mutable per-round dict + carrying the emission bookkeeping; the stored answer is unchanged (see + ``visible_content_after_qwen_thinking``). + """ + events = [] + text = str(content or '') + if not text: + return events + lowered = text.casefold() + lead = len('') if lowered.startswith('') else 0 + close = lowered.find('') + if close < 0: + # The opening is injected by the chat template, so a progressive + # trace can arrive with no marker until . Emit the buffered text + # as thinking - holding back a possible partial close tag - so the trace + # streams live instead of dumping in one block at the end. + think_region = text[lead:max(lead, len(text) - len(''))] + answer_region = '' + else: + think_region = text[lead:close] + answer_region = text[close + len(''):] + emitted_think = int(state.get('think_emitted', 0) or 0) + if len(think_region) > emitted_think: + events.append({'delta': think_region[emitted_think:], 'thinking': True}) + state['think_emitted'] = len(think_region) + if close >= 0: + state['closed'] = True + emitted_answer = int(state.get('answer_emitted', 0) or 0) + if len(answer_region) > emitted_answer: + events.append({'delta': answer_region[emitted_answer:]}) + state['answer_emitted'] = len(answer_region) + return events + + def prior_short_answer_for_no_tool_summary(user_text, history): """Reuse the immediately prior concise answer for an explicit no-tool recap.""" text = str(user_text or '') @@ -2714,6 +2998,18 @@ def normalize_preview_entity_anchor_args(name, args): value = args.get(field) if isinstance(value, str) and value.startswith(prefix): args[field] = value[len(prefix):] + if canonical(name) == 'manage_notes': + # Some models copy the rendered note href without its leading '#'. + # Decode only a complete UUID; partial selectors retain the existing + # owner-bound resolution policy instead of gaining a new alias. + value = args.get('note_id') + if isinstance(value, str): + match = re.fullmatch( + r'note-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', + value, re.I, + ) + if match: + args['note_id'] = match.group(1) return args @@ -2754,6 +3050,9 @@ def normalize_preview_function_args(name, args, *, user_text=''): args.pop('filter', None) if canonical(name) == 'manage_notes': action = str(args.get('action') or '').strip().replace('-', '_').casefold() + if action == 'read' and (args.get('id') or args.get('note_id')): + args['action'] = 'view' + action = 'view' if action == 'list': request = str(user_text or '').casefold() # The notes backend interprets title/query/content on ``list`` as @@ -2770,6 +3069,10 @@ def normalize_preview_function_args(name, args, *, user_text=''): args.pop('archived', None) if not re.search(r'\b(?:checklists?|freeform\s+notes?|notes?\s+only)\b', request): args.pop('note_type', None) + query, _ = note_read_intent(user_text) + if query: + args['action'] = 'search' + args['query'] = query if canonical(name) == 'manage_skills': action = str(args.get('action') or '').strip().replace('-', '_').casefold() if action in {'update', 'change', 'revise'}: @@ -2897,8 +3200,9 @@ def scope_preview_contract(preview_contract, routed_contract, active_capabilitie requested_family_tools = set().union(*(FAMILY_TOOLS.get(f, ()) for f in active)) available_requested = routed_canonical & offered_canonical & requested_family_tools routed_canonical.update(canonical(name) for name in extra_tools) - if 'image_generation' not in active: - # A past image request must not keep generation warm on email/doc turns. + if 'image_generation' not in active and 'photo_editor' not in active: + # A past image request must not keep generation warm on email/doc turns, + # unless the photo_editor family (which bundles generate_image) is active. routed_canonical.discard('generate_image') missing_active = { f'capability:{family}' for family in active @@ -3230,7 +3534,11 @@ def _successful_email_identifiers(history): for message in history or (): if not isinstance(message, dict): continue - if message.get('role') == 'system' and 'Message UID:' in str(message.get('content') or ''): + active_reader = ( + message.get('_agent_injected') == 'context' + and (message.get('metadata') or {}).get('source') == 'active email reader' + ) + if (message.get('role') == 'system' or active_reader) and 'Message UID:' in str(message.get('content') or ''): extracted = _email_identifiers_from_text(message.get('content')) known['uid'].update(extracted['uid']) known['message_id'].update(extracted['message_id']) @@ -3518,6 +3826,20 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, runtime_tools = PREVIEW_TOOLS | ( NATIVE_WORKSPACE_TOOLS if allow_native_workspace else frozenset() ) | ({canonical(tool) for tool in external_runtime_tools} if allow_native_workspace else set()) + # Spec turns are driven by the router: the Spec contract (its offered set) + # IS the model runtime, so a Spec pick is runnable. Keyed off the same + # marker the spec hook sets (`routing_experiment == 'spec-router'`), never + # a new flag, and never applied to compact/native turns. + try: + from src.turn_contract import active_turn_contract as _atc + _spec_contract = _atc() + if _spec_contract is not None and getattr( + _spec_contract, 'routing_experiment', '') == 'spec-router': + runtime_tools = runtime_tools | { + canonical(tool) for tool in getattr(_spec_contract, 'offered', ()) or () + } + except Exception: + pass if bare not in runtime_tools: return decision(False, 'tool_not_in_model_runtime') if bare in {canonical(tool) for tool in external_runtime_tools}: @@ -3559,6 +3881,9 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, or bare in BROKERED_JOB_TOOLS or (contract_required and bare in {'edit_image', 'generate_image'}) or (contract_required and bare == 'create_document' and standalone_code_request(user_text)) + # The editor command is conservatively classified as a write; + # document.info is a read even when the user requested no edit. + or (bare == 'run_editor_command' and args.get('op') == 'document.info') ) ) if family not in authorized and not contextual_revision and not contract_scoped_mutation and not fixture_delete and not offered_private_action: @@ -3619,6 +3944,18 @@ def evaluate_preview_call(name, args, user_text='', *, allow_execute_code=False, # editor is offered only for an explicit image-editing turn. blocked_effects.remove(ToolEffect.NETWORK_EGRESS) allowed_effects.add(ToolEffect.NETWORK_EGRESS) + if ( + (contract_required or family in turn_authorized_families) + and bare in {'plan_edit', 'run_editor_command', 'get_editor_screenshot', 'ai_edit_image'} + ): + # Scoped photo_editor agent tools. They are never in the general tool + # list; the immutable turn contract only selects them for an explicit + # editor request (Quick Edit popup / image-edit turn), so their UI and + # brokered-network effects are covered by the contract seal. + blocked_effects.discard(ToolEffect.UI_SIDE_EFFECT) + allowed_effects.add(ToolEffect.UI_SIDE_EFFECT) + blocked_effects.discard(ToolEffect.NETWORK_EGRESS) + allowed_effects.add(ToolEffect.NETWORK_EGRESS) if ( contract_required and bare in {'download_model', 'serve_preset', 'stop_served_model'} @@ -3849,6 +4186,29 @@ def conversation(history_session, messages, *, owner=None, diagnostics=None): from src.turn_contract import result_reference_followup source_context = (recent_source_reference_context(groups[-2]) if len(groups) > 1 and result_reference_followup(current_text) else None) + # Tool output can dwarf the dialogue (especially a web search). Keep the + # visible exchange before enforcing the history cap, so one search-heavy + # turn cannot erase the subject of the next follow-up. + for index, group in enumerate(groups[:-1]): + if len(json.dumps(group)) <= 5000 or len(group) < 2: + continue + # Keep non-web tool evidence intact: a note/list follow-up may need + # exact result identifiers, and shortening it would be misleading. + if not any( + canonical((call.get('function') or {}).get('name', '')) in {'web_search', 'web_fetch'} + for message in group if message.get('role') == 'assistant' + for call in message.get('tool_calls') or () + ): + continue + answer = next((message.get('content') for message in reversed(group) + if message.get('role') == 'assistant' + and str(message.get('content') or '').strip()), None) + if answer is None: + continue + answer_text = str(answer) + if len(answer_text) > 3500: + answer_text = answer_text[:3500] + '\n[Earlier answer shortened for context.]' + groups[index] = [group[0], {'role': 'assistant', 'content': answer_text}] # Drop whole turns only, never orphan tool results from their native calls. groups = groups[-8:] while len(groups) > 1 and len(json.dumps(groups)) > 22000: @@ -4207,6 +4567,26 @@ def denied_response(): return 'I can’t perform that operation in this preview. No changes were made.' +WORKSPACE_REQUIRED_REFUSAL = ( + 'Shell and Python commands only run inside a workspace. ' + 'Open the code workspace (or set a working directory) and try again.' +) + + +def workspace_process_denial(executions) -> bool: + """True when a blocked call was refused because process execution needs a + sealed workspace (a plain chat turn carries none).""" + for entry in executions or (): + blob = ' '.join( + str(entry.get(key) or '') for key in ('error', 'output', 'failure_kind') + ) + # Resource identity also covers missing note and document selectors. + # Those denials must not turn into instructions to open a code workspace. + if 'sealed workspace' in blob.casefold(): + return True + return False + + def execution_has_write_effect(tool_name, content, capability, *, native_workspace_enabled): """Recognize successful native code that visibly mutates the workspace.""" @@ -4226,7 +4606,7 @@ _MUTATION_REQUEST = re.compile( r'\b(?:add|creat(?:e|ing)?|make|writ(?:e|ing)|sav(?:e|ing)|updat(?:e|ing)|edit(?:ing)?|chang(?:e|ing)|' r'set|schedul(?:e|ing)|reschedul(?:e|ing)|paus(?:e|ing)|resum(?:e|ing)|toggle|mark|' r'pin|archive|delet(?:e|ing)|remov(?:e|ing)|send|reply|draft|publish|run|launch|serve|start|stop|' - r'remember|remeber|forget|remind|review|proofread|suggest(?:ions?)?|expand|broaden|deepen|lighten|feedback|reserve|block)\b|' + r'remember|remeber|forget|remind|review|proofread|suggest(?:ions?)?|expand|broaden|deepen|lighten|reserve|block)\b|' r'\bgo\s+deeper\b', re.I, ) @@ -4262,7 +4642,7 @@ def mutation_action_requested(user_text): actionable = re.sub( r"\b(?:do\s+not|don't|never|without)\s+" r"(?:(?:chang(?:e|ing)|modif(?:y|ying)|edit(?:ing)?|delet(?:e|ing)|" - r"remov(?:e|ing)|send(?:ing)?|writ(?:e|ing)|creat(?:e|ing))\b)" + r"remov(?:e|ing)|send(?:ing)?|writ(?:e|ing)|creat(?:e|ing)|schedul(?:e|ing))\b)" r"[^.;\n]{0,120}?(?=(?:[.;\n]|\bbut\b|\binstead\b|$))", '', text, flags=re.I, ) @@ -5242,6 +5622,10 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac external_tool_schemas=None, temperature=0.0, context_resolution=None, request_authority=MISSING_AUTHORITY, **ignored): + if spec_driver_active(endpoint_url, model, owner, headers): + yield event({'delta': 'Spec driver mode is unavailable because its authority and completion checks are not supported. No action was performed.'}) + yield 'data: [DONE]\n\n' + return from src.generation_sampling import validate_temperature temperature = validate_temperature(temperature) # This path sends requests directly with httpx and therefore bypasses @@ -5622,6 +6006,38 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'multimodal_image_count': image_context_count, 'attachment_reference_count': attachment_refs, 'image_rehydration': conversation_diagnostics.get('image_rehydration')}) + _photo_editor_active = 'photo_editor' in (getattr(turn_contract, 'active_capabilities', ()) or ()) + _photo_editor_open = bool( + ((client_runtime_context or {}).get('web_ui_state') or {}).get('photo_editor_open') + ) + if _photo_editor_active and not _photo_editor_open: + _editor_guidance = ( + "This image isn't open in the editor yet, so I can't apply edits to the " + "canvas from chat. Open the attached image in the editor (Quick Edit) and " + "send this request there \u2014 I'll apply it to the canvas." + ) + history.append({'role': 'assistant', 'content': _editor_guidance}) + yield event({'delta': _editor_guidance}) + _action_prompt = str(direct_user_text or '') + for _annotation in ('\n\n[Image:', '\n[Image:', '\n\n[Attached', '\n[No vision model'): + _cut = _action_prompt.find(_annotation) + if _cut != -1: + _action_prompt = _action_prompt[:_cut] + _action_prompt = _action_prompt.strip() or str(direct_user_text or '').strip() + yield event({ + 'type': 'final_response', + 'content': _editor_guidance, + 'photo_editor_action': { + 'kind': 'open_and_apply', + 'prompt': _action_prompt, + }, + }) + yield event({'type': 'metrics', 'data': { + 'model': model, 'rounds': 0, 'round_texts': [_editor_guidance], + 'tool_events': [], 'tool_calls': 0, 'agent_rounds': 0, + }}) + yield 'data: [DONE]\n\n' + return try: async with httpx.AsyncClient( timeout=preview_http_timeout( @@ -5703,6 +6119,40 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac usage_in += intent_accounting.get('input_tokens', 0) usage_out += intent_accounting.get('output_tokens', 0) has_real_usage = intent_accounting.get('usage_source') == 'real' + # Optional env-gated router decision (spec surface only). Computed + # once per user turn; disabled/None leaves behaviour unchanged. + _spec_route_think = await spec_router_thinking_decision_async( + endpoint_url, model, owner, headers, direct_user_text) + if _spec_route_think is not None: + logging.getLogger(__name__).info( + 'spec think router: think=%s p_think=%s ms=%s', + _spec_route_think.get('think'), + _spec_route_think.get('p_think'), + _spec_route_think.get('ms')) + # Optional env-gated direct-answer override (spec surface only): + # when the spec router chose answer_directly with high confidence, + # round 1 is sent with tool_choice 'none' (same tools list). + _spec_direct_p = spec_direct_no_tools_p( + direct_user_text, endpoint_url, model, owner, headers) + if (turn_contract.required or turn_contract.unavailable + or turn_contract.required_read_operation is not None): + _spec_direct_p = None + # Option B: send round 1 with no tools at all for a direct answer. + _spec_direct_empty_menu = ( + _spec_direct_p is not None and spec_direct_mode() == 'empty_menu') + # ODYSSEUS_SPEC_MODE=driver: the router (not the 9B) picks the tool + # every round; the 9B only writes the arguments or the answer. + # Driver execution is refused at entry. Keep this false even if + # configuration changes while the stream is already in progress. + _spec_driver_on = False + # Driver mode narrows the contract per round to the Spec pick. + # Rebuild each round from this frozen base so the narrowing does + # not accumulate/mutate across rounds. + _driver_base_contract = turn_contract + # Per-round Spec pick (driver mode): 'answer_directly' | 'ask_user' + # | 'tool' | None (router fallback). Set inside the round loop. + _driver_round_kind = None + _driver_round_pick = None # One extra iteration is available only when a provider emits raw # tool markup during the normal final no-tools round. Ordinary # turns still obey ``round_limit`` exactly. @@ -5839,7 +6289,9 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac 'completion_reserve_seconds': artifact_completion_reserve_seconds, }) request_messages = provider_request_messages( - prune_multimodal_images(history, max_images=3) + prune_multimodal_images( + keep_recent_image_turns(history, keep=2), max_images=3 + ) ) if prior_summary_answer or force_no_tools_next_round or reserve_final_synthesis: round_offered = [] @@ -6036,6 +6488,94 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac break request['tools'] = source_tools request['tool_choice'] = 'required' + # ODYSSEUS_SPEC_MODE=driver: the router is the tool driver. + # Ask it for this round's single next action (any confidence); + # the 9B then only writes the arguments (a named tool_choice) or + # the answer (tools key omitted; vLLM rejects an empty array). + # Router error/timeout keeps the normal menu for this round. + if _spec_driver_on and round_offered: + # The router chooses from the FULL Ajax Spec surface (~all + # built-ins minus disabled), not the compact round menu, so + # a Spec pick is never silently out of reach. The 9B then + # only writes the arguments (a named tool_choice) or the + # answer (tools key omitted; vLLM rejects an empty array). + # Router error/timeout keeps the normal menu this round. + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS as _SPEC_SURFACE + _drv = spec_driver_pick(history, disabled_tools, direct_user_text) + if _drv is not None: + _drv_choice = _drv.get('choice') + if _drv_choice == 'answer_directly': + _driver_round_kind = 'answer_directly' + request.pop('tools', None) + request.pop('tool_choice', None) + elif _drv_choice == 'ask_user': + _driver_round_kind = 'ask_user' + _ask = [s for s in _SPEC_SURFACE + if (s.get('function') or {}).get('name') == 'ask_user'] + if _ask: + request['tools'] = [_ask[0]] + request['tool_choice'] = { + 'type': 'function', + 'function': {'name': 'ask_user'}, + } + if _ask[0] not in round_offered: + round_offered = list(round_offered) + [_ask[0]] + elif _drv_choice == 'tool' and _drv.get('tool_name'): + _name = _drv['tool_name'] + _sel = [s for s in list(round_offered) + list(_SPEC_SURFACE) + if (s.get('function') or {}).get('name') == _name] + if _sel: + _driver_round_kind = 'tool' + _driver_round_pick = _name + request['tools'] = [_sel[0]] + request['tool_choice'] = { + 'type': 'function', + 'function': {'name': _name}, + } + # The per-call validator looks the schema up in + # `round_offered` (not `request['tools']`), so the + # Spec pick must be present there too. + round_offered = [_sel[0]] + # Driver mode narrows the whole contract to the + # Spec pick for this round: the router is the + # authority, and the executor must be able to run + # the picked tool. Build a valid contract (the + # TurnContract invariant requires + # required <= offered <= executable and + # schema_json names == offered). + try: + from src.turn_contract import ( + _ACTIVE_CONTRACT as _TC, + canonical_tool as _ct, + ) + _pick = _ct(_name) + turn_contract = replace( + _driver_base_contract, + required=frozenset(), + offered=frozenset({_pick}), + executable=frozenset( + set(_driver_base_contract.executable) | {_pick}), + schema_json=(json.dumps(_sel[0]),), + required_read_operation=None, + ) + _TC.set(turn_contract) + except Exception: + turn_contract = _driver_base_contract + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s contract admit failed; ' + 'normal contract kept', round_number, _name) + else: + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s not executable; normal menu this round', + round_number, _name) + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=%s p=%s', round_number, + (_drv.get('tool_name') if _drv_choice == 'tool' + else _drv_choice), + _drv.get('p')) + else: + logging.getLogger(__name__).info( + '[spec-driver] round=%s pick=fallback p=None', round_number) pending, content, round_reasoning = {}, '', '' document_preview_index = None document_preview_content = '' @@ -6056,6 +6596,49 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac request = search_tool_choice_request(request) tool_call_requested = request.get('tool_choice') not in (None, 'auto', 'none') request = provider_compatible_tool_choice_request(request, model) + # Log the tool surface actually sent to the provider: the + # metrics `tool_schema_names` reports the pre-narrowing + # `offered` set, which hid the fact that a forced choice had + # shrunk the provider tools array to one schema. + logging.getLogger(__name__).info( + "clean preview round %s sending %d tool(s): %s", + round_number, + len(request.get('tools') or []), + [ + (schema.get('function') or {}).get('name') + for schema in (request.get('tools') or []) + ][:40], + ) + if round_number == 1 and _spec_direct_p is not None: + # The router decided to answer directly with high + # confidence. + if _spec_direct_empty_menu: + # Option B: an empty menu — send no tools at all, so + # nothing can be executed and no markup is recovered. + request['tools'] = [] + request.pop('tool_choice', None) + tool_call_requested = False + # Keep the thinking router's decision; never let the + # no-tools progressive rule turn thinking on when the + # router said off. + if not (_spec_route_think or {}).get('think'): + ctk = dict(request.get('chat_template_kwargs') or {}) + ctk['enable_thinking'] = False + request['chat_template_kwargs'] = ctk + logging.getLogger(__name__).info( + '[spec-harness] direct answer: empty menu p=%s', + _spec_direct_p) + else: + # Default: keep the same tools list (prefix-cache + # friendly) but forbid a tool call this round. Thinking + # is left to the thinking router; offered stays + # non-empty so this does not switch progressive + # thinking on. + request['tool_choice'] = 'none' + tool_call_requested = False + logging.getLogger(__name__).info( + '[spec-harness] direct answer: tool_choice none p=%s', + _spec_direct_p) if active_editor_target and request.get('tools'): # Edits share mutable document state. Await a saved result # before asking for another call; one call may still batch @@ -6144,6 +6727,7 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac and not progressive_thinking and not source_lookup_pending and not tool_call_requested + and not _spec_direct_empty_menu ): text_event = {'delta': text} if replace_streamed_draft_on_finish and not streamed_round_text: @@ -6197,6 +6781,13 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if content and not prior_summary_answer and not source_lookup_pending: yield event({'delta': content}) streamed_round_text = True + if _spec_direct_empty_menu and pending: + # Option B: nothing is offered, so never execute a tool + # call the model still produced (native shape). + logging.getLogger(__name__).info( + '[spec-harness] direct answer: dropped %d tool call(s)', + len(pending)) + pending = {} proposed = [pending[i] for i in sorted(pending)] # A lead-in emitted before a tool call is live progress, not # part of the terminal answer. Replace that draft when the @@ -6204,7 +6795,9 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac if proposed and streamed_round_text: replace_streamed_draft_on_finish = True unexecutable_dsml_completion = False - if not proposed and 'DSML' in content: + if (not proposed and not _spec_direct_empty_menu + and _driver_round_kind != 'answer_directly' + and 'DSML' in content): offered_by_canonical = { canonical(schema['function']['name']): schema['function']['name'] for schema in round_offered @@ -6252,6 +6845,15 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac }) else: unexecutable_dsml_completion = True + if _spec_direct_empty_menu and content: + # Option B: show the answer text without any tool-call + # markup the model may still have written (nothing offered). + _stripped = re.sub( + r']*>.*?', '', + content, flags=re.DOTALL | re.IGNORECASE).strip() + if _stripped != content.strip(): + content = _stripped + replace_streamed_draft_on_finish = True if source_lookup_pending and not proposed: if source_answer_retries < 1 and round_number < round_limit: source_answer_retries += 1 @@ -6276,6 +6878,37 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac ) if active_editor_target: proposed = drop_redundant_editor_noops(proposed) + # ODYSSEUS_SPEC_MODE=driver: Spec is the tool driver, so execute + # ONLY the call Spec picked this round. Drop (never execute) any + # other native call and any text/DSML-recovered call; when Spec + # said answer_directly, keep no calls and strip tool markup so + # the remaining text is the answer. + if _driver_round_kind == 'answer_directly': + if proposed: + for _c in proposed: + logging.getLogger(__name__).info( + '[spec-driver] dropped off-pick call name=%s', + (_c.get('function') or {}).get('name')) + proposed = [] + if content: + _stripped = strip_tool_blocks(content, skip_fenced=True).strip() + _stripped = re.sub( + r']*>.*?', '', + _stripped, flags=re.DOTALL | re.IGNORECASE).strip() + if _stripped != content: + content = _stripped + replace_streamed_draft_on_finish = True + elif _driver_round_kind in ('tool', 'ask_user'): + _want = 'ask_user' if _driver_round_kind == 'ask_user' else _driver_round_pick + _keep = [] + for _c in proposed: + _cn = (_c.get('function') or {}).get('name') + if canonical(str(_cn or '')) == canonical(str(_want or '')): + _keep.append(_c) + else: + logging.getLogger(__name__).info( + '[spec-driver] dropped off-pick call name=%s', _cn) + proposed = _keep if model_choice_experiment: for proposal in proposed: yield event({'type': 'model_tool_proposal', 'round': round_number, @@ -6695,6 +7328,25 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac # that draft and persisted history with the evidence-backed result. history.pop() refusal = denied_response() + # A photo-editor turn with no canvas command actually executed + # (no browser editor attached) cannot claim the edit. Point the + # user at the editor instead of the generic no-op refusal. + if 'photo_editor' in (getattr(turn_contract, 'active_capabilities', ()) or ()): + touched_canvas = any( + str(e.get('tool') or '') in { + 'run_editor_command', 'get_editor_screenshot', + 'ai_edit_image', 'generate_image', + } + and not e.get('error') and e.get('exit_code', 0) == 0 + for e in executions + ) + if not touched_canvas: + refusal = ( + 'This photo is not open on the editor canvas, so I cannot ' + 'apply edits to it from chat. Open the attached image in the ' + 'editor (Quick Edit) and send the request there, and I will ' + 'apply it to the canvas.' + ) history.append({'role': 'assistant', 'content': refusal}) yield event({'type': 'final_response', 'content': refusal}) break @@ -7948,6 +8600,15 @@ async def stream_preview(*, endpoint_url, model, messages, headers, turn_contrac output, user_text=latest_user, max_items=contract_item_limit(turn_contract, 20), ) + followup = note_locator_followup(output, user_text=latest_user) + if followup: + if round_number < round_limit: + structured_terminal_response = '' + round_recovery_messages.append(followup) + else: + structured_terminal_response = ( + 'I found a note, but could not read its content in this turn.' + ) action = str(args.get('action') or '').replace('-', '_').casefold() if ( action in {'search', 'find'} diff --git a/src/context_compactor.py b/src/context_compactor.py index 05b53ce95..09e2e59f6 100644 --- a/src/context_compactor.py +++ b/src/context_compactor.py @@ -92,6 +92,84 @@ def prune_multimodal_images( return pruned +_IMAGE_TURN_LABEL_RE = re.compile(r"\[Image(?:s)?(?:[ \t]{1,64}attached)?:[ \t]{0,64}([^\]]{1,512})\]", re.IGNORECASE) + + +def _image_turn_label(message: Dict) -> str: + """Best-effort '[Image: name]' label for a message whose image parts are dropped.""" + name = "" + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES: + image_url = item.get("image_url") + candidate = item.get("name") or (image_url.get("name") if isinstance(image_url, dict) else None) + if candidate: + name = str(candidate) + break + if not name: + text = _content_as_text(content) + match = _IMAGE_TURN_LABEL_RE.search(text) + if match: + name = match.group(1).strip() + return f"[Image: {name}]" if name else "[Image]" + + +def keep_recent_image_turns(messages: List[Dict], *, keep: int = 2) -> List[Dict]: + """Keep image parts only for the last `keep` messages that carry an image. + + Older image blocks are dropped and replaced with a short ``[Image: name]`` + text label so the model still knows a picture was shared (and its name) + without paying the image tokens on every later turn. Non-text, non-image + multimodal blocks (e.g. audio) are left untouched. + """ + limit = max(0, int(keep)) + image_messages = [ + index for index, message in enumerate(messages) + if isinstance(message, dict) + and isinstance(message.get("content"), list) + and any( + isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES + for item in message["content"] + ) + ] + if len(image_messages) <= limit: + return list(messages) + selected = set(image_messages[len(image_messages) - limit:]) if limit else set() + + trimmed: List[Dict] = [] + for message_index, message in enumerate(messages): + if message_index in selected or not isinstance(message, dict): + trimmed.append(message) + continue + content = message.get("content") + if not isinstance(content, list): + trimmed.append(message) + continue + kept_blocks = [ + dict(item) if isinstance(item, dict) else item for item in content + if not (isinstance(item, dict) and item.get("type") in _MULTIMODAL_IMAGE_TYPES) + ] + if len(kept_blocks) == len(content): + trimmed.append(message) + continue + label = _image_turn_label(message) + # Attach the label to the trailing text block when possible so we do not + # introduce an extra content block; otherwise append a fresh text block. + labelled = False + for item in reversed(kept_blocks): + if isinstance(item, dict) and item.get("type") == "text": + item["text"] = (item.get("text") or "").rstrip() + f"\n{label}" + labelled = True + break + if not labelled: + kept_blocks.append({"type": "text", "text": label}) + cloned = dict(message) + cloned["content"] = kept_blocks + trimmed.append(cloned) + return trimmed + + COMPACT_THRESHOLD = 0.85 # Trigger compaction at 85% of context window SUMMARY_MAX_TOKENS = 1024 SMALL_CONTEXT_LIMIT = 8192 # Models with context <= this get aggressive trimming @@ -106,7 +184,7 @@ def auto_compact_threshold_percent() -> int: return max(50, min(95, value)) # Cursor-style self-summarization prompt — produces structured, dense summaries -SELF_SUMMARY_SYSTEM_PROMPT = """You are summarizing a conversation to preserve context after compaction. Produce a structured summary that lets the conversation continue seamlessly. +SELF_SUMMARY_SYSTEM_PROMPT = """You are summarizing a conversation with Ajax, the user's AI assistant, to preserve context after compaction. Produce a structured summary that lets Ajax continue seamlessly. Ajax is the assistant's name; Odysseus is the app name. Use this format: diff --git a/src/document_processor.py b/src/document_processor.py index 38643cb5a..a623c1a1b 100644 --- a/src/document_processor.py +++ b/src/document_processor.py @@ -2,6 +2,7 @@ """Document processing: PDF/OCR extraction, text file handling, image VL analysis, user content building.""" import os +import io import logging import mimetypes import base64 @@ -496,6 +497,49 @@ def analyze_image_with_vl(image_path: str, owner: str | None = None) -> str: return analyze_image_with_vl_result(image_path, owner=owner).get("text", "") +def _vision_max_side() -> int: + """Longest-side cap for images inlined into the chat request (tokens).""" + try: + value = int(os.getenv("ODYSSEUS_VISION_MAX_SIDE", "1280") or 1280) + except (TypeError, ValueError): + value = 1280 + return max(0, value) + + +def encode_image_data_url(path: str, fallback_format: str) -> tuple[str, str]: + """Return (data_url, image_format) for an image, downscaled if large. + + Downscale the longest side to ODYSSEUS_VISION_MAX_SIDE (default 1280) and + re-encode as JPEG so the chat model does not pay full-resolution image + tokens. Small/undecodable images fall back to the original bytes so a + real attachment is never dropped. + """ + max_side = _vision_max_side() + try: + from PIL import Image # local import: keep module import cost low + with Image.open(path) as img: + img = img.convert("RGB") if img.mode not in ("RGB", "L") else img + width, height = img.size + if max_side and max(width, height) > max_side: + scale = max_side / float(max(width, height)) + img = img.resize( + (max(1, round(width * scale)), max(1, round(height * scale))), + Image.LANCZOS, + ) + buffer = io.BytesIO() + img.save(buffer, format="JPEG", quality=85, optimize=True) + return ( + "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode("utf-8"), + "jpeg", + ) + except Exception as exc: + logger.debug(f"Image downscale failed for {path}: {exc}; sending original bytes") + with open(path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode("utf-8") + image_format = (fallback_format or "png").lower() + return f"data:image/{image_format};base64,{encoded_string}", image_format + + def build_user_content( text: str, attachment_ids: list[str] | None, @@ -542,15 +586,14 @@ def build_user_content( if upload_handler.is_image_file(display_name, mime): try: - with open(path, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode("utf-8") # Extensionless uploads (e.g. a pasted screenshot) have no ext, # so fall back to the resolved MIME subtype rather than emitting # an invalid "data:image/;base64," with an empty subtype. - image_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png") + fallback_format = ext[1:] or (mime.split("/", 1)[1] if mime.startswith("image/") else "png") + data_url, image_format = encode_image_data_url(path, fallback_format) content.append({ "type": "image_url", - "image_url": {"url": f"data:image/{image_format};base64,{encoded_string}"}, + "image_url": {"url": data_url}, }) except Exception as e: logger.error(f"Failed to encode image {fid}: {e}") diff --git a/src/editor_agent_model.py b/src/editor_agent_model.py new file mode 100644 index 000000000..7c9bdf98a --- /dev/null +++ b/src/editor_agent_model.py @@ -0,0 +1,732 @@ +"""Editor agent/planner model resolution. + +The image-editor Quick Edit planner (`/api/editor/plan`, `/api/editor/agent`) +runs on an explicitly configured local vision model when one is reachable, +with the configured chat model as a fallback. This module owns: + + * the local model base URLs / model id (settings-overridable), + * an ordered candidate list (per-request ``agent_model`` override, then the + local Qwen3.5-9B, then the hosted default/utility chat model), + * a vision fallback chain: local model WITH the canvas screenshot, else a + text-only call whose image is described by a separate VL caption model, + * a lightweight reachability probe for the configured local bases. + +Everything is best-effort: when no local server is reachable the planner +transparently falls back to the previously-used hosted chat model, so the +editor agent never regresses to "no plan". +""" + +from __future__ import annotations + +import contextvars +import json +import logging +import os +import re +import threading +from typing import Any, Dict, List, Optional, Tuple + +import httpx + +logger = logging.getLogger(__name__) + +# Local endpoints are opt-in through settings or ODYSSEUS_EDITOR_AGENT_BASES. +DEFAULT_LOCAL_BASES = [] +DEFAULT_LOCAL_MODEL = "Qwen/Qwen3.5-9B" + +# Vision caption model used when the editor-agent model cannot accept images. +DEFAULT_CAPTION_BASE = "" +DEFAULT_CAPTION_MODEL = "Qwen/Qwen3.5-9B" + +# Served-model-id cache per base (populated on first use). The local servers 400 +# with "Failed to load model" when asked for an id they do not serve (e.g. the +# client's default "Qwen3.5-9B-MLX-8bit", or a stale caption model), so we +# resolve the requested id against GET {base}/models and fall back to one that +# is actually served. +_MODELS_CACHE: Dict[str, set] = {} + +_CAPTION_INSTRUCTION = ( + "Describe this image in one or two sentences for a photo editor: the main " + "subject, the background/scene, lighting, dominant colours, and any obvious " + "flaws (blown highlights, colour cast, clutter, blur)." +) + + +def _setting(key: str, default: Any = "") -> Any: + try: + from src.settings import load_settings + + value = load_settings().get(key) + return value if value not in (None, "") else default + except Exception: + return default + + +def local_bases() -> List[str]: + override = _setting("editor_agent_base_url") + if override: + return [str(override)] + env = os.getenv("ODYSSEUS_EDITOR_AGENT_BASES") + if env: + bases = [b.strip() for b in env.split(",") if b.strip()] + if bases: + return bases + return list(DEFAULT_LOCAL_BASES) + + +def local_model() -> str: + return str(_setting("editor_agent_model", DEFAULT_LOCAL_MODEL)) + + +def caption_base() -> str: + return str(_setting("editor_caption_base_url", DEFAULT_CAPTION_BASE)) + + +def caption_model() -> str: + return str(_setting("editor_caption_model", DEFAULT_CAPTION_MODEL)) + + +# Per-request / per-plan context (contextvars keep this correct under the +# async event loop without sharing state between concurrent editor turns). +_override: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar( + "editor_agent_model_override", default=None +) +_last_model: contextvars.ContextVar[Optional[Tuple[str, str]]] = contextvars.ContextVar( + "editor_agent_last_model", default=None +) + + +def set_override(value: Optional[str]) -> None: + _override.set(value or None) + + +def get_override() -> Optional[str]: + return _override.get() + + +def set_last_model(model: str, source: str) -> None: + _last_model.set((model, source)) + + +def get_last_model() -> Optional[Tuple[str, str]]: + return _last_model.get() + + +def _chat_url(base: str) -> str: + b = str(base or "").rstrip("/") + if b.endswith("/chat/completions"): + return b + return b + "/chat/completions" + + +def _split_override(override: str) -> Tuple[str, str]: + """Parse ``base::model`` (or a bare model id).""" + raw = str(override or "").strip() + if "::" in raw: + base, model = raw.split("::", 1) + return base.strip(), model.strip() + return "", raw + + +def _has_image(messages: List[dict]) -> bool: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + return True + return False + + +def _first_image(messages: List[dict]) -> Optional[str]: + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + url = (part.get("image_url") or {}).get("url") + if isinstance(url, str): + return url + return None + + +def _strip_images(messages: List[dict]) -> List[dict]: + out: List[dict] = [] + for msg in messages: + content = msg.get("content") + if isinstance(content, list): + text = "\n".join( + str(part.get("text") or "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + out.append({"role": msg.get("role", "user"), "content": text}) + else: + out.append({"role": msg.get("role", "user"), "content": content}) + return out + + +def _with_caption(messages: List[dict], caption: str) -> List[dict]: + note = f"\n\n[Canvas screenshot caption: {caption}]" + out = [dict(m) for m in messages] + for msg in reversed(out): + if msg.get("role") == "user": + msg["content"] = str(msg.get("content") or "") + note + break + return out + + +async def _post_chat( + base: str, + model: str, + messages: List[dict], + headers: Optional[dict], + timeout: float, + max_tokens: int = 1024, + response_format: Optional[dict] = None, +) -> Optional[str]: + if not base or not model: + return None + payload: Dict[str, Any] = { + "model": model, + "messages": messages, + "temperature": 0.2, + "max_tokens": max_tokens, + "stream": False, + } + # Constrain the planner to a strict JSON ops schema (structured outputs / + # grammar) so the model can only emit valid editor ops. Unknown servers + # ignore the field; the server-side schema validation is the real enforcer. + if isinstance(response_format, dict): + payload["response_format"] = response_format + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(_chat_url(base), json=payload, headers=headers or {}) + if resp.status_code != 200: + logger.debug( + "editor agent chat %s -> %s (%s)", base, resp.status_code, resp.text[:200] + ) + return None + data = resp.json() + except Exception as exc: + logger.debug("editor agent chat %s failed: %s", base, exc) + return None + try: + return data["choices"][0]["message"]["content"] + except Exception: + return None + + +async def _served_models(base: str, timeout: float = 3.0) -> set: + """Return the set of model ids a base serves (cached; empty on failure).""" + if not base: + return set() + cached = _MODELS_CACHE.get(base) + if cached is not None: + return cached + ids: set = set() + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(base.rstrip("/") + "/models") + if resp.status_code == 200: + ids = { + str(m.get("id")) + for m in (resp.json().get("data") or []) + if isinstance(m, dict) and m.get("id") + } + except Exception: + ids = set() + _MODELS_CACHE[base] = ids + return ids + + +async def _resolve_model(base: str, requested: str, fallback: str) -> str: + """Return a model id ``base`` actually serves. + + If the requested id is served, keep it; otherwise use ``fallback`` (when + served) or any served id. When the served list is unknown (server down), + keep the requested id (or ``fallback``). + """ + requested = str(requested or "").strip() + fallback = str(fallback or "").strip() + ids = await _served_models(base) + if ids: + if requested and requested in ids: + return requested + if fallback in ids: + return fallback + return sorted(ids)[0] + return requested or fallback + + +async def caption_image(data_url: str, owner: Optional[str] = None, timeout: float = 90.0) -> Optional[str]: + if not isinstance(data_url, str) or not data_url.startswith("data:image/"): + return None + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": _CAPTION_INSTRUCTION}, + {"type": "image_url", "image_url": {"url": data_url}}, + ], + } + ] + base = caption_base() + if not base: + return None + model = await _resolve_model(base, caption_model(), local_model()) + return await _post_chat(base, model, messages, None, timeout, max_tokens=200) + + +_DONE_RE = re.compile(r"^\s*DONE\b", re.I) +_RETRY_RE = re.compile(r"RETRY:?\s*(.+)", re.I | re.S) + + +def _parse_self_check(text: str) -> Dict[str, Any]: + s = str(text or "").strip() + if _DONE_RE.match(s): + return {"done": True, "issues": [], "retry_hint": "", "text": s} + m = _RETRY_RE.search(s) + if m or re.search(r"\bretry\b", s, re.I): + hint = (m.group(1).strip() if m else s) or s + return {"done": False, "issues": [hint], "retry_hint": hint, "text": s} + return {"done": True, "issues": [], "retry_hint": "", "text": s} + + +async def self_check( + before: Any, + after: Any, + prompt: str, + model: Optional[str] = None, + owner: Optional[str] = None, + timeout: float = 30.0, +) -> Dict[str, Any]: + """Same-origin VL self-check: compare BEFORE vs AFTER against the intent. + + The browser cannot call the local VL server directly (it sends no CORS + headers), so ``/api/editor/self-check`` proxies here. Returns + ``{done, issues, retry_hint, text}``; on any failure (bad input, the model + is unreachable, or a timeout) it degrades to ``done=True`` with + ``reason="unavailable"`` so the client never gets stuck retrying. + """ + if not ( + isinstance(before, str) and before.startswith("data:image/") + and isinstance(after, str) and after.startswith("data:image/") + ): + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + bases = local_bases() or list(DEFAULT_LOCAL_BASES) + if not bases: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + base = bases[0] + mdl = await _resolve_model(base, str(model or local_model()), local_model()) + messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": str(prompt or "")}, + {"type": "image_url", "image_url": {"url": before}}, + {"type": "image_url", "image_url": {"url": after}}, + ], + }] + text = await _post_chat(base, mdl, messages, None, timeout, max_tokens=140) + if not text: + return {"done": True, "issues": [], "retry_hint": "", + "reason": "unavailable", "text": "DONE"} + out = _parse_self_check(text) + out["model"] = mdl + out["source"] = "local" + return out + + +def _resolve_override_candidate(override: str, owner: Optional[str]) -> Optional[Dict[str, Any]]: + """Resolve an explicit agent_model override under server authority. + + Guarantees: + - Client-provided raw URLs never determine arbitrary outbound network destinations. + - Registered endpoint IDs resolve only to enabled endpoints visible to the owner. + - Shared endpoints (owner IS NULL) remain visible to authorized users. + - Disabled, foreign-owned, forged, or unconfigured destinations fail closed (return None). + - Admin privileges do not allow arbitrary unconfigured destinations. + """ + raw = str(override or "").strip() + if not raw: + return None + + if "::" in raw: + target, requested_model = raw.split("::", 1) + target = target.strip() + requested_model = requested_model.strip() + else: + target = "" + requested_model = raw + + # 1. Explicit local selection or matching a configured local base + if target == "local": + bases = local_bases() + if not bases: + return None + return { + "base": bases[0], + "model": requested_model or local_model(), + "headers": None, + "source": "override", + } + for lb in local_bases(): + if target and target.rstrip("/") == lb.rstrip("/"): + return { + "base": lb, + "model": requested_model or local_model(), + "headers": None, + "source": "override", + } + + # 2. Explicit hosted fallback selection + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + if target in ("default", "hosted"): + return { + "base": url, + "model": requested_model or model, + "headers": headers, + "source": "hosted", + } + if target and target.rstrip("/") == url.rstrip("/"): + if not requested_model or requested_model == model: + return { + "base": url, + "model": requested_model or model, + "headers": headers, + "source": "hosted", + } + except Exception: + pass + if target in ("default", "hosted"): + return None + + # 3. Bare model id without endpoint prefix + if not target: + if requested_model == "local": + bases = local_bases() + return {"base": bases[0], "model": local_model(), "headers": None, "source": "local"} if bases else None + if requested_model in ("default", "hosted"): + return _resolve_override_candidate("default::", owner) + + # Exact match with local model + if requested_model == local_model(): + bases = local_bases() + if bases: + return {"base": bases[0], "model": local_model(), "headers": None, "source": "local"} + + # Exact match with hosted default/utility + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model and model == requested_model: + return {"base": url, "model": model, "headers": headers, "source": "hosted"} + except Exception: + pass + + # Check registered database endpoints for this bare model + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter, _auth_disabled + from src.endpoint_resolver import ( + resolve_endpoint_runtime, + build_chat_url, + build_headers, + _endpoint_enabled_models, + _endpoint_hidden_models, + ) + + db = SessionLocal() + try: + if not owner and not _auth_disabled(): + q = db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True, + ModelEndpoint.owner == None, + ) + else: + q = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), + ModelEndpoint, + owner, + ) + matching_eps = [] + for ep in q.all(): + enabled_models = _endpoint_enabled_models(ep) + hidden_models = _endpoint_hidden_models(ep) + if requested_model in enabled_models and requested_model not in hidden_models: + matching_eps.append(ep) + + # Avoid model ID collision across providers: only accept if unique + if len(matching_eps) == 1: + ep = matching_eps[0] + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + return { + "base": build_chat_url(base), + "model": requested_model, + "headers": build_headers(api_key, base), + "source": "registered", + "endpoint_id": ep.id, + } + finally: + db.close() + except Exception: + pass + return None + + # 4. Target is an endpoint_id or a registered legacy endpoint URL + try: + from core.database import SessionLocal, ModelEndpoint + from src.auth_helpers import owner_filter, _auth_disabled + from src.endpoint_resolver import ( + resolve_endpoint_runtime, + build_chat_url, + build_headers, + _endpoint_enabled_models, + _endpoint_hidden_models, + _first_chat_model, + ) + + db = SessionLocal() + try: + if not owner and not _auth_disabled(): + q = db.query(ModelEndpoint).filter( + ModelEndpoint.is_enabled == True, + ModelEndpoint.owner == None, + ) + else: + q = owner_filter( + db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), + ModelEndpoint, + owner, + ) + # Lookup by ID + ep = q.filter(ModelEndpoint.id == target).first() + + # If not found by ID, match legacy registered base URL (strictly registered rows only) + if ep is None and ("://" in target or target.startswith("http")): + target_norm = target.rstrip("/") + for cand_ep in q.all(): + cand_base = (cand_ep.base_url or "").rstrip("/") + if ( + cand_base == target_norm + or cand_base + "/v1" == target_norm + or cand_base == target_norm + "/v1" + ): + ep = cand_ep + break + + if ep is None: + # Target is disabled, foreign-owned, nonexistent, or arbitrary unconfigured URL -> fail closed! + return None + + enabled_models = _endpoint_enabled_models(ep) + hidden_models = _endpoint_hidden_models(ep) + if requested_model: + if requested_model in hidden_models: + return None + if enabled_models and requested_model not in enabled_models: + # Model not served by this endpoint -> fail closed against forged model IDs + return None + model = requested_model + else: + model = _first_chat_model(enabled_models) + if not model: + return None + + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + return { + "base": build_chat_url(base), + "model": model, + "headers": build_headers(api_key, base), + "source": "registered", + "endpoint_id": ep.id, + } + finally: + db.close() + except Exception: + return None + + +def _candidates(owner: Optional[str], override: Optional[str]) -> List[Dict[str, Any]]: + cands: List[Dict[str, Any]] = [] + for base in local_bases(): + cands.append({"base": base, "model": local_model(), "headers": None, "source": "local"}) + try: + from src.endpoint_resolver import resolve_endpoint + + for prefix in ("utility", "default"): + try: + url, model, headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + cands.append({"base": url, "model": model, "headers": headers, "source": "hosted"}) + break + except Exception: + pass + if override: + cand = _resolve_override_candidate(override, owner) + if cand is not None: + cands.insert(0, cand) + return cands + # Unauthorized, invalid, foreign, disabled, or forged override: + # Fail closed! Never silently dispatch sensitive user images/state to an unexpected fallback. + return [] + return cands + + +async def plan_with_editor_agent( + messages: List[dict], + owner: Optional[str] = None, + override: Optional[str] = None, + timeout: float = 90.0, + response_format: Optional[dict] = None, +) -> Optional[Tuple[str, str, str]]: + """Return (raw_plan_text, model_used, source) or None. + + Tries the local model WITH the screenshot, then the same model text-only + with a VL caption of the screenshot, then the hosted chat model. + + ``response_format`` (a JSON-schema structured-output constraint) is sent + ONLY to the local editor-agent bases, so a hosted fallback model that does + not support the field never rejects the request. + """ + has_img = _has_image(messages) + text_messages = _strip_images(messages) if has_img else messages + image = _first_image(messages) if has_img else None + caption: Optional[str] = None + _local = {str(b).rstrip("/") for b in local_bases()} + for cand in _candidates(owner, override): + base, model = cand["base"], cand["model"] + headers = cand.get("headers") + is_local = cand["source"] == "local" or str(base).rstrip("/") in _local + rf = response_format if (response_format and is_local) else None + extra = {"response_format": rf} if rf else {} + if has_img: + raw = await _post_chat(base, model, messages, headers, timeout, **extra) + if raw: + return raw, model, cand["source"] + if caption is None and image: + caption = await caption_image(image, owner) + if caption: + raw = await _post_chat( + base, model, _with_caption(text_messages, caption), headers, timeout, **extra, + ) + if raw: + return raw, model, cand["source"] + "+caption" + else: + raw = await _post_chat( + base, model, text_messages, headers, timeout, **extra + ) + if raw: + return raw, model, cand["source"] + return None + + +async def list_editor_agent_models(owner: Optional[str] = None) -> List[Dict[str, Any]]: + """Models available to the editor agent, for the model picker. + + Lists the local Qwen3.5-9B (one entry per reachable base) plus the hosted + default/utility chat model that the planner falls back to. + """ + models: List[Dict[str, Any]] = [] + seen: set = set() + for base in local_bases(): + model = local_model() + key = ("local", base, model) + if key in seen: + continue + seen.add(key) + models.append({ + "id": model, + "label": f"{model} · local", + "source": "local", + "vision": True, + "base_url": base, + "endpoint_id": "local", + }) + try: + from src.endpoint_resolver import resolve_endpoint + from src.settings import get_user_setting, load_settings + + for prefix in ("utility", "default"): + try: + url, model, _headers = resolve_endpoint(prefix, owner=owner) + except Exception: + continue + if url and model: + key = ("hosted", model) + if key not in seen: + seen.add(key) + ep_id = None + try: + settings = load_settings() + ep_id = (get_user_setting(f"{prefix}_endpoint_id", owner or "", settings.get(f"{prefix}_endpoint_id", "")) or "").strip() or None + except Exception: + pass + models.append({ + "id": model, + "label": model, + "source": "hosted", + "vision": None, + "base_url": url, + "endpoint_id": ep_id or prefix, + }) + break + except Exception: + pass + return models + + +# --------------------------------------------------------------------------- +# Reachability probe (best effort, non-blocking). +# --------------------------------------------------------------------------- + +_PROBE: Dict[str, bool] = {} + + +def probe_models(base: str, timeout: float = 3.0) -> bool: + try: + with httpx.Client(timeout=timeout) as client: + resp = client.get(str(base).rstrip("/") + "/models") + return resp.status_code == 200 + except Exception: + return False + + +def _probe_worker() -> None: + for base in local_bases(): + ok = probe_models(base) + _PROBE[base] = ok + logger.info("editor agent local base %s reachable=%s", base, ok) + cap = caption_base() + if cap: + _PROBE[cap] = probe_models(cap) + logger.info("editor agent caption base %s reachable=%s", cap, _PROBE[cap]) + + +def startup_probe() -> None: + """Kick off a background reachability probe (never blocks startup).""" + try: + threading.Thread(target=_probe_worker, name="editor-agent-probe", daemon=True).start() + except Exception: + logger.debug("editor agent probe thread failed", exc_info=True) + + +def probe_status() -> Dict[str, bool]: + return dict(_PROBE) diff --git a/src/editor_style_guide.py b/src/editor_style_guide.py new file mode 100644 index 000000000..1355c2608 --- /dev/null +++ b/src/editor_style_guide.py @@ -0,0 +1,1108 @@ +"""Single source of truth for photo-editor intent routing. + +Both the server-side planner (``routes/editor_agent_routes.py``) and the +browser fallback planner (``static/js/editor/nl-edit.js``) read this guide so +their behaviour cannot drift. It is served verbatim to the client at +``GET /api/editor/style-guide``. + +Calibrated strengths live here; this file is the only place to tune routing. +It is intentionally dependency-free so the route module can import it cheaply. +""" +from __future__ import annotations + +import json +import re +from collections import Counter +from typing import Any, Dict, List, Optional + +STYLE_GUIDE_VERSION = 8 + +# New editor ops (implemented client-side, static) with canonical params. The +# server op catalog must include these or a valid plan would be filtered out. +NEW_OPS: List[Dict[str, Any]] = [ + { + "op": "photo.autoTone", + "params": {}, + "help": "Camera-Raw-style auto tone/colour (per-channel levels, auto white " + "balance, exposure, contrast). The single preferred global lift.", + }, + { + "op": "retouch.skin", + "params": {"smooth": 8, "opacity": 0.35, "blemish": True, + "radius": 12, "threshold": 16}, + "help": "Gentle frequency-separation skin smoothing that keeps texture, over " + "the selection intersected with skin tones. Keep it subtle: smooth " + "<=14, opacity <=0.45; it never brightens the face.", + }, + { + "op": "retouch.blemish", + "params": {"radius": 4, "threshold": 16}, + "help": "Remove small blemishes inside the selection or skin region.", + }, + { + "op": "retouch.teeth", + "params": {"desat": 0.6, "bright": 16}, + "help": "Whiten teeth within the current selection (desaturate yellow, " + "lift brightness with a small feather).", + }, + { + "op": "selection.face", + "params": {"mode": "replace"}, + "help": "Select a face (SAM text 'face', upper-centre ellipse fallback).", + }, + { + "op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}, + "help": "Select the background (SAM subject inverted, corner-colour " + "fallback). Subject-aware; prefer over a plain global blur.", + }, + { + "op": "fill.flood", + "params": {"color": "#ff0000", "x": 0, "y": 0, "tolerance": 32}, + "help": "Paint-bucket flood-fill a pixel layer from a point " + "{x,y,color,tolerance}. Use for 'fill ... with ' when there " + "is no active selection.", + }, + { + "op": "selection.fill", + "params": {"color": "#ff0000"}, + "help": "Fill the current selection with a colour on the active pixel layer " + "{color}. Use for 'fill the selection with '.", + }, + { + "op": "text.edit", + "params": {"content": "New text", "size": 96, "color": "#ff0000"}, + "help": "Edit an EXISTING text layer {id|index|name, content, font, size, " + "color, align, tracking, lineHeight}. Use for 'make the text " + "bigger/smaller' and 'make the title ' — never text.add.", + }, + { + "op": "layers.align", + "params": {"mode": "center"}, + "help": "Align/distribute the selected layers to the canvas {mode:left|center|" + "right|top|middle|bottom|distribute-h|distribute-v}.", + }, + { + "op": "layer.rotate", + "params": {"degrees": 90}, + "help": "Rotate the ACTIVE LAYER by 90/180/270 {degrees, id|index|name}. Use " + "unless the prompt names the whole image/canvas/photo.", + }, + { + "op": "layer.flip", + "params": {"axis": "h"}, + "help": "Flip the ACTIVE LAYER in place {axis:h|v, id|index|name}.", + }, + { + "op": "document.rotate", + "params": {"degrees": 90}, + "help": "Rotate the WHOLE document (all layers + canvas) by 90/180/270 " + "{degrees} — for 'rotate the whole image/canvas/photo'.", + }, + { + "op": "document.flip", + "params": {"axis": "h"}, + "help": "Flip the WHOLE document {axis:h|v} — for 'flip the canvas/whole " + "image'.", + }, + { + "op": "selection.copy", + "params": {}, + "help": "Copy (or cut, {cut:true}) the selection pixels to the internal " + "clipboard.", + }, + { + "op": "selection.paste", + "params": {}, + "help": "Paste the internal clipboard as a new layer {name}.", + }, + { + "op": "layers.flatten", + "params": {}, + "help": "Flatten all layers into a single layer.", + }, + { + "op": "layers.mergeDown", + "params": {}, + "help": "Merge the active layer down into the layer below.", + }, +] + +NEW_OP_NAMES: List[str] = [op["op"] for op in NEW_OPS] + + +def _adj(kind: str, params: Dict[str, Any]) -> Dict[str, Any]: + return {"op": "adjustments.apply", "params": {"type": kind, "params": params}} + + +def _filter(fid: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return {"op": "filters.apply", "params": {"id": fid, **(params or {})}} + + +# Ordered intent routes. The first route whose keyword appears wins. +INTENT_ROUTES: List[Dict[str, Any]] = [ + { + "id": "teeth", + "keywords": ["teeth", "whiten my smile", "whiter smile", "smile looks"], + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.teeth", "params": {"desat": 0.6, "bright": 16}}, + ], + "reply": "Selecting the face and whitening teeth.", + }, + { + "id": "portrait_retouch", + "keywords": ["skin", "complexion", "blemish", "acne", "pores", "retouch", + "look better", "look my best", "make me look", "beautify", + "flawless", "smooth my face", "clear up", "clean up my face", + "portrait", "selfie"], + # Do NOT lead with a whole-image photo.autoTone: lightening the whole frame + # washes out the face (the client suppresses autoTone for portrait intents + # unless the histogram shows real under/over-exposure). Go straight to a + # gentle skin retouch; keep it subtle and never brighten the face. + "min_steps": 3, + "steps": [ + {"op": "selection.face", "params": {"mode": "replace"}}, + {"op": "retouch.skin", "params": {"smooth": 8, "opacity": 0.35, + "blemish": True, "radius": 12, + "threshold": 16}}, + {"op": "selection.deselect", "params": {}}, + ], + "reply": "Selecting the face and gently retouching the skin.", + }, + { + "id": "restore", + "keywords": ["restore", "restoration", "old photo", "old photograph", + "old picture", "faded photo", "faded picture", "faded", + "recover this photo", "bring back this photo", + "photo restoration"], + # Never a large exposure lift: recover contrast + colour from a faded + # baseline, then a slight sharpen. + "min_steps": 4, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("brightness-contrast", {"brightness": 0, "contrast": 18}), + _adj("white-balance", {"temperature": 0, "tint": 5}), + _adj("vibrance", {"vibrance": 18}), + _filter("sharpen"), + ], + "reply": "Restoring contrast and colour, then a light sharpen.", + }, + { + "id": "background", + "keywords": ["background"], + "steps": [ + {"op": "selection.background", + "params": {"mode": "replace", "subject": "person", "fuzziness": 30}}, + {"op": "filters.applyMasked", "params": {"id": "blur-gaussian"}}, + ], + "reply": "Selecting the background and softening it.", + }, + { + "id": "lighting", + # Low-key lighting: one auto-tone pass (the preferred global lift) plus + # at most one shadow-lifting, highlight-protecting adjustment. Both are + # sanctioned together; never a stack of brightness/exposure lifts. + "keywords": ["lighting", "exposure", "too dark", "underexposed", + "underexpose", "backlit", "back-lit", "brighten", "brighter", + "lift the shadows", "dark photo", "dark image"], + "min_steps": 2, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + {"op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}}, + ], + "reply": "Auto tone (levels and white balance), then one shadow lift " + "that protects the highlights.", + }, + { + "id": "cool", + "keywords": ["moodier", "moody", "cooler", "cool it", "cinematic", "teal", + "blue tone", "cold", "night mood", "colder"], + "steps": [_adj("white-balance", {"temperature": -25})], + "reply": "Cooling the white balance (negative temperature).", + }, + { + "id": "warm", + "keywords": ["warmer", "warm it", "golden", "summer", "sunset", + "golden hour", "warmth"], + "steps": [_adj("white-balance", {"temperature": 25})], + "reply": "Warming the white balance (positive temperature).", + }, + { + "id": "black_white", + "keywords": ["black and white", "black & white", "grayscale", "greyscale", + "monochrome", "b&w"], + "steps": [_adj("black-white", {})], + "reply": "Converting to black and white.", + }, + { + "id": "polish", + # Generic "make it look good" intents: a decisive multi-step grade with + # visible strength (a single timid adjustment scores as imperceptible). + "keywords": ["pretty", "beautiful", "make it look good", "look good", + "professional", "professionally", "editorial", "magazine", + "glam", "glamour", "pop", "punchy", "striking", "vibrant", + "film", "film look", "cinematic look", "instagram", + "social media", "elevate", "polish"], + "min_steps": 3, + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("vibrance", {"vibrance": 25}), + _filter("clarity", {"amount": 22, "radius": 30}), + _filter("vignette", {"amount": 22, "feather": 0.6}), + ], + "reply": "Balancing tone, adding vibrance and local contrast, with a light vignette.", + }, + { + "id": "vivid", + "keywords": ["vivid", "dramatic", "saturate", "saturation", + "colourful", "colorful"], + "steps": [_adj("vibrance", {"vibrance": 35})], + "reply": "Boosting vibrance.", + }, + { + "id": "sharpen", + "keywords": ["sharpen", "crisper", "crisp", "more detail", "sharper"], + "steps": [_filter("sharpen")], + "reply": "Sharpening.", + }, +] + +# A prompt that only asks for a vague "improvement" with no concrete defect and +# no matching intent cannot be planned deterministically -> ask the user. +_AMBIGUOUS_PATTERNS = [ + "clean this up", "clean it up", "fix it", "fix this", "make it better", + "make this better", "improve", "touch up", "touch it up", "do something", + "enhance", "make it nice", "polish", +] +_AMBIGUOUS_OPTIONS = [ + {"label": "Auto tone & color", + "description": "photo.autoTone: balanced levels, white balance, contrast."}, + {"label": "Brighten (lift shadows)", + "description": "shadows-highlights: brighten without blowing highlights."}, + {"label": "Cooler / cinematic", + "description": "white-balance negative: cooler, moodier look."}, +] +_DEFECT_HINTS = [ + "skin", "teeth", "background", "light", "dark", "colour", "color", "warm", + "cool", "blur", "sharpen", "contrast", "bright", "shadow", "face", + "blemish", "noise", "grain", "crop", "vibrance", "saturat", "black and white", + "grayscale", "portrait", "moody", "exposure", "tone", +] + +# Low-key safety: at most ONE global lift; prefer shadows/highlights and keep +# highlights protected. Global-lift adjustment types are counted together. +GLOBAL_LIFT_TYPES = ("autoTone", "brightness-contrast", "exposure", "levels", + "shadows-highlights") +_SAFE_LOWKEY_LIFT = { + "op": "adjustments.apply", + "params": {"type": "shadows-highlights", + "params": {"shadows": 30, "highlights": -15}}, +} + + +def match(prompt: str) -> Optional[Dict[str, Any]]: + """Return the first matching intent route for a prompt, or None.""" + text = str(prompt or "").lower() + for route in INTENT_ROUTES: + if any(keyword in text for keyword in route["keywords"]): + return route + return None + + +def is_ambiguous(prompt: str) -> bool: + text = str(prompt or "").lower() + if match(text): + return False + if any(hint in text for hint in _DEFECT_HINTS): + return False + return any(pattern in text for pattern in _AMBIGUOUS_PATTERNS) + + +def ask_user_plan() -> Dict[str, Any]: + question = "What would you like me to do with this photo?" + return { + "reply": question, + "planner": "fallback", + "steps": [], + "ask_user": { + "question": question, + "options": [dict(o) for o in _AMBIGUOUS_OPTIONS], + "multi": False, + }, + } + + +def has_concrete_intent(prompt: str) -> bool: + """True when a prompt maps to a concrete edit (a recipe, look, recolour, + structural op) or names a concrete defect. Used to decide whether a plan made + only of generic whole-image tone ops should be trusted or replaced by an + ask_user prompt. + """ + text = str(prompt or "").lower() + if match(text) or look_plan(text) or recolor_plan(text) or structural_plan(text): + return True + return any(hint in text for hint in _DEFECT_HINTS) + + +def fallback_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan from the guide, or None when nothing matches.""" + route = match(prompt) + if route: + return { + "reply": route["reply"], + "planner": "fallback", + "steps": enforce_guards([dict(step) for step in route["steps"]]), + } + if is_ambiguous(prompt): + return ask_user_plan() + return None + + +# ── Structural op routing ────────────────────────────────────────────── +# Deterministic plans for structural editor ops the sampling model reliably +# substitutes with a generic adjustments/transform step. Mirrors the browser +# fallback (static/js/editor/nl-edit.js localPlan) so the server and client +# agree; the server path must emit these or a wrong model plan suppresses the +# browser fallback. +_NAMED_COLORS = { + "red": "#ff0000", "blue": "#0000ff", "green": "#00a000", "yellow": "#ffff00", + "orange": "#ff8000", "purple": "#8000ff", "pink": "#ff69b4", "black": "#000000", + "white": "#ffffff", "grey": "#808080", "gray": "#808080", "cyan": "#00ffff", + "magenta": "#ff00ff", "brown": "#8b4513", "teal": "#008080", +} +_COLOR_TOKEN = (r"(red|blue|green|yellow|orange|purple|pink|black|white|grey|" + r"gray|cyan|magenta|brown|teal|#[0-9a-fA-F]{3,6})") + + +def color_for_name(name: str) -> Optional[str]: + n = str(name or "").strip().lower() + if re.match(r"^#[0-9a-f]{3,6}$", n): + return n + return _NAMED_COLORS.get(n) + + +# ── Named reference looks + targeted recolour ────────────────────────── +# Server-only deterministic plans (reached via routes._plan_special -> +# structural_plan). Deliberately NOT added to INTENT_ROUTES: the browser +# client consumes `intents` and cannot run an ai_edit_image step, so strong +# looks must stay on the server path. +_LOOK_KEEP_CLAUSE = " Keep the main subject and composition exactly the same." + +LOOKS = [ + { + "id": "wes_anderson", + "keywords": ["wes anderson"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 10, "tint": 6}), + _adj("vibrance", {"vibrance": 18}), + _adj("brightness-contrast", {"brightness": 6, "contrast": -10}), + ], + "reply": "Gave it a Wes Anderson look: pastel palette, warm highlights and soft contrast.", + }, + { + "id": "film_noir", + "keywords": ["film noir"], + "mode": "recipe", + "steps": [ + _adj("black-white", {}), + _adj("brightness-contrast", {"contrast": 38}), + _adj("shadows-highlights", {"shadows": -22, "highlights": 18}), + _filter("vignette", {"amount": 45, "feather": 0.5}), + ], + "reply": "Gave it a film-noir look: high-contrast black and white with deep shadows.", + }, + { + "id": "vintage_kodachrome", + "keywords": ["kodachrome", "vintage film"], + "mode": "recipe", + "steps": [ + {"op": "photo.autoTone", "params": {}}, + _adj("white-balance", {"temperature": 12, "tint": 8}), + _adj("vibrance", {"vibrance": 22}), + _adj("brightness-contrast", {"contrast": 14}), + _filter("vignette", {"amount": 12, "feather": 0.7}), + ], + "reply": "Gave it a vintage Kodachrome look: warm saturated colour and lifted contrast.", + }, + { + "id": "cyberpunk", + "keywords": ["cyberpunk"], + "mode": "instruction", + "prompt": "Restyle the photo as a cyberpunk scene: vivid neon magenta and cyan " + "lighting, wet reflective surfaces, deep shadows and a futuristic " + "atmosphere." + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a cyberpunk scene with the image model.", + }, + { + "id": "studio_ghibli", + "keywords": ["ghibli", "studio ghibli"], + "mode": "instruction", + "prompt": "Restyle the photo as a soft hand-painted anime scene in a " + "Studio-Ghibli-inspired pastel style: luminous pastel palette, " + "gentle painterly shading, clean edges and warm sunlight." + + _LOOK_KEEP_CLAUSE, + "reply": "Restyling it as a Studio-Ghibli-inspired pastel scene with the image model.", + }, +] + + +def look_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a named reference look, or None. + + Recipe looks (Wes Anderson, film noir, vintage Kodachrome) are pure + adjustment/filter stacks; strong looks (cyberpunk, Studio Ghibli) use + ai_edit_image instruction mode. + """ + p = str(prompt or "").lower() + for look in LOOKS: + if any(k in p for k in look["keywords"]): + if look["mode"] == "instruction": + steps = [{"op": "ai_edit_image", + "params": {"op": "generative", "mode": "instruction", + "prompt": look["prompt"], "steps": 8}}] + else: + steps = [dict(s) for s in look["steps"]] + return {"reply": look["reply"], "planner": "deterministic", "steps": steps} + return None + + +_RECOLOR_ACTION_RE = re.compile( + r"\brecolou?r\b" + r"|\bmake\s+only\b" + r"|\bmake\s+the\b[^.]{0,30}?\b" + _COLOR_TOKEN + r"\b" + r"|\b(?:change|turn|convert|paint|colou?r)\b[^.]{0,40}?\b" + _COLOR_TOKEN + r"\b", + re.I, +) +_RECOLOR_EXCLUDE_RE = re.compile( + r"\b(?:background|backdrop|blur|black\s+and\s+white|grayscale|greyscale|" + r"monochrome|b&w|text|title|font|caption|label|logo|watermark|drop\s+shadow|" + r"generative)\b", + re.I, +) + + +_RECOLOR_TRAILING_ADJECTIVES = frozenset({ + "bright", "brighter", "vivid", "vibrant", "dark", "darker", "light", + "lighter", "deep", "pale", "more", "less", "very", "really", + "slightly", "totally", "completely", +}) + + +def _hex_rgb01(hex_color: str): + h = str(hex_color or "").lstrip("#") + if len(h) == 3: + h = "".join(ch * 2 for ch in h) + try: + return tuple(int(h[i:i + 2], 16) / 255.0 for i in (0, 2, 4)) + except Exception: + return None + + +def _hue_degrees(hex_color: str) -> Optional[float]: + rgb = _hex_rgb01(hex_color) + if rgb is None: + return None + r, g, b = rgb + mx, mn = max(rgb), min(rgb) + if mx == mn: + return 0.0 + d = mx - mn + if mx == r: + h = ((g - b) / d) % 6 + elif mx == g: + h = (b - r) / d + 2 + else: + h = (r - g) / d + 4 + return h * 60.0 + + +def _hue_delta(src_hex: str, tgt_hex: str) -> int: + sh = _hue_degrees(src_hex) + th = _hue_degrees(tgt_hex) + if sh is None or th is None: + return 0 + return int(round((th - sh + 180.0) % 360.0 - 180.0)) + + +def _is_chromatic(hex_color: str) -> bool: + """True when a pure hue rotation can plausibly reach this colour. + + White / black / grey targets have (near) zero saturation and cannot be + reached by rotating hue, so they go through the image model instead. + """ + rgb = _hex_rgb01(hex_color) + if rgb is None: + return False + mx, mn = max(rgb), min(rgb) + return (mx - mn) >= 0.15 and mx >= 0.12 + + +def _recolor_object_phrase(prompt: str, target_word: str) -> str: + """Best-effort object description for the mask ("red car" in 'the red car').""" + m = re.search( + r"\b(?:make|recolou?r|change|turn|convert|paint|colou?r)\b\s+(?:only\s+)?(.+)", + prompt, re.I, + ) + phrase = (m.group(1) if m else prompt).strip() + if target_word: + phrase = re.sub(r"\b" + re.escape(target_word) + r"\b", " ", phrase, + count=1, flags=re.I) + words = " ".join(phrase.split()).strip(" .,!?").split() + if len(words) > 1 and words[0].casefold() in {"to", "into", "the", "a", "an"}: + words.pop(0) + if len(words) > 1 and words[-1].casefold() in {"to", "into"}: + words.pop() + while len(words) > 1 and words[-1].lower() in _RECOLOR_TRAILING_ADJECTIVES: + words.pop() + phrase = " ".join(words) + return phrase or "the subject" + + +def recolor_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for a targeted recolour, or None. + + 'make only the red car blue' -> select the OBJECT by description + (``selection.samText`` "red car"; falling back to a colour-range selection + via ``fallbackColor`` when the mask is empty) and then shift the MASKED + HUE toward the target with ``adjustments.applyMasked`` (preserving shading, + highlights and reflections). Never a flat ``selection.fill`` and never a + whole-image ``adjustments.apply`` tint. When the target cannot be reached by + a hue rotation (white/black/grey) or no source colour is known, the masked + region is recoloured with ``ai_edit_image`` instruction mode instead. + """ + p = str(prompt or "") + if not _RECOLOR_ACTION_RE.search(p): + return None + if _RECOLOR_EXCLUDE_RE.search(p): + return None + colors = [m.group(1).lower() for m in + re.finditer(r"\b" + _COLOR_TOKEN + r"\b", p)] + colors = [c for c in colors if color_for_name(c)] + if not colors: + return None + target = colors[-1] + tgt_hex = color_for_name(target) + source = colors[0] if len(colors) >= 2 and colors[0] != target else None + src_hex = color_for_name(source) if source else None + phrase = _recolor_object_phrase(p, target) + select = { + "op": "selection.samText", + "params": {"text": phrase}, + } + if src_hex: + select["params"]["fallbackColor"] = src_hex + select["params"]["fallbackFuzziness"] = 40 + if source and src_hex and _is_chromatic(tgt_hex): + delta = _hue_delta(src_hex, tgt_hex) + src_hue = int(round(_hue_degrees(src_hex) or 0)) + return { + "reply": "Selected the %s and shifted its hue toward %s " + "(shading kept)." % (phrase, target), + "planner": "deterministic", + "steps": [ + select, + {"op": "adjustments.applyMasked", + "params": {"hue": delta, "saturation": 0, + "mode": "hue-saturation", + "sourceHue": src_hue, + "source": src_hex, "target": tgt_hex}}, + ], + } + instruction = ("Recolour the %s to %s, preserving its shading, highlights " + "and reflections." % (phrase, target)) + return { + "reply": "Recolouring the %s with the image model." % phrase, + "planner": "deterministic", + "steps": [ + select, + {"op": "ai_edit_image", + "params": {"op": "inpaint", "mode": "masked", + "prompt": instruction, "steps": 8}}, + ], + } + + +def structural_plan(prompt: str) -> Optional[Dict[str, Any]]: + """Deterministic plan for structural editor ops, or None. + + Covers fill/flood, text.edit, colour-overlay layer style, layer align, + layer/document rotate+flip, selection copy/paste, flatten and merge-down. + Also dispatches named reference looks and targeted recolours. + Returns ``{"reply", "planner": "deterministic", "steps": [...]}``. + """ + look = look_plan(prompt) + if look is not None: + return look + recolor = recolor_plan(prompt) + if recolor is not None: + return recolor + p = str(prompt or "").lower() + steps: List[Dict[str, Any]] = [] + labels: List[str] = [] + + fm = (re.search(r"\bfill\b[^.]*?\bwith\s+" + _COLOR_TOKEN + r"\b", p) + or re.search(r"\bfill\b[^.]*?\b" + _COLOR_TOKEN + r"\b", p)) + if fm: + color = color_for_name(fm.group(1)) + if color: + if re.search(r"\bselection\b|\bselected\b", p): + steps.append({"op": "selection.fill", "params": {"color": color}}) + labels.append("filled the selection") + else: + steps.append({"op": "fill.flood", + "params": {"color": color, "x": 0, "y": 0}}) + labels.append("flood-filled the layer") + + if re.search(r"\b(text|title|caption|font|label)\b", p) and not re.search( + r"\b(add|insert|create|write|put|place)\b", p): + tparams: Dict[str, Any] = {} + if re.search(r"\b(bigger|larger|increase)\b", p): + tparams["size"] = 96 + elif re.search(r"\b(smaller|reduce|decrease)\b", p): + tparams["size"] = 28 + cm = re.search(r"\b" + _COLOR_TOKEN + r"\b", p) + if cm: + c = color_for_name(cm.group(1)) + if c: + tparams["color"] = c + if tparams: + steps.append({"op": "text.edit", "params": tparams}) + labels.append("updated the text layer") + + if re.search(r"\bcolo?ur?\s*overlay\b", p): + steps.append({"op": "layers.addStyle", + "params": {"type": "color-overlay", "params": {}}}) + labels.append("added a colour-overlay layer style") + + am = re.search( + r"\balign\b[^.]*?\b(centre|center|left|right|top|middle|bottom)\b", p) + if am: + mode = "center" if am.group(1) in ("centre", "center") else am.group(1) + steps.append({"op": "layers.align", "params": {"mode": mode}}) + labels.append("aligned to the %s" % mode) + + whole = bool(re.search( + r"\b(?:whole|entire|full)\s+(?:image|canvas|photo|picture|document)\b", p)) \ + or bool(re.search(r"\b(?:canvas|document)\b", p) + and re.search(r"\b(?:rotate|flip|mirror|turn)\b", p)) + if re.search(r"\b(?:flip|mirror)\b", p): + axis = "v" if re.search(r"\b(?:vertical(?:ly)?|v)\b", p) else "h" + if whole: + steps.append({"op": "document.flip", "params": {"axis": axis}}) + labels.append("flipped the whole image") + else: + steps.append({"op": "layer.flip", "params": {"axis": axis}}) + labels.append("flipped the layer") + # Rotation needs an explicit angle/quarter-turn or the verb "rotate"; a bare + # "turn ... into