Squash Odysseus development history

.dockerignore

@@ -30,6 +30,8 @@ secrets.env~
 .idea/
 dev-docs/
 docs/
+website/
+assets/branding/
 *.md
 *.db
 *.sqlite

.env.example

@@ -67,6 +67,11 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Auth & Security
 # ============================================================
 
+# Optional backend workspace used automatically by the WebUI when no workspace
+# is saved in the browser. This must be a directory visible to the backend;
+# with host-workspace mapping, a host path is translated before vetting.
+# ODYSSEUS_WORKSPACE_DEFAULT=/workspace/project
+
 # Enable authentication (default: true)
 # AUTH_ENABLED=true
 
@@ -88,6 +93,14 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Keep false for Docker, LAN, reverse proxy, and any shared deployment.
 # LOCALHOST_BYPASS=false
 
+# Skip the external-context exact-approval pause for unattended local agents.
+# Keep false for shared or internet-exposed deployments.
+# ODYSSEUS_UNATTENDED_MODE=false
+
+# Optional post-external-context tool approval gate. Off by default because it
+# can block normal agent work; enable only for deployments that want this fence.
+# ODYSSEUS_TOOL_APPROVAL_GATE=0
+
 # Mark session cookies Secure. Left unset, this follows the request scheme:
 # an HTTPS login gets a Secure cookie, a plain-HTTP one does not. Set true to
 # force it on, or false to force it off while you still serve plain HTTP.
@@ -238,6 +251,37 @@ SEARXNG_INSTANCE=http://localhost:8080
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
 
+# ============================================================
+# Host workspace access (explicit opt-in)
+# ============================================================
+# Docker installs normally see only the container filesystem and /app/data.
+# Enable this when the agent should edit a real host workspace like Codex.
+# This is high-trust: the mounted tree is writable by the Odysseus container.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+# ODYSSEUS_HOST_WORKSPACE_DIR=/home/you
+# ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+#
+# Host workspace access can be combined with host Docker access and GPU overlays:
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-docker.yml
+
+# ============================================================
+# Host network access (explicit opt-in, Linux Docker)
+# ============================================================
+# Docker bridge networking hides some host/LAN/VPN behavior from the agent:
+# mDNS, some LAN discovery, local VPN/Tailscale state, and host namespace
+# assumptions may differ from native Codex. Enable this only for high-trust
+# local installs where the Odysseus container should share the host network.
+#
+# With host networking, Docker port publishing is disabled and the app listens
+# directly on APP_PORT. The bundled SearXNG/Chroma services stay in Docker and
+# are reached through their host-published loopback ports.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+# APP_BIND=127.0.0.1
+# APP_PORT=7000
+# ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE=http://127.0.0.1:8080
+# ODYSSEUS_HOST_NETWORK_CHROMADB_HOST=127.0.0.1
+# ODYSSEUS_HOST_NETWORK_CHROMADB_PORT=8100
+
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================
@@ -266,3 +310,5 @@ SEARXNG_INSTANCE=http://localhost:8080
 
 # APP_DATA_DIR=./data
 # APP_LOGS_DIR=./logs
+# Maximum serialized layered photo-editor draft size (default: 256 MiB).
+ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=268435456

.github/CODEOWNERS

@@ -6,4 +6,4 @@
 # A per-area ownership map (security/auth, CI, frontend, agent internals, with
 # multiple named owners per line) is being worked out in issue #593; once
 # agreed it replaces this file. Until then, required reviews and the security
-# CI gate (docs/security-ci.md) remain in force via branch protection.
+# CI gate (website/security-ci.md) remain in force via branch protection.

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
@@ -73,10 +73,10 @@ jobs:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: "3.11"
       # Byte-compile sources — catches syntax errors without installing deps.
@@ -86,10 +86,10 @@ jobs:
     name: JS syntax (node --check)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6.4.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
           node-version: "20"
       # Syntax-check our own JS (skip vendored libs in static/lib).
@@ -105,12 +105,12 @@ jobs:
     runs-on: ubuntu-latest
     # Make Python test validation authoritative for the configured scope.
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
 
-      # Detect whether this PR only touches documentation files.
+      # Detect whether this PR only touches repository prose outside the Pages site.
       # If so, skip the expensive pytest run while still reporting a passing check.
       - name: Check for docs-only changes
         id: docs-check
@@ -122,9 +122,10 @@ jobs:
             BASE="${{ github.event.before }}"
             HEAD="${{ github.sha }}"
           fi
-          # List all changed files; if every file matches docs/markdown patterns, skip pytest.
+          # Keep website/ and assets/branding/ out of this bypass: pytest owns
+          # regression guards for their published-file and orphan-asset contracts.
           changed=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null || git diff --name-only HEAD~1 HEAD)
-          non_docs=$(echo "$changed" | grep -Ev '^(docs/|.*\.md$|\.github/[^/]+\.md$)' || true)
+          non_docs=$(echo "$changed" | grep -Ev '^(docs/|[^/]+\.md$|\.github/[^/]+\.md$)' || true)
           if [ -z "$non_docs" ]; then
             echo "docs_only=true" >> "$GITHUB_OUTPUT"
             echo "Docs-only change detected — skipping pytest."
@@ -132,7 +133,7 @@ jobs:
             echo "docs_only=false" >> "$GITHUB_OUTPUT"
           fi
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         if: steps.docs-check.outputs.docs_only != 'true'
         with:
           python-version: "3.11"

.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
         language: [actions, javascript-typescript, python]
     steps:
       - name: Checkout
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           languages: ${{ matrix.language }}
           build-mode: none
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/container-scan.yml

@@ -37,12 +37,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Lint Dockerfile
-        uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5  # v3.3.0
+        uses: hadolint/hadolint-action@2a66e89f53d0771bb131a7fa31f3136336094aa6  # v3.4.0
         with:
           dockerfile: Dockerfile
           # DL3008: pinning apt package versions is impractical on a -slim base

.github/workflows/container-trivy.yml

@@ -23,12 +23,16 @@ on:
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
   push:
     branches: [main]
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
   workflow_dispatch:
 
@@ -52,17 +56,17 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
 
       # Build without pushing so a broken Dockerfile is caught here, and the
       # exact image we ship is what gets scanned.
       - name: Build image
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           push: false
@@ -93,15 +97,15 @@ jobs:
       security-events: write  # upload SARIF to the Security tab
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
 
       - name: Build image
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           push: false
@@ -119,7 +123,7 @@ jobs:
           TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
 
       - name: Upload Trivy results
-        uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           sarif_file: trivy-results.sarif
           category: trivy-image

.github/workflows/dependency-review.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
@@ -55,12 +55,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: '3.12'
 

.github/workflows/deploy-pages.yml

@@ -0,0 +1,50 @@
+name: Deploy GitHub Pages
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'website/**'
+      - '.github/workflows/deploy-pages.yml'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Package static site
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pages: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d  # v6.0.0
+      - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697  # v1.0.13
+        with:
+          source: website
+          destination: _site
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9  # v5.0.0
+        with:
+          path: _site
+
+  deploy:
+    name: Deploy static site
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128  # v5.0.0

.github/workflows/docker-publish.yml

@@ -14,6 +14,8 @@ on:
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
 
 concurrency:
@@ -45,20 +47,20 @@ jobs:
             arch: arm64
             runner: ubuntu-24.04-arm
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
       - name: Log in to GHCR
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee  # v4.2.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push by digest
         id: build
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           platforms: ${{ matrix.platform }}
@@ -86,7 +88,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Read APP_VERSION + short sha
@@ -103,16 +105,16 @@ jobs:
           pattern: digest-*
           merge-multiple: true
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
       - name: Log in to GHCR
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee  # v4.2.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Compute tags
         id: meta
-        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9  # v6.1.0
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302  # v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |

.github/workflows/issue-description-check.yml

@@ -14,7 +14,7 @@ jobs:
     # Skip bots (Dependabot, release-drafter, etc.)
     if: ${{ github.event.issue.user.type != 'Bot' }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           sparse-checkout: .github/scripts
           persist-credentials: false

.github/workflows/pr-description-check.yml

@@ -27,7 +27,7 @@ jobs:
     # Skip bots: they open PRs programmatically and have their own process.
     if: github.event.pull_request.user.type != 'Bot'
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           ref: ${{ github.base_ref }}
           sparse-checkout: .github/scripts

.github/workflows/secret-scan.yml

@@ -35,7 +35,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           # Full history so a secret committed in an earlier commit (and later
           # deleted) is still caught -- deletion does not remove it from Git.

.github/workflows/workflow-security.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
@@ -61,12 +61,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: '3.12'
 

.gitignore

@@ -85,6 +85,24 @@ output.txt.txt
 !docs/**/*.gif
 !docs/**/*.webp
 
+# …except shipped website and branding media.
+!website/**/*.jpg
+!website/**/*.jpeg
+!website/**/*.png
+!website/**/*.gif
+!website/**/*.bmp
+!website/**/*.webp
+!website/**/*.tiff
+!website/**/*.pdf
+!assets/branding/**/*.jpg
+!assets/branding/**/*.jpeg
+!assets/branding/**/*.png
+!assets/branding/**/*.gif
+!assets/branding/**/*.bmp
+!assets/branding/**/*.webp
+!assets/branding/**/*.tiff
+!assets/branding/**/*.pdf
+
 # Reports and temp files
 reports/
 tasks/

Dockerfile

@@ -18,6 +18,10 @@ FROM python:3.14-slim
 # launch inside Docker.
 # nodejs/npm provide npx for the built-in Browser MCP server.
 # chromium provides the actual browser binary used by that MCP server.
+# fontconfig + Noto CJK provide real fallback glyphs for multilingual pages;
+# Chromium otherwise renders Chinese/Japanese/Korean labels as empty boxes.
+# iproute2/iputils-ping/net-tools/dnsutils/nmap give Docker-hosted agents the
+# basic network inspection toolkit expected by local LAN/debugging tasks.
 # gosu lets the entrypoint drop privileges cleanly so signals still reach
 # uvicorn directly (no extra shell layer like `su`/`sudo` would add).
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -28,8 +32,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     nodejs \
     npm \
     chromium \
+    fontconfig \
+    fonts-noto-cjk \
     tmux \
     openssh-client \
+    iproute2 \
+    iputils-ping \
+    net-tools \
+    dnsutils \
+    nmap \
     gosu \
     libgl1 \
     libglib2.0-0t64 \
@@ -37,6 +48,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libmagic1 \
     && rm -rf /var/lib/apt/lists/*
 
+# Private browser automation wrapper used by the native `private_browser` tool.
+# Chromium is installed above, so agent-browser can drive the existing browser
+# binary without paying `npx` startup/install overhead on each tool call.
+RUN npm install -g agent-browser@0.35.0 --omit=dev --loglevel=error
+
 # libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
 # libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
 # slim base omits them, so the Cookbook "install realesrgan" path imports cv2

HARNESS_VERSION

@@ -0,0 +1 @@
+0.20.5

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
+  <img src="assets/branding/odysseus-wordmark.png" alt="Odysseus" width="238">
 </p>
 
 <p align="center">
@@ -8,7 +8,7 @@
 
 <p align="center">
   <a href="#quick-start">Quick Start</a> ·
-  <a href="docs/setup.md">Setup Guide</a> ·
+  <a href="website/setup.md">Setup Guide</a> ·
   <a href="CONTRIBUTING.md">Contributing</a> ·
   <a href="ROADMAP.md">Roadmap</a>
 </p>
@@ -18,7 +18,7 @@
 </p>
 
 <p align="center">
-  <img src="docs/odysseus-browser.jpg" alt="Odysseus interface">
+  <img src="assets/branding/odysseus-browser.jpg" alt="Odysseus interface">
 </p>
 
 ---
@@ -36,7 +36,7 @@ docker compose up -d --build
 
 Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`.
 
-Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md).
+Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](website/setup.md).
 
 ## Features
 
@@ -51,7 +51,7 @@ Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration
 
 ## Demo
 
-A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html).
+A full hover-to-play tour lives on the [Odysseus landing page](https://odysseus-dev.github.io/odysseus/). Its source lives under [`website/`](website/).
 
 ## Contributing
 
@@ -64,7 +64,7 @@ Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
 
-Deployment details are in the [setup guide](docs/setup.md#security-notes).
+Deployment details are in the [setup guide](website/setup.md#security-notes).
 
 ## Star History
 

app.py

@@ -4,6 +4,8 @@ import os
 import sys
 import asyncio
 import time
+import shutil
+import socket
 
 # On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
 # When started via `python -m uvicorn` from a terminal, uvicorn sets this
@@ -160,7 +162,8 @@ app.add_middleware(
 # model-probe — all served with media_type="text/event-stream") are never
 # compressed or buffered; only complete bodies over minimum_size are. The
 # security-header middleware composes cleanly on top.
-app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
+if os.getenv("RESPONSE_COMPRESSION_ENABLED", "true").strip().lower() not in {"0", "false", "no", "off"}:
+    app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
 
 # ========= SECURITY HEADERS MIDDLEWARE =========
 app.add_middleware(SecurityHeadersMiddleware)
@@ -684,6 +687,7 @@ app.include_router(setup_session_routes(
     session_config,
     webhook_manager=webhook_manager,
     upload_handler=upload_handler,
+    skills_manager=skills_manager,
 ))
 
 # Admin Danger Zone wipes (Settings → System → Danger Zone)
@@ -949,8 +953,12 @@ async def serve_login(request: Request):
 
 @app.get("/api/version")
 async def get_version():
-    from core.constants import APP_VERSION
-    return {"version": APP_VERSION}
+    from core.constants import APP_BUILD_VERSION, APP_SOURCE_COMMIT, APP_VERSION
+    return {
+        "version": APP_VERSION,
+        "build": APP_BUILD_VERSION,
+        "source_commit": APP_SOURCE_COMMIT,
+    }
 
 @app.get("/api/health")
 async def health_check() -> Dict[str, str]:
@@ -1010,11 +1018,76 @@ async def runtime_info() -> Dict[str, object]:
         or os.getenv("OLLAMA_URL")
         or ("http://host.docker.internal:11434/v1" if in_docker else "http://127.0.0.1:11434/v1")
     )
+    network_mode = os.getenv("ODYSSEUS_CONTAINER_NETWORK_MODE", "").strip()
+    host_gateway_reachable = False
+    host_gateway_address = ""
+    if in_docker and network_mode != "host":
+        try:
+            resolved = socket.getaddrinfo("host.docker.internal", None)
+            for item in resolved:
+                sockaddr = item[4] if len(item) >= 5 else ()
+                candidate = sockaddr[0] if sockaddr else ""
+                if candidate:
+                    host_gateway_address = str(candidate)
+                    break
+            host_gateway_reachable = True
+        except OSError:
+            host_gateway_reachable = False
+        if not host_gateway_address:
+            host_gateway_address = _docker_default_gateway_ip()
+    container: Dict[str, object] = {
+        "engine": "docker" if in_docker else "",
+        "networkMode": network_mode,
+        "hostAccess": bool(in_docker and network_mode == "host"),
+        "hostGatewayReachable": host_gateway_reachable,
+    }
+    if host_gateway_address:
+        container["hostGatewayAddress"] = host_gateway_address
+    command_names = (
+        "ip",
+        "ss",
+        "arp",
+        "nmap",
+        "ping",
+        "dig",
+        "ssh",
+        "git",
+        "docker",
+    )
+    commands = {name: bool(shutil.which(name)) for name in command_names}
+    capabilities = {
+        "networkInspection": bool(commands["ip"] and (commands["ss"] or commands["arp"])),
+        "lanScan": bool(commands["nmap"]),
+        "dnsLookup": bool(commands["dig"]),
+        "sshClient": bool(commands["ssh"]),
+        "git": bool(commands["git"]),
+        "dockerClient": bool(commands["docker"]),
+    }
     return {
         "in_docker": in_docker,
         "ollama_base_url": ollama_url,
+        "container": container,
+        "commands": commands,
+        "capabilities": capabilities,
     }
 
+
+def _docker_default_gateway_ip() -> str:
+    try:
+        with open("/proc/net/route", "r", encoding="utf-8", errors="ignore") as fh:
+            for line in fh.readlines()[1:]:
+                parts = line.split()
+                if len(parts) < 3 or parts[1] != "00000000":
+                    continue
+                raw = parts[2]
+                if len(raw) != 8:
+                    continue
+                octets = [str(int(raw[i:i + 2], 16)) for i in range(6, -1, -2)]
+                return ".".join(octets)
+    except Exception:
+        return ""
+    return ""
+
 # ========= LIFECYCLE =========
 
 @asynccontextmanager
@@ -1054,6 +1127,15 @@ async def _startup_event():
     # GC tasks created with `asyncio.create_task(...)` before they finish.
     _startup_tasks: list[asyncio.Task] = getattr(app.state, "_startup_tasks", [])
     app.state._startup_tasks = _startup_tasks
+    from src.background_tool_jobs import BackgroundToolJobs
+    from routes.chat_routes import _active_streams
+    from src import agent_runs
+    app.state.background_tool_jobs = BackgroundToolJobs(
+        is_busy=lambda sid: sid in _active_streams or agent_runs.is_active(sid),
+        session_manager=session_manager, research_handler=research_handler,
+    )
+    app.state.background_tool_delivery_task = asyncio.create_task(app.state.background_tool_jobs.run())
+    _startup_tasks.append(app.state.background_tool_delivery_task)
     if upload_cleanup_func:
         upload_cleanup_task = asyncio.create_task(upload_cleanup_func())
     # Always-on monitor that auto-continues the agent when a background bash
@@ -1080,23 +1162,34 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
 
-    # Startup warmups are opt-in. They make later requests a little warmer, but
-    # they also compete with the first seconds of real UI use on slow or busy
-    # machines. Default to clear/idle startup and let requests warm what they use.
-    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
-    if _startup_warmups_enabled:
+    # Semantic tool selection is part of the agent serving contract. Initialize
+    # it in a background thread by default so startup remains nonblocking while
+    # harness deployments can wait for the explicit readiness state.
+    from src.tool_index import prewarm_tool_index, tool_index_prewarm_enabled
+    if tool_index_prewarm_enabled():
         async def _warmup_tool_index():
-            try:
-                from src.tool_index import get_tool_index
-                idx = await asyncio.to_thread(get_tool_index)
-                if idx:
-                    await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
-                    logger.info("[startup] Tool index pre-warmed")
-            except Exception as e:
-                logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
+            status = await asyncio.to_thread(prewarm_tool_index)
+            if status.get("ready"):
+                logger.info(
+                    "[startup] Tool index pre-warmed lanes=%s tools=%s duration_ms=%s",
+                    [lane.get("name") for lane in status.get("lanes", [])],
+                    status.get("builtin_tools"),
+                    status.get("duration_ms"),
+                )
+            else:
+                logger.warning(
+                    "Tool index warmup degraded (non-critical): %s",
+                    status.get("error_type") or status.get("state"),
+                )
 
         _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
+    else:
+        logger.info("Tool index prewarm disabled (ODYSSEUS_TOOL_INDEX_PREWARM=0)")
 
+    # Model endpoint pings remain opt-in. They can compete with the first seconds
+    # of UI use on slow or busy machines and are not required for local startup.
+    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
+    if _startup_warmups_enabled:
         async def _warmup_endpoints():
             try:
                 import httpx
@@ -1116,7 +1209,7 @@ async def _startup_event():
 
         _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
     else:
-        logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
+        logger.info("Model endpoint warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
 
     # Keep-alive is opt-in. The ping path performs model discovery, and when
     # stale LAN endpoints are configured it can add periodic backend pressure
@@ -1184,6 +1277,14 @@ async def _startup_event():
     # Disk-backed skills are not covered by the DB legacy-owner sweep. Repair
     # ownerless or deleted/test-owner SKILL.md files so strict owner filtering
     # does not make an existing library look empty after auth/account changes.
+    try:
+        from services.memory.builtin_skills import install_builtin_skills
+        installed = install_builtin_skills(skills_manager, ())
+        if installed:
+            logger.info("Installed %s built-in skill file(s)", installed)
+    except Exception as e:
+        logger.debug(f"Built-in skill installation skipped: {e}")
+
     try:
         import json as _json
         auth_path = AUTH_FILE
@@ -1229,35 +1330,10 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_null_owner_sweep_loop()))
 
-    # Nightly skill audit — at ~02:00 local, test + judge a batch of the
-    # least-recently-checked skills, auto-fixing/escalating weak ones (never
-    # deletes). Rotates through the library so each night covers different
-    # skills. Gated by the `skill_audit_nightly` setting (default on); hour via
-    # `skill_audit_hour` (default 2), batch size via `skill_audit_batch` (8).
-    async def _skill_audit_nightly_loop():
-        from datetime import timedelta
-        while True:
-            try:
-                from src.settings import get_setting
-                hour = int(get_setting("skill_audit_hour", 2) or 2)
-            except Exception:
-                hour = 2
-            now = datetime.now()
-            nxt = now.replace(hour=hour % 24, minute=0, second=0, microsecond=0)
-            if nxt <= now:
-                nxt += timedelta(days=1)
-            await asyncio.sleep(max(60, (nxt - now).total_seconds()))
-            try:
-                from src.settings import get_setting
-                if not get_setting("skill_audit_nightly", True):
-                    continue
-                batch = int(get_setting("skill_audit_batch", 8) or 8)
-                from routes.skills_routes import run_scheduled_skill_audit
-                await run_scheduled_skill_audit(skills_manager, owner=None, max_skills=batch)
-            except Exception as e:
-                logger.warning(f"Nightly skill audit failed: {e}")
-
-    _startup_tasks.append(asyncio.create_task(_skill_audit_nightly_loop()))
+    # Skills Audit is scheduled per owner by TaskScheduler. Do not also start
+    # an ownerless audit here: its sidecar results cannot be read back through
+    # an authenticated owner's skill namespace, and its model activity can
+    # defer the real per-owner task at the same time of night.
 
     # Cookbook serve lifecycle — kills scheduler-launched serves whose
     # window-end has passed. Paired with the cookbook_serve builtin
@@ -1272,6 +1348,18 @@ async def _startup_event():
 
 async def _shutdown_event():
     logger.info("Application shutting down...")
+    background_delivery = getattr(app.state, 'background_tool_delivery_task', None)
+    if background_delivery:
+        background_delivery.cancel()
+        try:
+            await background_delivery
+        except asyncio.CancelledError:
+            pass
+    try:
+        from src.agent_tools.web_tools import shutdown_private_browser_sessions
+        await shutdown_private_browser_sessions()
+    except Exception as e:
+        logger.warning(f"Private browser shutdown error: {e}")
     if upload_cleanup_task:
         upload_cleanup_task.cancel()
         try:
@@ -1300,6 +1388,6 @@ if __name__ == "__main__":
     import uvicorn
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
 
     uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

build-macos-app.sh

@@ -27,13 +27,13 @@ echo "  port:        $PORT"
 rm -rf "$APP"
 mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources"
 
-# ── Icon (best effort) — center-crop docs/odysseus.jpg to a square .icns ──
-if [ -f "$REPO_DIR/docs/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
+# ── Icon (best effort) — center-crop the branding image to a square .icns ──
+if [ -f "$REPO_DIR/assets/branding/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
   TMPIMG="$(mktemp -d)"
   # Center-crop to a square, scale to 512 (sips' icns encoder caps at 512), and
   # let sips emit the .icns directly — more robust across macOS versions than
   # building an .iconset by hand.
-  sips -c 720 720 "$REPO_DIR/docs/odysseus.jpg" --out "$TMPIMG/sq.png" >/dev/null 2>&1 || cp "$REPO_DIR/docs/odysseus.jpg" "$TMPIMG/sq.png"
+  sips -c 720 720 "$REPO_DIR/assets/branding/odysseus.jpg" --out "$TMPIMG/sq.png" >/dev/null 2>&1 || cp "$REPO_DIR/assets/branding/odysseus.jpg" "$TMPIMG/sq.png"
   sips -z 512 512 "$TMPIMG/sq.png" --out "$TMPIMG/icon.png" >/dev/null 2>&1
   if sips -s format icns "$TMPIMG/icon.png" --out "$APP/Contents/Resources/odysseus.icns" >/dev/null 2>&1; then
     echo "  icon:        odysseus.icns"
@@ -42,7 +42,7 @@ if [ -f "$REPO_DIR/docs/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
   fi
   rm -rf "$TMPIMG"
 else
-  echo "  icon:        (skipped — no docs/odysseus.jpg)"
+  echo "  icon:        (skipped — no assets/branding/odysseus.jpg)"
 fi
 
 # ── Info.plist ──

core/atomic_io.py

@@ -30,11 +30,20 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
     """
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = f"{path}.tmp.{uuid.uuid4().hex}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=indent)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=indent)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
 
 
 def atomic_write_text(path: str, text: str) -> None:
@@ -42,8 +51,17 @@ def atomic_write_text(path: str, text: str) -> None:
         raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = f"{path}.tmp.{uuid.uuid4().hex}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        f.write(text)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            f.write(text)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass

core/database.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.parse import unquote, urlparse
-from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text
+from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, Float, ForeignKey, JSON, Index, func, inspect, text
 from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
@@ -75,7 +75,7 @@ DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database
 # Create engine
 engine = create_engine(
     DATABASE_URL,
-    connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
+    connect_args={"check_same_thread": False, "timeout": 30} if "sqlite" in DATABASE_URL else {}
 )
 
 
@@ -144,6 +144,8 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     if isinstance(dbapi_connection, sqlite3.Connection):
         cursor = dbapi_connection.cursor()
         cursor.execute("PRAGMA foreign_keys=ON")
+        cursor.execute("PRAGMA busy_timeout=30000")
+        cursor.execute("PRAGMA journal_mode=WAL")
         cursor.close()
 
 
@@ -191,9 +193,15 @@ class Session(TimestampMixin, Base):
     # Configuration flags
     rag = Column(Boolean, default=False)
     archived = Column(Boolean, default=False)
+    memory_extraction_enabled = Column(Boolean, default=True)
+    skill_injection_enabled = Column(Boolean, default=True)
+    thinking_mode = Column(String, nullable=True, default="off")
+    temperature_override = Column(Float, nullable=True, default=None)
+    max_tokens_override = Column(Integer, nullable=True, default=None)
 
     # Organization
     folder = Column(String, nullable=True, default=None)
+    cwd = Column(String, nullable=True, default=None)
     
     # Headers stored as JSON
     headers = Column(JSON, default=dict)
@@ -219,6 +227,7 @@ class Session(TimestampMixin, Base):
     message_count = Column(Integer, default=0)
     total_input_tokens = Column(Integer, default=0)
     total_output_tokens = Column(Integer, default=0)
+    total_cost_usd = Column(Float, default=0.0)
     mode = Column(String, nullable=True)  # 'agent', 'chat', or 'research'
     crew_member_id = Column(String, nullable=True)  # links to crew_members.id
 
@@ -239,6 +248,11 @@ class Session(TimestampMixin, Base):
             'endpoint_url': self.endpoint_url,
             'rag': self.rag,
             'archived': self.archived,
+            'memory_extraction_enabled': self.memory_extraction_enabled is not False,
+            'skill_injection_enabled': self.skill_injection_enabled is not False,
+            'thinking_mode': self.thinking_mode or '',
+            'temperature_override': self.temperature_override,
+            'max_tokens_override': self.max_tokens_override,
             'created_at': self.created_at.isoformat() if self.created_at else None,
             'updated_at': self.updated_at.isoformat() if self.updated_at else None,
             'last_accessed': self.last_accessed.isoformat() if self.last_accessed else None,
@@ -248,6 +262,7 @@ class Session(TimestampMixin, Base):
             'folder': self.folder,
             'total_input_tokens': self.total_input_tokens or 0,
             'total_output_tokens': self.total_output_tokens or 0,
+            'total_cost_usd': self.total_cost_usd or 0.0,
             'crew_member_id': self.crew_member_id,
         }
 
@@ -280,6 +295,22 @@ class ChatMessage(Base):
         Index('ix_messages_session_time', 'session_id', 'timestamp'),  # Composite for efficient message retrieval
     )
 
+class BackgroundToolJob(Base):
+    """Durable origin and once-only chat delivery for background tool work."""
+    __tablename__ = "background_tool_jobs"
+    id = Column(String, primary_key=True)
+    session_id = Column(String, ForeignKey("sessions.id", ondelete="CASCADE"), nullable=False, index=True)
+    owner = Column(String, nullable=False, index=True)
+    tool = Column(String, nullable=False)
+    query = Column(Text, nullable=False)
+    rounds = Column(Integer, nullable=True)
+    status = Column(String, nullable=False, default="running", index=True)
+    payload = Column(Text, nullable=True)
+    summary = Column(Text, nullable=True)
+    message_id = Column(String, nullable=True)
+    created_at = Column(DateTime, default=utcnow_naive)
+
+
 class Document(TimestampMixin, Base):
     """Living document that the AI can create and edit in-place."""
     __tablename__ = "documents"
@@ -544,6 +575,9 @@ class ModelEndpoint(TimestampMixin, Base):
     # can be toggled per-endpoint in the UI. NULL = unknown, falls
     # back to the model-name keyword heuristic in agent_loop.py.
     supports_tools = Column(Boolean, nullable=True, default=None)
+    # JSON object: model id -> native tool schema surface preference.
+    # Values: none, compact, full. Missing key = legacy automatic behavior.
+    model_tool_modes = Column(Text, nullable=True)
     # Per-user ownership. NULL = legacy/shared (visible to every user) — this
     # is the historical default. When non-null, the model picker only shows
     # the endpoint to that user (admins always see everything).
@@ -830,6 +864,23 @@ class TaskRun(Base):
     )
 
 
+class NotificationLog(Base):
+    """Persisted task notifications, including completion and error text."""
+    __tablename__ = "notification_logs"
+
+    id         = Column(String, primary_key=True, index=True)
+    owner      = Column(String, nullable=True, index=True)
+    task_name  = Column(String, nullable=False)
+    task_id    = Column(String, nullable=True, index=True)
+    status     = Column(String, nullable=False, default="success")
+    body       = Column(Text, nullable=True)
+    timestamp  = Column(DateTime, nullable=False, default=utcnow_naive, index=True)
+
+    __table_args__ = (
+        Index('ix_notification_logs_owner_time', 'owner', 'timestamp'),
+    )
+
+
 class Memory(Base):
     """
     SQLAlchemy model for Memory table.
@@ -910,6 +961,74 @@ def _migrate_add_last_message_at_column():
         except Exception:
             pass
 
+def _migrate_add_memory_extraction_enabled_column():
+    """Add per-session auto memory extraction toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "memory_extraction_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN memory_extraction_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added memory_extraction_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"memory_extraction_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_skill_injection_enabled_column():
+    """Add per-session skill injection toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "skill_injection_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN skill_injection_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added skill_injection_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"skill_injection_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_session_generation_settings_columns():
+    """Add per-chat model generation controls."""
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = {row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()}
+        additions = {
+            "thinking_mode": "VARCHAR DEFAULT 'off'",
+            "temperature_override": "FLOAT",
+            "max_tokens_override": "INTEGER",
+        }
+        for name, sql_type in additions.items():
+            if name not in columns:
+                conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {sql_type}")
+        conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"session generation settings migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
 def _migrate_add_document_archived_column():
     """Add `archived` to documents (soft-archive flag). Guarded + idempotent."""
     import sqlite3
@@ -1159,6 +1278,30 @@ def _migrate_add_supports_tools_column():
             pass
 
 
+def _migrate_add_model_tool_modes_column():
+    """Add per-model tool-surface preferences to model_endpoints if missing."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(model_endpoints)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if columns and "model_tool_modes" not in columns:
+            conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_tool_modes TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'model_tool_modes' column to model_endpoints")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"model_tool_modes migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+
 def _migrate_add_cached_models_column():
     """Add cached_models column to model_endpoints if it doesn't exist."""
     import sqlite3
@@ -1282,6 +1425,29 @@ def _migrate_add_folder_column():
         except Exception:
             pass
 
+def _migrate_add_session_cwd_column():
+    """Add cwd column to sessions table if it doesn't exist."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "cwd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN cwd TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'cwd' column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for cwd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_token_columns():
     """Add cumulative token tracking columns to sessions table."""
     import sqlite3
@@ -1306,6 +1472,29 @@ def _migrate_add_token_columns():
         except Exception:
             pass
 
+def _migrate_add_total_cost_usd():
+    """Add cumulative USD cost column to sessions table."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "total_cost_usd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN total_cost_usd REAL DEFAULT 0.0")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added total_cost_usd column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for total_cost_usd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_owner_to_table(table_name: str, index_name: str):
     """Generic helper: add owner TEXT column + index to a table if missing."""
     import sqlite3
@@ -1824,6 +2013,7 @@ class Note(TimestampMixin, Base):
     session_id = Column(String, nullable=True)
     sort_order = Column(Integer, default=0)
     image_url  = Column(String, nullable=True)      # uploaded image URL (relative path)
+    gallery_id = Column(String, nullable=True, index=True)  # stable Gallery image for drawings
     repeat     = Column(String, default="none")     # none, daily, weekly, monthly, yearly
     # Auto-AI fields — populated by /api/notes/{id}/classify. The classification
     # JSON shape is { kind, solvable, confidence, task_prompt, tools, items?: [...] }.
@@ -2109,12 +2299,18 @@ def init_db():
     _migrate_add_model_endpoint_owner_column()
     _migrate_add_provider_auth_id_column()
     _migrate_add_supports_tools_column()
+    _migrate_add_model_tool_modes_column()
     _migrate_add_task_run_model_column()
     _migrate_add_owner_column()
     _migrate_add_document_archived_column()
     _migrate_add_last_message_at_column()
+    _migrate_add_memory_extraction_enabled_column()
+    _migrate_add_skill_injection_enabled_column()
+    _migrate_add_session_generation_settings_columns()
     _migrate_add_folder_column()
+    _migrate_add_session_cwd_column()
     _migrate_add_token_columns()
+    _migrate_add_total_cost_usd()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
     _migrate_add_gallery_caption_column()
@@ -2142,6 +2338,7 @@ def init_db():
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
     _migrate_add_calendar_recurrence_exdates()
+    _migrate_add_note_gallery_id()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2239,17 +2436,33 @@ def _migrate_chat_messages_fts():
             END;
             """
         )
-        conn.execute(
-            f"""
-            INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-            SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
-            FROM chat_messages cm
-            WHERE NOT EXISTS (
-                SELECT 1 FROM chat_messages_fts fts
-                WHERE fts.message_id = cm.id
+        # message_id is deliberately UNINDEXED in the FTS table.  A correlated
+        # NOT EXISTS against it therefore becomes quadratic once the transcript
+        # grows large, even when there is nothing left to backfill.  Build a
+        # temporary indexed set only when the row counts show that reconciliation
+        # is needed.  Normal inserts/updates/deletes stay synchronized by the
+        # triggers above.
+        chat_count = conn.execute("SELECT COUNT(*) FROM chat_messages").fetchone()[0]
+        fts_count = conn.execute("SELECT COUNT(*) FROM chat_messages_fts").fetchone()[0]
+        if chat_count != fts_count:
+            conn.execute(
+                "CREATE TEMP TABLE IF NOT EXISTS _odysseus_fts_message_ids "
+                "(message_id TEXT PRIMARY KEY) WITHOUT ROWID"
+            )
+            conn.execute("DELETE FROM temp._odysseus_fts_message_ids")
+            conn.execute(
+                "INSERT OR IGNORE INTO temp._odysseus_fts_message_ids(message_id) "
+                "SELECT message_id FROM chat_messages_fts"
+            )
+            conn.execute(
+                f"""
+                INSERT INTO chat_messages_fts(content, message_id, session_id, role)
+                SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
+                FROM chat_messages cm
+                LEFT JOIN temp._odysseus_fts_message_ids known ON known.message_id = cm.id
+                WHERE known.message_id IS NULL
+                """
             )
-            """
-        )
         _scrub_legacy_chat_message_fts_media(conn)
         conn.commit()
     except Exception as e:
@@ -2565,6 +2778,27 @@ def _migrate_add_calendar_recurrence_exdates():
         except Exception:
             pass
 
+def _migrate_add_note_gallery_id():
+    """Keep a drawn note linked to one Gallery image across edits."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(notes)").fetchall()]
+        if columns and "gallery_id" not in columns:
+            conn.execute("ALTER TABLE notes ADD COLUMN gallery_id VARCHAR")
+            conn.execute("CREATE INDEX IF NOT EXISTS ix_notes_gallery_id ON notes(gallery_id)")
+            conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"notes gallery_id migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
+
 def get_db():
     """
     Dependency to get a database session.

core/models.py

@@ -8,6 +8,11 @@ These are simple datacontainers. All persistence is handled by SessionManager.
 from dataclasses import dataclass
 from typing import Dict, List, Any, Optional, TYPE_CHECKING
 
+from src.tool_approval_scopes import (
+    CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
+    CHAT_SESSION_APPROVAL_DECISION,
+)
+
 if TYPE_CHECKING:
     from .session_manager import SessionManager
 
@@ -31,6 +36,35 @@ set_session_manager = set_session_manager_instance
 get_session_manager = get_session_manager_instance
 
 
+def _history_grants_chat_session_approval(
+    history: List["ChatMessage"],
+    session_id: str,
+) -> bool:
+    """Return whether this exact chat has a resolved session-scope grant."""
+
+    expected_session = str(session_id or "")
+    if not expected_session:
+        return False
+    for message in reversed(history or []):
+        metadata = getattr(message, "metadata", None)
+        if not isinstance(metadata, dict):
+            continue
+        tool_events = metadata.get("tool_events")
+        if not isinstance(tool_events, list):
+            continue
+        for event in reversed(tool_events):
+            ask_user = event.get("ask_user") if isinstance(event, dict) else None
+            if not isinstance(ask_user, dict):
+                continue
+            if (
+                ask_user.get("kind") == "tool_approval"
+                and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
+                and str(ask_user.get("session_id") or "") == expected_session
+            ):
+                return True
+    return False
+
+
 @dataclass
 class ChatMessage:
     """A single chat message."""
@@ -74,6 +108,12 @@ class Session:
     owner: Optional[str] = None
     is_important: bool = False
     message_count: int = 0
+    memory_extraction_enabled: bool = True
+    skill_injection_enabled: bool = True
+    thinking_mode: str = "off"
+    temperature_override: Optional[float] = None
+    max_tokens_override: Optional[int] = None
+    cwd: Optional[str] = None
 
     def __post_init__(self):
         if self.headers is None:
@@ -116,11 +156,45 @@ class Session:
         the model. Display/history-load paths use the raw ``history`` and are
         unaffected.
         """
-        return [
+        messages = [
             msg.to_dict()
             for msg in self.history
             if (msg.metadata or {}).get("source") != "slash"
         ]
+        from src.background_tool_jobs import background_result_context
+        messages = [part for message in messages for part in (
+            *background_result_context(message.get('metadata')), message,
+        )]
+        # Resume an interrupted thinking-only response from its actual model
+        # reasoning channel. Restrict this to the latest assistant message so
+        # old traces do not accumulate in context or cause reasoning loops.
+        for index in range(len(messages) - 1, -1, -1):
+            message = messages[index]
+            if message.get("role") != "assistant":
+                continue
+            metadata = message.get("metadata") or {}
+            thinking = str(metadata.get("thinking") or "").strip()
+            if metadata.get("stopped") and thinking:
+                resumed = dict(message)
+                resumed["reasoning_content"] = thinking
+                messages[index] = resumed
+            break
+        if not _history_grants_chat_session_approval(self.history, self.id):
+            return messages
+
+        # Keep the grant close to the latest user request so route-neutral
+        # compaction/trimming preserves it. Copy the metadata instead of
+        # mutating the durable transcript object.
+        for index in range(len(messages) - 1, -1, -1):
+            if messages[index].get("role") != "user":
+                continue
+            message = dict(messages[index])
+            metadata = dict(message.get("metadata") or {})
+            metadata[CHAT_SESSION_APPROVAL_CONTEXT_MARKER] = True
+            message["metadata"] = metadata
+            messages[index] = message
+            break
+        return messages
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""

core/session_manager.py

@@ -150,6 +150,12 @@ class SessionManager:
             history=[],
             owner=getattr(db_session, "owner", None),
             is_important=getattr(db_session, "is_important", False) or False,
+            memory_extraction_enabled=getattr(db_session, "memory_extraction_enabled", True) is not False,
+            skill_injection_enabled=getattr(db_session, "skill_injection_enabled", True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
         session.message_count = getattr(db_session, "message_count", 0) or 0
         return session
@@ -208,6 +214,12 @@ class SessionManager:
             history=history,
             owner=getattr(db_session, 'owner', None),
             is_important=getattr(db_session, 'is_important', False) or False,
+            memory_extraction_enabled=getattr(db_session, 'memory_extraction_enabled', True) is not False,
+            skill_injection_enabled=getattr(db_session, 'skill_injection_enabled', True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
 
         # The rows just loaded are the whole transcript, so they — not the
@@ -485,6 +497,7 @@ class SessionManager:
             session.archived = db_session.archived
             session.owner = getattr(db_session, "owner", None)
             session.is_important = getattr(db_session, "is_important", False) or False
+            session.cwd = getattr(db_session, "cwd", None) or None
             session.message_count = (
                 db.query(DbChatMessage)
                 .filter(DbChatMessage.session_id == session_id)
@@ -545,9 +558,12 @@ class SessionManager:
         endpoint_url: str,
         model: str,
         rag: bool = False,
-        owner: str = None
+        owner: str = None,
+        cwd: str = None,
+        headers: Optional[Dict[str, str]] = None,
     ) -> Session:
         """Create a new session and save to database."""
+        session_headers = dict(headers or {})
         db = SessionLocal()
         try:
             db_session = DbSession(
@@ -556,8 +572,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
                 created_at=datetime.now(timezone.utc),
                 updated_at=datetime.now(timezone.utc)
             )
@@ -570,8 +587,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
             )
 
             self.sessions[session_id] = session

docker-compose.gpu-amd.yml

@@ -14,7 +14,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -59,6 +59,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -66,9 +71,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.gpu-nvidia.yml

@@ -13,7 +13,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -58,6 +58,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -65,9 +70,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker-compose.yml

@@ -2,7 +2,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -47,6 +47,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -54,9 +59,16 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
       - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

docker/host-network.yml

@@ -0,0 +1,21 @@
+# High-trust host network access. Enable only when the Odysseus agent needs
+# host-native LAN/VPN/mDNS behavior that Docker bridge networking cannot
+# provide. Linux only; Docker Desktop does not provide equivalent host
+# networking semantics.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+#   APP_PORT=7011
+services:
+  odysseus:
+    network_mode: host
+    ports: !reset []
+    environment:
+      - APP_PORT=${APP_PORT:-7011}
+      - APP_BIND=${APP_BIND:-0.0.0.0}
+      - SEARXNG_INSTANCE=${ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE:-http://127.0.0.1:8080}
+      - CHROMADB_HOST=${ODYSSEUS_HOST_NETWORK_CHROMADB_HOST:-127.0.0.1}
+      - CHROMADB_PORT=${ODYSSEUS_HOST_NETWORK_CHROMADB_PORT:-8100}
+      - ODYSSEUS_CONTAINER_NETWORK_MODE=host
+    command:
+      - sh
+      - -c
+      - exec uvicorn app:app --host "$${APP_BIND:-0.0.0.0}" --port "$${APP_PORT:-7011}"

docker/host-workspace.yml

@@ -0,0 +1,11 @@
+# High-trust host workspace access. Enable only when the Odysseus agent should
+# work on a host directory outside the container's normal /app/data sandbox.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+#   ODYSSEUS_HOST_WORKSPACE_DIR=/absolute/host/path
+#   ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+services:
+  odysseus:
+    volumes:
+      - ${ODYSSEUS_HOST_WORKSPACE_DIR:?set ODYSSEUS_HOST_WORKSPACE_DIR}:${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}:rw,z
+    environment:
+      - ODYSSEUS_HOST_WORKSPACE_MOUNT=${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}

docs/AGENT_TURN_CONTRACT.md

@@ -0,0 +1,75 @@
+# Agent turn contract
+
+Scope: product Agent turns on 7011. Environment-owned native/TUI bridges retain
+their existing execution contract. No model weights or training settings change.
+
+## Boundaries
+
+1. `src/turn_contract.py` classifies capabilities, including explicit compound
+   requests and referential follow-ups. Classification is selection, not permission.
+2. `routes/chat_routes.py` resolves toggles, privileges, global/plan/incognito
+   restrictions, fixture restrictions and available schema inventory before
+   freezing the offered set. Web enabled alone does not select web tools.
+3. `TurnContract` checks `required <= offered <= executable`, stores immutable
+   serialized schema copies, and records unavailable requirements. An unavailable
+   request stops without inference or substitution; unknown actions ask for clarity.
+   Exact account-discovery requests narrow selection to account metadata only;
+   compounds retain their declared family scope. Media operations declare their
+   existing tool dependencies rather than falling back to shell generation.
+4. The agent's prompt/schema route and fallback use that same logical scope.
+   Native versus textual serialization remains model-specific. Answer-only phases
+   can suppress tool calls without granting a different scope.
+   Contract turns preserve the already-compacted conversation and tool-call/result
+   IDs. The standalone specialist prompt's latest-message-only behavior is not used
+   for these product turns. Prompt domains also come from the contract.
+   Accepted in-scope calls retain their model-provided arguments and native IDs;
+   the explicit-intent fallback must not overwrite them with the whole user turn.
+5. The context-bound dispatcher checks membership **and** existing runtime policy,
+   owner restrictions and exact-action approvals. A contract is not authorization
+   to bypass those gates. Contract work bypasses terminating legacy shortcuts.
+6. `_AgentRenderState` explicitly identifies streamed versus canonical output.
+   Later synthesis transfers ownership with turn-scoped replacement. The frontend
+   reconciles visible DOM, not just accumulated strings; tool evidence is retained.
+   Ownership is included in saved metrics and `message_saved` events.
+   History and resume honor replacement scope. Single-capability turns retain
+   canonical output: an always-synthesize trial caused a live notes loop and was
+   reverted. Compound turns cannot terminate after only one capability's result.
+
+## Verification
+
+Use the project's configured Python environment, not an unrelated system Python:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python -m pytest -q \
+  tests/test_turn_contract.py tests/test_turn_contract_integration.py \
+  tests/test_agent_turn_contract_boundaries.py tests/test_turn_rendering_js.py \
+  tests/test_contract_prompt_conversation.py tests/test_product_turn_contract_route.py \
+  tests/test_contract_explicit_fallback.py \
+  tests/test_history_resume_rendering_js.py \
+  tests/test_chat_route_tool_policy.py tests/test_tool_policy.py \
+  tests/test_frontend_module_version_parity.py
+node scripts/verify_agent_turn_contract.mjs --max-turns 80 --total-ms 900000
+```
+
+The browser verifier uses `sft_alex_creator` and actual 7011 Agent controls. It
+captures request toggles, SSE contract/tool events, visible output and persisted
+history. Ten families have four initial/follow-up Web-toggle combinations.
+Blocked or unrun cases are not passes. Email requires verified fixture isolation;
+do not enable global fixture mode on the user's live service to make a test pass.
+
+## Remaining limits
+
+- Classification is deterministic and vocabulary-based, not a proof of semantic
+  understanding. Add independent behavior examples for confirmed misses.
+- Schema registration and policy permission do not guarantee a remote provider
+  stays healthy throughout a turn. Runtime failure must remain visible.
+- Separate tool/argument errors, tool-service failures, rendering failures and
+  verifier defects in reports. Do not infer model accuracy from routing alone.
+- Canonical summaries can still ignore presentation constraints such as a
+  requested item count. Do not count those as full functional passes. Forcing an
+  extra model round is not a validated general repair for this deployed model.
+- Keep all imports of a local JS module on the same URL identity. Distinct query
+  versions instantiate separate module state even when source files are identical.
+
+Live baseline and current matrix results are in `reports/agent-turn-contract-*`.
+The implementation is not a claim that every family has passed live verification.

docs/BACKGROUND_TOOL_JOBS.md

@@ -0,0 +1,55 @@
+# Background research → originating chat
+
+Chat `trigger_research` calls carry a **dispatcher-supplied** `origin_chat_id`.
+The research start route verifies chat ownership before registering a durable
+`background_tool_jobs` row and starting the existing research service. Panel
+jobs have no origin and never inject a chat reply.
+
+- Chat default: **2 rounds**, 120-second *soft* research budget. Explicit
+  deeper/Auto rounds regain the normal research time budget. Panel defaults
+  remain unchanged. This is not a guaranteed two-minute wall-clock deadline.
+- A completion callback stores the report and sources. A startup worker also
+  reconciles missed callbacks and research errors/restarts.
+- When the origin has no active foreground/detached run, its model summarizes
+  the report with thinking off and no tools. An outer 75-second deadline also
+  bounds model-slot waits. If synthesis is unavailable, deliver an honest
+  notice plus the report link; preserve the evidence for follow-ups.
+- Message and delivery marker commit in one transaction with a deterministic
+  message ID. Report context is stored in server message metadata and injected
+  as untrusted evidence in regular and compact model history. Long excerpts
+  are explicitly marked; the saved full research report remains accessible.
+- The browser polls owner-scoped `/api/research/chat-jobs/{chat_id}`, appending
+  unseen message IDs only when that chat is current and not streaming. No
+  transcript replacement or forced navigation. Reloaded history deduplicates.
+- Chat uses the existing agent-thread rail and expandable rows. The compact
+  header shows status and a right-aligned BG task label with the shared whirlpool
+  while running; expanding reveals topic, phase/round, source count and report
+  link. Rows update in place, preserving expansion/focus while chat streams.
+  Completed rows remain visible; zero-source runs show a warning, not success.
+  Progress polling excludes reports and internal fields.
+
+Other tools are **not automatically backgrounded**. The durable handoff can be
+reused, but each future producer needs explicit launch/result/permission wiring.
+
+## Verification
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_background_tool_jobs.py tests/test_research_chat_runtime.py
+node --test tests/backgroundToolJobs.test.mjs
+node scripts/verify_background_delivery_isolation.mjs
+node scripts/verify_background_research_cards.mjs
+node scripts/verify_background_research_chat.mjs
+```
+
+The last script uses disposable `sft_alex_creator` chats and real research/model
+calls, then removes only its own reports/chats. Do not use real-user mutations.
+It checks two-round launch, continued chat, automatic arrival, no transcript
+rebuild/duplicates, reload, and a follow-up. Inspect retained report excerpts
+and generated summary when it fails; do not equate job launch with good research.
+
+Initial live runs verified delivery/navigation/follow-ups but exposed a summary
+attempt-count bug (fixed: helper requires **1 attempt**, not `max_retries=0`).
+A later full run was interrupted by an inference endpoint outage. The corrected
+summary path separately passed a real-model evidence/limitations/citation probe.
+All targeted Python tests passed (441); real DOM isolation checks passed. A clean
+full live run with useful retrieved evidence remains to be recorded.

docs/CLEAN_LOOP_V3_EXPERIMENT_20260909.md

@@ -0,0 +1,99 @@
+# No-RAG clean loop: first diagnostic
+
+## Setup
+
+No live UI, service configuration, or weights changed. The standalone loop sends
+conversation history, native assistant calls and matching tool results directly
+to the served pre-Heretic model. It never rewrites queries, invents calls, swaps
+families, or strips output. Invalid calls return errors. Six executions per turn
+and seven model rounds bound the test.
+
+Both arms use temperature 0, thinking disabled, 768 output tokens, and the
+original tool-work evaluator's `tools_for_mode(..., 'compact_contract_v3')`.
+This matters: the app's plain compact scrubber deletes descriptions, whereas v3
+retains empirically tested micro-hints. Previous plain-compact tests were not
+exact reproductions of the passing benchmark setup.
+
+The 76 tools come from the current app's ten-family inventory, transformed by
+the original v3 builder. This is not a byte-identical frozen 99-tool benchmark
+inventory or proof of training-data identity. The report records schema and
+builder hashes. No schemas are invented for this experiment.
+
+- **Stable:** same compact inventory on every turn, irrespective of spelling.
+- **Routed:** same loop, but existing `requested_capabilities` chooses inventory
+  each turn. This isolates that selector; it is not the complete production RAG
+  or Agent UI path. Other production normalizers are absent in both arms.
+- Private records are synthetic. No real private dispatcher is imported.
+  Only fixture reads and optional public SearXNG calls execute. Other operations
+  return explicit errors, so this does not validate their functionality.
+- Live search sends the exact model query to local SearXNG Bing/Yep, bypassing
+  app query rewriting/filtering. Source results may vary between arms.
+
+## Observations, not a blind score
+
+| Case | Stable compact inventory | Selector arm |
+|---|---|---|
+| `whats the current stock mraket` | Selected `web_search`, query `current stock market` | Offered zero tools; declined live lookup |
+| Exact seeded failed exchange, then `can you look up` | Searched with corrected query | Also searched with corrected query |
+| Summarize search, explicitly no tools | Answered without tools or permission failure | Same |
+| Calendar → email → calendar | Recalled second event at 14:30 | Same |
+| Notes → second note → what does it say | Correct `view` ID and content | Same after fixture correction |
+| Deliberately irrelevant search result | Did not automatically retry | Did not automatically retry |
+| User asks for a better source | Refined and executed another search | Proposed search was not offered and was rejected |
+| Web-disabled lookup | Attempted network access via bash; sandbox rejected it | Invented unsupported current market news without tools |
+
+The initial stable stock answer listed sources, not current index values. It
+does not establish that the market question was fully answered. Its subsequent
+`can you look up` elicited clarification after it had already searched. The
+separate seeded replay removes that differing-history confound.
+
+The first notes fixture incorrectly accepted `get/read`, not the real `view`
+action. Both models selected the correct action, but the fixture rejected it.
+Those six original turns are invalid for execution comparison. A corrected
+six-turn rerun succeeded in both arms; the failed evidence is retained.
+
+Web-off results are a release blocker: removing named web tools alone does not
+enforce network denial across general-purpose tools. The fixture prevented real
+execution, but any UI integration must use the real cross-tool permissions and
+clearly communicate unavailable capabilities. Neither arm is ready for a live
+switch. Source recovery and grounded completion also remain weak.
+
+## What this changes
+
+There is direct evidence that the selector can withhold needed tools, and that
+the model can repair the misspelled query itself when offered the tool. Clean
+history also supports the tested topic switches without synthetic substitutions.
+This supports continuing the clean-path experiment, not retraining or declaring
+the UI fixed. Full inventory is slower in these requests; overlapping runs and
+different source content prevent a controlled latency conclusion.
+
+Next: integrate the clean loop behind a test-only UI profile with real permission
+enforcement and one renderer, preserving the v3 contract. Test live read-only
+follow-ups and explicit Web-off behavior before any rollout. Separately compare
+a generic evidence-check/retry instruction on the weak-result fixture; do not
+manufacture a retry query in the harness.
+
+## Reproduce
+
+Eight boundary tests pass:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_clean_tool_loop.py
+```
+
+Run with a fresh report filename (existing evidence is never overwritten):
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python scripts/test_clean_tool_loop.py --live-search --report reports/clean-loop-v3-new-run.json
+```
+
+Evidence:
+
+- `reports/clean-loop-v3-20260909.json`: original 24 turns; notes fixture caveat above.
+- `reports/clean-loop-v3-stock-seeded-20260909.json`: four matched seeded follow-up turns.
+- `reports/clean-loop-v3-notes-fixture-corrected-20260909.json`: corrected six notes turns.
+
+Each report retains model requests, responses, offered inventory and execution
+results. The `completed` status means the request loop finished, **not** that
+the answer passed functional evaluation. These are synthetic/public traces, not
+private user conversations. This test does not measure UI rendering or streaming.

docs/CLEAN_V3_UI_PREVIEW.md

@@ -0,0 +1,220 @@
+# Tools v3 — No-RAG preview
+
+Select this endpoint in the 7011 model picker, with model
+`odysseus-qwen3.5-tools-pre-heretic`. This endpoint owns its complete tool loop
+and enters Agent mode server-side on every turn, including ambiguous follow-ups;
+it does not depend on the legacy per-message intent classifier. Start a new chat
+for an uncontaminated comparison. Enable Web for searches. Clean routing is
+owned by the exact model identity, so both the normal `preheret` endpoint and
+the `cleanv3` alias use this runtime. Every other model remains on legacy RAG.
+
+Endpoint ID: `cleanv3`. Its base URL uses the same inference server's Tailscale
+DNS name, `http://odysseus.tailb895f4.ts.net:18182/v1`, to distinguish it from
+the original IP-address route when existing chats omit endpoint IDs.
+
+## Implementation
+
+- `src/clean_agent_preview.py` is a separate streamed native-tool loop, entered
+  before legacy routing and substitutions. It uses real authenticated tool
+  dispatch, the tool-work `compact_contract_v5` builder, temperature 0,
+  and thinking disabled. No weights change or inference server was started.
+- The offered tool inventory is stable except for permissions/toggles. Safe,
+  explicit personal creates/updates are enabled for notes, tasks, calendar,
+  memory, skills and documents. Destructive operations, shell/code, outbound
+  email, browser interaction, deployment/admin changes and unrelated-family
+  write substitution remain blocked. No tool or argument substitution is
+  applied by the loop.
+- Native calls and matching results persist in `clean_v3_turn` metadata so
+  follow-ups use actual evidence. History retains at most eight complete turns,
+  trimming oldest whole turns for size; individual outputs cap at 8000 chars.
+- Real search still uses the existing search backend and its provider handling;
+  this does not claim that provider quality or every backend transform is fixed.
+- All routing, privileges and default settings outside this exact Odysseus model
+  remain unchanged. The loop has six execution/eight-round limits.
+- Write completion is evidence-bound: affirmative success text is replaced
+  unless a private-write tool succeeded during the turn. Proposed call batches
+  are policy-preflighted atomically, so a batch containing a blocked operation
+  cannot partially execute before denial.
+
+## Verification
+
+399 focused Python tests passed after route integration. Browser runs r1/r2
+accidentally exercised the old loop and are not preview evidence. The runner
+now explicitly asserts `selection_mode=clean_compact_v3_preview`.
+
+`reports/clean-v3-live-ui-r3-20260909.json` confirms the preview route, real notes
+execution, correct repetition from history, successful search and no-tool
+summary, plus visible incremental growth. Its notes assertions were for the
+old routed contract: they prohibited offering web tools even with Web enabled,
+and required another notes call for a verbatim repeat. The updated preview
+checks permit stable offers and accept an exact match to the preceding saved
+answer without re-execution; execution permissions are still asserted.
+
+`reports/clean-v3-live-ui-r4-20260909.json` is the corrected four-turn check,
+including notes with Web off and search with Web on: **4/4 passed**, with the
+preview selection mode explicitly confirmed on every turn.
+These are UI smoke tests, not all-family or factual-answer benchmark scores.
+
+## Disable
+
+Disabling only endpoint `cleanv3` removes the duplicate picker alias; it does
+not disable this model-owned runtime. To roll back the runtime, revert the exact
+model route in `routes/chat_routes.py`. Do not delete weights, adapters, or user
+chats. The v3 schema builder dependency is
+`/home/pewds/odysseus-tool-work/scripts/eval_alltools_unseen_compare.py` and its
+schema-dropout helper; preserve those with this deployment.
+
+## Expanded UI checks — 2026-09-09
+
+24 additional turns completed through the preview: 23 automated passes and one
+checker false alarm. The Cookbook follow-up correctly shortened the previous
+six-server result to the first three requested names without another call. The
+checker required either a fresh call or a verbatim repeat; manual inspection
+confirmed the requested subset. Raw failure evidence is retained, not rescored.
+
+Covered notes/misspellings/second-note selection, calendar/second-event time,
+tasks, documents, memory, skills, Cookbook listing, misspelled search, and Web
+toggle changes. Cross-family flows passed: Germany news → “whats my notes”,
+notes → “seach current stock mraket news”, and calendar → “now show my noes”.
+The model chose `current stock market news` itself. Every completed turn's
+audit confirmed the preview mode. Search source factual accuracy is not graded
+by this suite, and successful reads do not establish mutation coverage.
+
+Email was separately attempted but the test guard stopped it because the
+stable offered inventory exceeded its metadata-only verified scope. Email
+therefore remains unverified in this expanded run; the guard was not weakened.
+No production code, service settings or weights changed during these tests.
+
+Evidence under `reports/`:
+
+- `clean-v3-broader-ui-20260909.json`: 16 turns, 15 automatic passes, Cookbook caveat.
+- `clean-v3-topic-switch-ui-20260909.json`: 6/6 passed.
+- `clean-v3-second-note-ui-20260909.json`: 2/2 passed.
+- `clean-v3-email-notes-ui-20260909.json`: blocked email attempt; notes not run in that file.
+
+## Picker route fix
+
+The previous tests selected sessions through the API, missing a real picker
+bug: local entries were deduplicated by model ID, hiding alternative endpoints
+with the same weights. The picker now uses endpoint+model identity for local
+routes too, displays the endpoint name, and scopes its last-picked send override
+to the current chat. `/api/sessions` returns owner-filtered endpoint identity
+for unambiguous saved URLs, so reload labels do not depend on loading the model
+catalog. Ambiguous identical URLs are not guessed.
+
+The user-authorized chat `ec0683a2-015f-41d7-aa1f-34135c9640cb` was switched to
+`cleanv3` using the authenticated session PATCH API; no messages were inserted
+and no tool actions ran in that chat. Defaults and other chats were unchanged.
+
+The runner's `--picker-route true` starts on the original route, clicks the
+preview in the real picker, sends a greeting, reloads the chat permalink, then
+asks for notes. Early picker/reload reports are incomplete, not passes: their
+label check exposed the unloaded-catalog issue. Focused route/picker/history
+tests: 16 passed.
+
+Final picker test: `reports/clean-v3-picker-reload-r5-20260909.json`, **2/2
+passed**. Real picker click, greeting, permalink reload, and notes follow-up
+all confirmed the preview route. The label survived reload. R4 retained a
+history/DOM mismatch from sending before restored history was ready; the final
+driver explicitly waits for the saved first answer to render before sending.
+This does not claim a general fix for sending during unfinished history loading.
+
+## Native image/VL status
+
+The inference launcher previously set `--limit-mm-per-prompt` to zero images,
+so vLLM rejected attachments before the model saw them. The durable Odysseus
+launcher now permits up to three images per prompt; video remains disabled.
+
+`reports/clean-v3-vl-live-r4-20260909.json` proves the real 7011 attachment
+path, clean compact route, object/color/spatial recognition, permalink reload,
+and ambiguous image follow-up. Those checks pass. Exact OCR of the deterministic
+`ODYSSEUS 42` heading fails in both the untouched Qwen 3.5 9B base and the
+fine-tune, so it remains a base/runtime capability limitation rather than a
+fine-tune regression or harness failure.
+
+The same native path also passes JPEG and lossless WebP transport, object
+recognition, reload, and follow-up grounding. Evidence:
+`reports/clean-v3-vl-jpeg-r1-20260909.json` and
+`reports/clean-v3-vl-webp-r1-20260909.json`. Both remain `partial` only because
+the shared OCR check fails.
+
+## Reversible write check
+
+`scripts/verify_clean_v3_write.mjs` runs against only `sft_alex_creator`. It
+creates one UUID-named note through the real 7011 UI, verifies that exact row,
+requests a destructive bulk deletion, verifies the row still exists, and then
+deletes only its own test row through the authenticated API. The cleanup is
+verified by a 404 lookup.
+
+Final evidence: `reports/clean-v3-write-ui-r8-20260909.json`, **passed**. Both
+turns reported `selection_mode=clean_compact_v3_preview`; creation executed via
+`manage_notes(action=add)`, the destructive action did not execute, and the
+canonical response was “No changes were made.” The earlier r3/r5 files are
+startup/placement failures, while r4/r6/r7 retained genuine intermediate
+harness and verifier failures; none should be interpreted as passes.
+
+## Stateful, search, and email checks
+
+The reversible stateful runner passes all six mutation families in one run:
+calendar, notes, tasks, documents, memory, and skills (**6/6**). Each flow
+creates a UUID-only artifact through the real Agent UI, verifies it by
+owner-scoped API, applies a noun-free correction, verifies persistence, and
+removes only that artifact. A direct database audit found zero active synthetic
+calendar, note, task, or document rows afterward.
+
+The document failure was harness-owned. Compact description dropout left a
+vague free-form `command` field, error envelopes defaulted to exit code 0, and
+the clean loop dropped the active document ID. Compact v5 now exposes only
+required structured `edits`, reports errors truthfully, and executes against
+the request's explicit active document. Fresh document and combined stateful
+runs pass.
+
+Search Web-toggle combinations `00`, `01`, `10`, and `11` pass **8/8** across
+two turns. A web question can no longer silently enable Bash because it says
+“official source”, and an unavailable Web capability exposes no unrelated
+fallback family. The quality suite passes **3/3**: evidence reuse without a
+second call, explicit official-page inspection with `web_fetch`, correction of
+“stock mraket” in actual search arguments, and a truthful unsupported result
+for a synthetic company.
+
+Production-path email reads pass **3/3** through the running email MCP: account
+list, latest inbox list, and referential read of the first result. The report
+retains no account names, addresses, subjects, bodies, prompts, or answers.
+
+Post-fix representative direct/follow-up coverage also passes for every family:
+notes/calendar 4/4, tasks/documents/memory/skills/Cookbook/search/shell 14/14,
+and email 3/3 in its privacy-preserving runner. The combined legacy verifier's
+metadata-only email guard correctly refused its broader stable inventory; that
+stopped report is not counted as a model failure.
+
+Evidence:
+
+- `reports/clean-v3-stateful-all-r3-20260909.json`
+- `reports/clean-v3-stateful-documents-r2-20260909.json`
+- `reports/clean-v3-search-toggle-final-r6-20260909.json`
+- `reports/clean-v3-search-quality-r3-20260909.json`
+- `reports/clean-v3-email-read-r1-20260909.json`
+- `reports/clean-v3-ten-family-tail-postfix-r1-20260909.json`
+
+These checks verify routing, execution, persistence, follow-up, and selected
+answer-quality invariants. They are not yet the sealed all-action ship score.
+
+## Compact v5 and corrected contract evidence
+
+Compact v5 keeps the compact-v3 surface and adds only development-positive
+field hints for Email, Search/Hugging Face quant selection, and Shell/files.
+A Calendar date hint regressed development and was excluded. The Python tool now
+emits one final bare expression, REPL-style, without duplicating explicit
+`print(...)`; this turns otherwise correct computation calls into visible tool
+evidence for all models.
+
+Under frozen scorer `odysseus.contract.v2.5`, development is 327/344 raw
+(95.06%) and 327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%)
+and 311/336 scorable (92.56%), with zero reasoning leakage. Calendar, Shell,
+and Tasks remain below the 90% family ship floor, so the model is not yet a
+full benchmark ship candidate.
+
+Fresh post-deploy real-UI evidence passes: stateful flows 6/6, Email 3/3,
+Search quality/recovery 3/3, private browser 3/3, and VL workflow 3/3. The
+Search check accepts a failed attempt only when a later tool succeeds and the
+final answer remains grounded.

docs/REGULAR_MODEL_TOOL_COMPATIBILITY.md

@@ -0,0 +1,125 @@
+# Regular-model tool compatibility
+
+Last verified: 2026-09-09 through the authenticated 7011 Agent UI as
+`sft_alex_creator`.
+
+This is the legacy-RAG track. The exact model
+`odysseus-qwen3.5-tools-pre-heretic` is excluded and remains on its model-owned
+clean compact runtime.
+
+## Current baseline
+
+| Endpoint | Model | Ten-family result | State |
+|---|---|---:|---|
+| DeepSeek | `deepseek-v4-flash` | 10/10 | passed |
+| DeepSeek | `deepseek-v4-pro` | 10/10 | passed |
+| OpenAI | `gpt-5.5` | 10/10 | passed |
+| OpenAI | `gpt-5.6-sol` | 10/10 | passed |
+| OpenAI | `gpt-5.6-terra` | 10/10 | passed |
+| OpenAI | `gpt-5.6-luna` | 10/10 | passed |
+| OpenRouter | `moonshotai/kimi-k3` | 10/10 | passed |
+| OpenRouter | `x-ai/grok-4.5` | 10/10 | passed |
+| OpenRouter | `qwen/qwen3-vl-235b-a22b-instruct` | 10/10 | passed |
+| OpenRouter | `openai/gpt-5-image` | n/a | image generation; chat tools unsupported |
+| Local `100.69.120.65:8062` | `Qwen/Qwen3.5-9B` | not run | endpoint unavailable |
+| Local `100.69.120.65:8062` | `GLM-5.3-Flash-Alis-MLX-4bit` | not run | endpoint unavailable |
+
+The ten-family baseline covers one read-only functional turn each for notes,
+calendar, email accounts, tasks, documents, memory, skills, Cookbook/admin,
+web search, and shell. It verifies the legacy route, expected native tool call,
+execution result, visible UI answer, and absence of reasoning leakage. It is not
+yet a claim that every mutation/action variant, typo, or follow-up passes.
+
+## Typo and follow-up profile
+
+The stricter real-UI profile sends one misspelled read-only request to every
+family, followed immediately by a noun-free reference to the returned result.
+Read-only follow-ups must not call any tool; search follow-ups may either use
+the existing evidence or fetch the prior link. Across the nine chat-capable API
+models, the composited post-repair result is **178/180 turns (98.89%)**:
+
+| Model | Conversation result |
+|---|---:|
+| `deepseek-v4-flash` | 20/20 |
+| `deepseek-v4-pro` | 20/20 |
+| `gpt-5.5` | 20/20 |
+| `gpt-5.6-sol` | 20/20 |
+| `gpt-5.6-terra` | 20/20 |
+| `gpt-5.6-luna` | 18/20 |
+| `moonshotai/kimi-k3` | 20/20 |
+| `x-ai/grok-4.5` | 20/20 |
+| `qwen/qwen3-vl-235b-a22b-instruct` | 20/20 |
+
+Luna's only remaining family miss is a deliberately misspelled Shell request.
+The correct-spelling baseline passes. The harness does not auto-execute a shell
+command to hide that model-owned limitation.
+
+The shared repair recognizes a uniquely misspelled action verb and family noun,
+then seals only declared safe private reads with immutable canonical arguments.
+This repaired Tasks/Documents/Memory and adjacent read families across providers
+without widening mutation or Shell authority. A compact native-tool instruction
+also tells regular API models to map clear typos to a currently offered tool.
+
+Conversation evidence:
+
+- `reports/regular-model-conversation-flash-r3-20260909.json`
+- `reports/regular-model-conversation-remaining-r1-20260909.json`
+- `reports/regular-model-conversation-repair-r1-20260909.json`
+- `reports/regular-model-conversation-shell-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-repair-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-tail-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-search-r1-20260909.json`
+
+Evidence:
+
+- `reports/regular-model-tools-provider-final-r4-20260909.json` — Flash, GPT-5.5, Kimi: 30/30.
+- `reports/regular-model-tools-repair-r3-20260909.json` — Pro and Sol: 20/20; retained Qwen pre-final 9/10 miss.
+- `reports/regular-qwen-vl-full-r4-20260909.json` — Qwen-VL final family-switch run: 10/10.
+- `reports/regular-model-tools-remaining-20260909.json` — Terra, Luna, Grok: 30/30; records unavailable/unsupported models and pre-repair failures.
+- `reports/regular-model-tools-postfix-r1-20260909.json` — post-hardening
+  rerun: nine chat-capable API models passed 90/90 family turns with zero model
+  failures. Its overall status is non-passing only because the two configured
+  local endpoints were offline; the image-only model remains unsupported.
+
+## Family switch and page inspection
+
+The six-turn switch/back flow covers notes → calendar → notes from prior
+evidence → web search → explicit `web_fetch` → calendar from prior evidence.
+All nine API models have a clean 6/6 reproduction (**54/54**). Kimi skipped
+search once in the retained first run and passed a fresh reproduction; that
+variability remains visible instead of being erased.
+
+Evidence:
+
+- `reports/regular-model-switchback-flash-r2-20260909.json`
+- `reports/regular-model-switchback-remaining-r1-20260909.json`
+- `reports/regular-model-switchback-kimi-r1-20260909.json`
+
+## Repair that produced the clean baseline
+
+Regular models no longer inherit up to three stale tool families into every
+explicit new request. Referential follow-ups still resolve from typed recent
+tool evidence, while explicit family switches receive the current family only.
+Safe required reads use `active_capabilities`, so stale offered context cannot
+disable their immutable operation. The stream layer also stops an exact long
+block repeated twice instead of waiting for a provider's full timeout.
+
+The composer no longer treats generic words such as “source”, “system”, “app”,
+or “review” as authority to silently enable Bash. Explicit shell, terminal,
+repository, code-file, and direct coding requests retain workspace
+auto-escalation. This is a shared UI authority fix, not a model-name exception.
+
+Run a bounded subset with:
+
+```sh
+MODELS='deepseek-v4-flash,gpt-5.5' \
+FAMILIES='notes,calendar' WORKERS=2 \
+REPORT_PATH=reports/regular-model-check.json \
+node scripts/verify_regular_model_tools.mjs
+```
+
+Set `PROFILE=conversation` to run the typo plus follow-up profile.
+
+The runner discovers only enabled pinned models (visible cached local models
+when no pins exist), retains no tool outputs or private rows, and deletes only
+the exact sessions it creates.

docs/SEARCH_HARNESS_AUDIT_20260909.md

@@ -0,0 +1,92 @@
+# Search and compact-tool experiment — 2026-09-09
+
+## Decision
+
+Keep the normal routed profile on 7011. The all-tools compact experiment is
+implemented but **disabled**: direct routing success did not translate into a
+working Agent UI. Do not retrain or promote a profile on these measurements.
+
+## Changes
+
+- Short public-web lookups on the target model have an execution budget: two
+  distinct token-normalized searches, one fetch, and up to three browser calls
+  after the two searches. This bounds attempts, not just recovery prose. Existing
+  permissions still apply; this does not make unavailable tools executable.
+- Failed/weak searches reach the model for evaluation and query refinement,
+  instead of the earlier unconditional terminal evidence veto. Some legacy
+  heuristics and official-site shortcuts remain; this is not a completed rewrite.
+- Search providers retain query/engine/date provenance. Unconfigured credentialed
+  fallbacks are skipped. When SearXNG is the sole configured usable provider, Yep
+  on the same instance is an additional fallback.
+- An unavailable warm-only family no longer vetoes an otherwise ordinary reply.
+- The all-tools experiment offers the trained compact inventory subject to
+  permissions. It requires the exact test-owner environment flag and exact model
+  match. The temporary service flag was removed after failed UI testing.
+
+## Evidence and limits
+
+| Measurement | Result | What it establishes |
+|---|---|---|
+| Focused Python regression suite | 401 passed | Covered policy, contract, provider and recovery-budget behavior |
+| Direct family-only compact schemas | 10/10 tool routing | Small public smoke test, not functional or blind accuracy |
+| Direct all-family compact schemas | 10/10 tool routing | Inventory did not break these first calls; roughly 3–4x slower in this run |
+| Full compact Agent UI, revision 3 | All eight turns failed one or more checks | Not suitable for activation; leaks, duplicate/incorrect rendering or missing expected calls |
+| Normal-profile final UI control | Five passed checks, two product failures, one capture error | Not accepted; suite status incomplete |
+| Clean synthetic notes tool-result continuation | Clean answer with both schema sizes | Model can continue correctly on that isolated input, not proof that UI failure is solely harness |
+
+Direct probes used temperature 0; the UI target-model sampling path can cap at
+0.2. Prompts, history and tool-result serialization also differ. Match those
+before attributing UI failures to weights versus harness. Existing UI checks are
+not a grounded factual-answer benchmark. Unit tests are not UI acceptance.
+
+Normal-profile control details: notes initial, both calendar turns, AI search
+initial, and history initial passed the automated checks. Notes follow-up hit a
+Playwright `Network.getResponseBody` capture error and is inconclusive. AI search
+follow-up explicitly requested a summary with no tools, but the contract still
+required `search_browser` and returned a permission failure. The history search
+follow-up failed the visible-leak check. These are separate from source relevance;
+the earlier warm-only fix did not cover classification as an active requirement.
+There is no matched pre-change control establishing a net improvement.
+
+Provider isolation bypassed app relevance filters. Bing general often returned
+broad or unrelated results despite the full query. Google/Mojeek returned no
+results, DDG hit CAPTCHA, and Presearch timed out. Yep returned useful PostgreSQL
+documentation, but was weak or empty for several other questions. Engine health
+and source quality remain unresolved. Fallback cannot help when an earlier weak
+result survives filtering; there is no claim of universal relevance here.
+
+## Reproduce and inspect
+
+- `scripts/audit_search_pipeline.py`: raw provider comparison, no model.
+- `scripts/compare_compact_tool_inventory.py`: read-only model schema comparison;
+  proposed calls are never executed.
+- `scripts/verify_agent_turn_contract.mjs`: real 7011 Agent UI and persisted-history
+  checks. Use the dedicated test account; reports can contain private tool data.
+- `reports/search-provider-isolation.json`, `reports/search-yep-isolation.json`:
+  public provider evidence.
+- `reports/compact-inventory-ablation.json`: direct routing probe.
+- `reports/full-compact-ui-audit-r3.json`: completed rejected UI experiment.
+  Earlier experiment reports include an initialization error and an aborted run;
+  do not combine them into an accuracy score.
+- `reports/routed-control-ui-audit-final.json`: normal-profile control replay;
+  eight attempts, incomplete because of the capture error; failures retained.
+
+Focused suite:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_turn_contract.py tests/test_turn_contract_integration.py tests/test_service_search_provider_guards.py tests/test_web_recovery_budget.py tests/test_tool_policy.py
+```
+
+UI replay (read-only prompts, creates test chats):
+
+```sh
+node scripts/verify_agent_turn_contract.mjs --families notes,calendar,search_ai,search_history --pairs notes:11,calendar:11,search_ai:11,search_history:11 --max-turns 8 --total-ms 360000 --turn-ms 45000 --report reports/routed-control-ui-audit-final.json
+```
+
+## Next discriminating test
+
+Replay the same captured UI request directly, preserving sampling, compact
+schemas, history and tool results. Then change one layer at a time. Separately
+score retrieved-source relevance and supported answers. Replace failing generic
+boundaries only when the replay identifies them; do not add rules for individual
+user phrasings or treat successful tool routing as successful execution.

docs/TYPO_ROUTING_AUDIT_20260909.md

@@ -0,0 +1,73 @@
+# Typo-tolerant tool routing audit
+
+The 9B SFT model was not retrained. This audit targets the earlier harness
+stage that decides which complete tool families the model is allowed to see.
+
+## Method
+
+- Source prompts: real `sft_alex_creator` sessions from `a37dcb3b-...` onward.
+- Labels: recorded single-family tool calls, excluding mixed/ambiguous traces.
+- Variants: deletion, adjacent transposition, duplicated character,
+  keyboard-neighbor substitution, and accidental word split.
+- Split: deterministic SHA-256 assignment before scoring (75% dev, 25% blind).
+- Safety: static routing only; no historical mutation or send action is replayed.
+- Acceptance: at least 95% blind exact-family accuracy and below 1% blind
+  wrong-family authorization. Abstention is measured separately.
+
+## Results
+
+| Router | Dev family supplied | Blind family supplied | Blind exact | Blind wrong-family |
+|---|---:|---:|---:|---:|
+| Previous exact rules | 63.64% | 65.69% | — | — |
+| Conservative fuzzy fallback r4 | 96.31% | 98.31% | 96.62% | 0.00% |
+| Final router + safe-read repair | 98.31% | 98.73% | 97.05% | 0.00% |
+
+The fallback runs only for action/lookup-shaped requests, resolves exactly one
+nearby family term, and abstains on ambiguity. Conceptual questions remain
+tool-free. Complete family schemas are still selected by the immutable turn
+contract; fuzzy matching never chooses an individual tool or its arguments.
+
+Authoritative machine reports:
+
+- `reports/typo-tool-routing-baseline-20260909.json`
+- `reports/typo-tool-routing-fuzzy-r4-20260909.json`
+- `reports/typo-tool-routing-final-20260909.json`
+- `reports/post-followup-agent-80-20260909.json`
+- `reports/post-typo-routing-agent-80-20260909.json`
+- `reports/live-typo-agent-20-20260909.json`
+- `reports/live-typo-unresolved-r3-20260909.json`
+- `reports/live-typo-agent-final-20-20260909.json`
+- `reports/post-typo-safe-read-agent-final-80-20260909.json`
+
+## Live 7011 findings
+
+The post-deployment standard matrix passed 80/80 through the real Agent UI.
+The first read-only typo matrix then attempted 17 of 20 planned turns before
+its total-time limit. Initial Notes, Calendar, Email, Tasks, Documents, and
+Cookbook calls passed. Completed failing turns still had the correct family
+and required tool in `turn_contract.offered`; the 9B model sometimes answered
+without calling that offered tool. Memory and Search also exposed timeouts.
+
+This separates three failure classes:
+
+1. **Tool injection:** addressed by conservative fuzzy family routing; blind
+   exact routing is 96.62% with zero blind wrong-family authorizations.
+2. **Required read execution:** a correctly offered safe list/refresh tool can
+   still be skipped by the model, especially after a typo or on “list those
+   again” follow-ups. This should be handled by the generic deterministic
+   safe-read path, not additional prompt-specific hints.
+3. **Runtime timeout:** Search and one Memory follow-up require loop/backend
+   diagnosis. A timeout is not counted as a model-accuracy or routing result.
+
+The generic safe-read parser and search-family precedence were then repaired.
+The previously unresolved Calendar, Email, Search, and Shell/Files cases passed
+8/8. The complete typo matrix passed 20/20, including initial requests and
+follow-ups for all ten families. The final standard Agent UI compatibility
+matrix passed 80/80 across family, Web-toggle, and follow-up combinations.
+
+The broad routing regression suite passed 458 tests. The model was not
+retrained and no DeepSeek API was used: the measured defect was in harness
+family selection and deterministic safe-read execution, upstream of the
+model. All 1,535 unique labeled historical turns were statically audited to
+mine failure categories. Historical write/send/delete actions were not replayed
+against live data; live verification used the deduplicated read-only matrices.

docs/skills-lifecycle.md

@@ -0,0 +1,26 @@
+# Skills lifecycle
+
+The UI exposes All, Built-in, Approved, and Draft. Draft includes archived
+records so they remain inspectable and recoverable. Built-ins are not audited.
+Approved means published, passing, at the configured confidence threshold,
+and not marked unnecessary. Baseline speed measurements remain evidence, not
+an additional hidden UI approval gate.
+
+Automatic audits process at most eight eligible records at a time, oldest first.
+New records are eligible immediately; inconclusive checks retry after a day;
+failed repairs retry after a week. Passed, duplicate-skipped, and archived records
+are excluded. Existing daily Skills Audit tasks drive this queue. Their quiet
+window deferrals propagate to the scheduler rather than becoming task failures.
+Automatic runs use background model scheduling. Existing self-repair and teacher
+repair stages remain in place; failed candidates remain drafts.
+
+The skill index advertises short descriptions; the agent loads a relevant full
+procedure on demand and applies already-injected procedures directly. Extraction
+prefers verified discoveries and specific workarounds over routine tool usage.
+
+Reference reviewed: NousResearch/hermes-agent, MIT license, commit
+cfdbbb6e35010ace89fbe8243ee82fa4de143e10, cloned to
+/home/pewds/hermes-skills-reference. In particular tools/skills_tool.py and
+agent/prompt_builder.py use progressive disclosure and task-triggered procedure
+loading. These changes adapt that approach to Odysseus's existing registry;
+no Hermes implementation code was copied.

launcher.py

@@ -130,7 +130,7 @@ if __name__ == "__main__":
     from app import app
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
     url = f"http://{bind_host}:{bind_port}"
 
     if getattr(sys, 'frozen', False):

package-lock.json

@@ -4,19 +4,84 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "odysseus",
       "devDependencies": {
-        "@antithesishq/bombadil": "^0.6.1"
+        "@antithesishq/bombadil": "^0.7.0",
+        "@playwright/test": "^1.62.1"
       }
     },
     "node_modules/@antithesishq/bombadil": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
-      "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.7.0.tgz",
+      "integrity": "sha512-alJmnphJ/iUoL5mCsnV3DwtajGy/sEQ3NJJCiMhgjqXshSq2BUtAs0vqdXEiiSkB8HbsOX5CLrAcaogYdwfAJg==",
       "dev": true,
       "license": "MIT",
       "bin": {
         "bombadil": "bin/bombadil.js"
       }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
+      "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
+      "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
+      "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
     }
   }
 }

package.json

@@ -1,9 +1,18 @@
 {
+  "name": "odysseus",
+  "private": true,
   "repository": {
     "type": "git",
     "url": "https://github.com/odysseus-dev/odysseus.git"
   },
+  "scripts": {
+    "test:photo-editor": "playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:install": "playwright install chromium firefox webkit",
+    "test:photo-editor:firefox": "PHOTO_EDITOR_E2E_BROWSER=firefox playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js"
+  },
   "devDependencies": {
-    "@antithesishq/bombadil": "^0.6.1"
+    "@antithesishq/bombadil": "^0.7.0",
+    "@playwright/test": "^1.62.1"
   }
 }

plans/ODYSSEUS_TOOL_HARDENING_PLAN.md

@@ -0,0 +1,326 @@
+# Odysseus Tool Runtime Hardening Plan
+
+## Objective
+
+Ship `odysseus-qwen3.5-tools-pre-heretic` with one compact, model-specific tool
+runtime that supports realistic multi-turn use. Keep the existing RAG runtime
+unchanged for every other model. Prove routing, execution, answer quality,
+follow-ups, safety, rendering, latency, and native image/VL understanding through
+the real 7011 Agent UI.
+
+Current evidence is a baseline, not a ship claim:
+
+- Corrected v2.5 + compact-v5 development is 327/344 raw (95.06%) and
+  327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%) and
+  311/336 scorable (92.56%), with zero reasoning leakage.
+- Notes, Skills, and Cookbook/admin clear 95% scorable blind. Calendar 87.5%,
+  Shell/files 86.11%, and Tasks 87.5% remain below the 90% family ship floor.
+- Compact-v5 hints improved Email, Search/HF quant, and Shell on development;
+  a Calendar hint regressed and was rejected rather than shipped.
+
+- Ten-family focused baseline: 19/20 functional and 20/20 routing/execution.
+- Typo and cross-family read flows: 26/26 passed.
+- Real use exposed untested write correction and search-to-fetch follow-ups.
+- Email production access, browser interaction, search quality, and broader
+  multi-turn mutations are not yet proven.
+- Nine enabled chat-capable regular API models pass the ten-family read-only
+  legacy-RAG baseline (90/90 combined). Their stricter typo/follow-up profile is
+  178/180 turns: eight models are 20/20 and Luna is 18/20 due only to its
+  misspelled Shell request. One pinned image-generation model is explicitly
+  unsupported and two visible local models are currently offline.
+- Native VL object/spatial recognition and reload follow-up pass. Exact OCR
+  fails equally on the fine-tune and untouched 9B base and remains unresolved.
+  PNG, JPEG, and WebP transport all pass.
+- Reversible create/correct/API-verify/cleanup flows pass 6/6 across every
+  stateful family.
+- Search Web-toggle combinations pass 8/8 and the focused quality suite passes
+  3/3. Production-path email account/inbox/referential reads pass 3/3.
+- The latest regular-model regression is 90/90 across the nine enabled
+  chat-capable API models, with zero failed model turns; two local endpoints
+  remain offline and the image-only model is unsupported.
+- The Epictetus OMLX endpoint was recovered after an unsupported
+  `qwen3_5_mtp` model load wedged the server. Its supported Qwen 27B 4-bit
+  model passes the ten-family real-7011 legacy-RAG smoke 10/10; the unsupported
+  MTP artifact is recorded as a runtime limitation rather than a timeout.
+- Fresh compact-v5 UI regressions pass stateful 6/6, Email 3/3, Search 3/3,
+  private-browser 3/3, and VL workflow 3/3.
+- The exact-model, family-scoped compact runtime now passes 20/20 direct and
+  same-family turns across all ten families on the real 7011 Agent UI. A
+  separate 36/36 robustness run passes misspellings, bounded repeats, browser
+  and news continuation, ambiguous follow-ups, family switchbacks, and a
+  greeting before a tool request.
+- The mobile active-email editor path passes 1/1: `Write reply this email`
+  offers and executes only `update_document`, mutates the open draft, and
+  preserves its reply headers and quoted thread.
+- The active-editor classifier now also covers short mobile wording without a
+  pronoun (`Write reply` / `Draft a reply`) while explicit note, code, file, and
+  new-object requests retain their own families. Whole-draft requests are bound
+  to the sole offered `update_document` writer until one successful write, then
+  tools are removed for the confirmation round. The deployed real-route email
+  regression passes 3/3—including the exact unspecified `Write reply to this
+  email` form—with one write, verified mutation, and preserved reply headers.
+  Clean-v3 now also emits the established `doc_update` event and flattened
+  document metadata on `tool_output`, so a successful database write updates
+  the already-open editor instead of leaving stale UI beside a success message.
+- The client now reuses the existing assistant bubble for `agent_step` round 1
+  instead of replacing it before the first token. A real-7011 sampled
+  greeting-to-Notes conversation passes 2/2 with stable first-round DOM
+  identity; round 2+ remains the only continuation-bubble path.
+- Clean-runtime metrics now expose provider-counted initial injected tokens,
+  all-round input/output, TTFT, tok/s, schema count, agent rounds, and tool-call
+  count. A real 7011 browser run passes 2/2 and visibly renders compact footers
+  plus the full details popup; the sampled Notes turns streamed progressively.
+- The deployed startup bottleneck was an unindexed quadratic transcript-FTS
+  reconciliation. Live-database import fell from about 36 seconds to 0.54
+  seconds; 7011 now answers in about 3 seconds after a controlled restart.
+- A controlled identical-compact comparison already proves the fine-tune's
+  accuracy benefit: 94.48% (325/344) versus the untouched base's 77.91%
+  (268/344). Raw serving speed is effectively tied, so product speed comes
+  from the compact contract and fewer failed/redundant rounds.
+- A fully merged 10,000-row category-repair candidate reached 97.32% scorable
+  development but only 92.26% scorable sealed blind. Calendar (87.5%), Tasks
+  (87.5%), and Shell/files (86.11%) remained below the family floor, so it was
+  rejected and not deployed. Compact-v4/full development A/Bs did not improve
+  Calendar or Tasks over compact-v5; full-schema Shell also fell from 97.22%
+  to 94.44%. This rules out compactness as the primary cause of the remaining
+  blind gaps and supports keeping the compact contract.
+
+## Non-negotiable architecture rules
+
+1. Runtime selection follows exact model identity. The trained Odysseus model
+   uses the clean compact runtime across endpoint aliases; all other models use
+   legacy RAG. Add a regression test for both sides.
+2. Resolve permissions, toggles, and available backends once per turn. Produce
+   one immutable contract satisfying `required ⊆ offered ⊆ executable`.
+3. Never offer a tool that the preview policy will categorically reject. Add a
+   contract self-check covering every offered action/effect combination.
+4. Follow-ups consume typed prior evidence: native call, result, success state,
+   family, and object identifiers. Do not infer continuity from keyword RAG.
+5. Contextual write authority may revise only a recently proven object in the
+   same family. It may not authorize a new object, another family, a destructive
+   action, or an external side effect.
+6. The model chooses tools and valid arguments. The harness validates and
+   executes; it does not silently substitute another family, rewrite arguments,
+   fabricate success, or replace a failed tool with prose claiming completion.
+7. One owner renders each turn: streamed prose or canonical structured output.
+   Never both, and never expose hidden prompts or raw untrusted wrappers.
+8. No exact-prompt production patches. A fix must name the failed layer, add a
+   generic failing invariant test, and cover neighboring cases.
+
+## Failure layers
+
+Every failure is assigned to exactly one primary layer before code changes:
+
+1. **Route:** wrong model runtime or endpoint identity.
+2. **Contract:** required tool absent, forbidden tool present, or toggle drift.
+3. **Model:** wrong/no tool or semantically wrong required arguments despite a
+   correct contract.
+4. **Policy:** valid proposed operation incorrectly allowed or denied.
+5. **Execution:** canonical arguments, backend dispatch, timeout, or result
+   envelope is wrong.
+6. **Evidence:** result is empty, irrelevant, truncated badly, or insufficient.
+7. **Answer:** model misstates or ignores valid tool evidence.
+8. **Rendering:** duplicate, dump-at-end, missing structured output, or stopped
+   stream.
+9. **Performance:** startup, TTFT, tool latency, or oversized context.
+
+Reports store aggregate category, relevant contract/tool metadata, timings, and
+sanitized outputs. Do not copy private hidden benchmark prompts or create a log
+dump that nobody can audit.
+
+## Test matrix
+
+Use the real authenticated 7011 Agent UI and the normal `preheret` picker alias.
+Use `sft_alex_creator` for reversible writes. Never mutate the personal account
+from an automated test.
+
+### A. Every one of the ten families
+
+For calendar, notes, email, tasks, documents, memory, skills, Cookbook/admin,
+search/browser, and shell/files, test:
+
+- direct request;
+- natural misspelling;
+- ambiguous same-family follow-up;
+- switch to another family and back;
+- no-tool greeting before the tool request;
+- requested count/field limit;
+- backend failure rendered truthfully;
+- reload the permalink before a follow-up.
+
+### B. Stateful mutation families
+
+For notes, calendar, tasks, documents, memory, and skills:
+
+- create → verify by API → referential correction → verify;
+- create → list/read → correction → verify;
+- typo correction such as name/date/title without repeating the family noun;
+- correction after one unrelated conversational turn;
+- destructive request is denied atomically;
+- failed write never produces a success claim;
+- cleanup deletes only the UUID-owned test artifact and verifies absence.
+
+### C. Search and browser conversations
+
+- search → summarize existing results without a new call;
+- search → inspect one result with `web_fetch`;
+- poor results → refine query once;
+- insufficient evidence → say so without fabrication;
+- Web toggle combinations `00`, `01`, `10`, and `11` across two turns;
+- private browser open/snapshot/click only after its permission boundary is
+  deliberately enabled and specified; do not smuggle it in via web search.
+
+Grade source relevance, freshness, authority, and whether claims are supported,
+not merely whether `web_search` was called.
+
+### D. Email and shell
+
+- Separate fixture accuracy from production connectivity. A fixture pass cannot
+  promote production email health.
+- Test account listing, inbox listing, reading, and referential follow-up against
+  the configured production-like backend before enabling email actions.
+- Shell remains toggle-gated. Test off/on transitions, canonical raw command
+  dispatch, read-only output, and denial of network/destructive commands.
+
+### E. Rendering and performance
+
+- Assert first visible streamed token, monotonic DOM growth, one final answer,
+  persistence/reload equality, stop behavior, and structured list rendering.
+- Record request preparation, TTFT, tool duration, post-tool TTFT, total time,
+  input/output tokens, and tool-result bytes.
+- Diagnose the 30–40 second 7011 restart separately from inference latency.
+- Bound large calendar/search results before replaying them into later rounds,
+  while preserving IDs and fields needed for follow-ups.
+
+### F. Image/VL recognition
+
+- Attach real PNG, JPEG, and WebP images through the 7011 UI and verify the
+  trained model receives native multimodal message content on its clean route.
+- Test object recognition, visible text/OCR, spatial relationships, charts, and
+  screenshots. Score required facts instead of stylistic wording.
+- Test image → ambiguous follow-up, image → tool request, and tool result → image
+  comparison without requiring the user to attach the same image again.
+- Verify image references survive persistence and permalink reload without raw
+  base64, local paths, or hidden wrappers appearing in chat output.
+- Separate direct model vision from `inspect_media`, browser screenshots, and
+  image generation. The harness must not silently substitute one for another.
+- Compare the fine-tune with its base VL model on the same images to detect
+  whether tool training regressed visual understanding.
+
+### G. Regular-model legacy RAG and tool coverage
+
+- Inventory every enabled non-Odysseus endpoint/model visible in 7011, including
+  its provider, schema mode, native-tool support, context limit, and configured
+  permissions. Do not assume every provider supports the same wire format.
+- Assert that no non-Odysseus model enters the clean-v3 runtime. These models
+  retain the regular RAG/tool loop and are repaired only in that owning path.
+- For each model, test every tool family the effective user policy offers:
+  direct request, misspelling, ambiguous follow-up, family switch, backend
+  failure, and Web/Bash toggle transitions. Record unsupported families as an
+  explicit capability limitation, not a silent pass.
+- Test full schemas versus compact schemas only where both are valid for that
+  model. Store the selected schema mode in every report.
+- Verify provider-native tool calls, textual fallback parsing where required,
+  canonical argument conversion, execution, evidence replay, and rendering.
+- Group fixes by shared legacy-runtime or provider-adapter defect. Do not add
+  model-name prompt exceptions when a transport, schema, or RAG ranking issue is
+  responsible.
+- Maintain a per-model compatibility matrix so adding or changing an endpoint
+  cannot silently regress previously working tools.
+
+## Fix protocol
+
+For each failure:
+
+1. Preserve the raw report and reproduce once on a fresh test session.
+2. Identify the primary failure layer from the taxonomy above.
+3. Add the smallest generic red test at that layer.
+4. Fix the owning module or invariant—not the literal prompt.
+5. Run the focused unit tests, the original scenario, two adjacent scenarios,
+   and the affected family suite.
+6. After a batch of category fixes, rerun the ten-family matrix and legacy-RAG
+   isolation test. Do not rerun training unless the contract and harness are
+   proven correct and failures remain model-owned.
+
+If three failures share a layer, pause case-by-case patching and refactor that
+layer before continuing.
+
+## Execution phases
+
+### Phase 1 — Make the runtime auditable
+
+- Add a sanitized per-turn decision record: model runtime, contract, proposed
+  calls, policy decisions with reason codes, executions, render owner, timings.
+- Add startup/runtime provenance to the UI so a linked chat proves which harness
+  handled it.
+- Add the offered-versus-policy compatibility self-test.
+- Correct stale preview documentation.
+
+### Phase 2 — Build the conversation suite
+
+- Extend the current Playwright verifier with reusable multi-turn scenarios and
+  reversible artifact fixtures.
+- Implement the matrix above, prioritizing search continuations and all
+  stateful corrections because real usage already exposed those gaps.
+- Run independent family groups in parallel, but serialize writes that share a
+  backend or fixture account.
+- Add a small versioned VL fixture set with locally generated, non-private
+  images and deterministic answer keys.
+
+### Phase 3 — Repair by architecture category
+
+- Consolidate model-specific runtime selection in one function.
+- Represent prior successful objects explicitly for referential follow-ups.
+- Align tool capability classification, contract offering, and policy decisions.
+- Standardize tool results into bounded envelopes with source/object IDs.
+- Keep search refinement and evidence sufficiency generic.
+
+### Phase 4 — Accuracy and speed comparison
+
+- Compare the clean fine-tune with the base model using identical compact tools,
+  prompts, toggles, backend state, and semantic scoring.
+- Report functional accuracy, argument accuracy, unsupported success claims,
+  TTFT, total latency, and tokens. Do not compare one model on full schemas and
+  another on compact schemas.
+- Only consider more SFT/RL for failures classified as model-owned after the
+  harness audit.
+
+### Phase 4B — Regular-model repair and verification
+
+- Snapshot the enabled non-Odysseus model inventory.
+- Run the legacy-RAG compatibility matrix in bounded parallel groups, respecting
+  endpoint rate limits and shared backend write serialization.
+- Fix shared harness/provider defects first, then rerun all affected models.
+- Publish separate per-model scores and limitations; do not blend them into the
+  Odysseus fine-tune score.
+
+### Phase 5 — Ship gate
+
+Ship only when:
+
+- every family is at least 90% on sealed functional holdout;
+- overall functional accuracy is at least 95%;
+- realistic follow-up suite is at least 95%, with no repeated failure category;
+- image/VL fixture accuracy does not regress materially from the base model and
+  all attachment/follow-up/persistence flows pass;
+- routing/execution and safety invariants are 100%;
+- all reversible writes are API-verified and cleaned up;
+- search quality and production email are reported separately and honestly;
+- non-Odysseus models demonstrably retain legacy RAG;
+- every enabled regular model has a complete tested-tool compatibility record,
+  and every tool advertised as supported passes its functional checks;
+- no hidden prompt leakage, duplicate rendering, or false success remains;
+- pre-heretic passing weights and merged adapter backups remain recoverable.
+
+## Immediate next batch
+
+1. Expand VL fixtures to charts, screenshots, and image-to-tool turns;
+   investigate the shared base-model OCR limitation without hiding it behind a
+   silent external fallback.
+2. Add deliberately permissioned private-browser open/snapshot/click checks;
+   keep browser interaction unavailable when its boundary is not enabled.
+3. Bring the two configured local regular models online and run their matrix.
+4. Compare fine-tune versus untouched base with identical compact contracts,
+   backend state, prompts, and timing instrumentation.
+5. Run the sealed all-action holdout and prioritize failures by shared
+   layer rather than by prompt.

plans/photo-editor-professional-roadmap.md

@@ -0,0 +1,445 @@
+# Plan: Odysseus Professional Photo Editor
+
+> Source PRD: Conversation goal, "a Photoshop/Photopea clone with Odysseus style"
+
+## Product boundary
+
+Odysseus should provide the editing loop people expect from a professional
+layer-based photo editor without copying Photoshop's visual design or trying to
+match every specialist feature. The target is a dependable browser editor for
+real photo work: direct manipulation, non-destructive layers, precise masking,
+retouching, typography, export, recovery, and optional AI assistance.
+
+The existing quiet Odysseus interface remains the visual language. Dense tools
+are acceptable, but controls should stay restrained, compact, predictable, and
+usable on both desktop and touch devices.
+
+## Existing foundation
+
+The current editor already provides meaningful parts of this product:
+
+- Raster and editable text layers
+- Multi-layer selection, nested groups, clipping, visibility, opacity, and locks
+- Layer, group, and selection masks
+- Marquee, lasso, wand, SAM, Quick Mask, and saved selections
+- Brush, eraser, clone, crop, transform, and text tools
+- Blend modes, adjustment stacks, blur, and several image corrections
+- Rulers, guides, grid, snapping, zooming, and panning
+- Undo/redo history with a memory budget
+- Versioned layered-project serialization, autosave drafts, recovery, and export
+- Optional endpoint-backed inpaint and image-processing tools
+- Desktop and mobile editor layouts with Playwright release-gate coverage
+
+## Architectural decisions
+
+Durable decisions that apply across every phase:
+
+- **Editor ownership**: The editor remains an Odysseus feature. Do not embed a
+  third-party editor or imitate another product's chrome.
+- **Document format**: Continue the versioned Odysseus editor document. Every
+  new persistent capability requires a migration, validation, round-trip test,
+  and corrupt-input recovery behavior.
+- **Layer model**: Grow the document into explicit layer kinds rather than
+  hiding more behavior in raster canvases. The intended kinds are raster, text,
+  shape, adjustment, and placed/smart content.
+- **Non-destructive default**: Preserve source pixels and editable parameters
+  whenever practical. Destructive actions remain available as explicit Apply,
+  Rasterize, or Merge commands.
+- **Interaction engine**: Transform, crop, selections, text frames, masks, and
+  shapes share one pointer-session model for hit testing, pointer capture,
+  modifiers, snapping, cancellation, and undo transactions.
+- **Rendering**: Keep Canvas 2D as the compatibility renderer initially. Move
+  expensive compositing and pixel operations behind renderer/worker boundaries
+  before considering WebGL or WebGPU acceleration.
+- **History**: One continuous gesture creates one undo entry. Preview frames are
+  never separate history entries, and Cancel restores the exact starting state.
+- **Persistence routes**: Continue using `/api/editor-drafts` for layered draft
+  persistence and `/api/gallery` for media-library save/replace operations.
+- **AI boundary**: AI features consume capability-based image endpoints. Core
+  editing never requires a particular model, repository, or provider.
+- **Responsive behavior**: Desktop favors precision; touch targets gain larger
+  invisible hit areas without visually enlarging the whole interface.
+- **Testing**: Every phase adds deterministic geometry/unit tests and at least
+  one complete Playwright workflow covering persistence and undo where relevant.
+- **Incremental architecture**: New behavior leaves the main editor orchestrator
+  through small domain modules. Avoid broad refactors that do not deliver a
+  visible editing improvement in the same phase.
+
+---
+
+## Phase 1: Accurate Transform Frame
+
+**User stories**: I can clearly see and grab the transform frame at any zoom. I
+can resize from corners or sides without grabbing invisible or incorrect areas.
+
+### What to build
+
+Replace the four-corner-only frame with a shared frame geometry model. Render
+four corners, four edge handles, a rotation control, and an optional center
+pivot from the same geometry used for hit testing. Keep handles visually compact
+while providing touch-sized invisible targets. Make the frame stay aligned
+during zoom, pan, viewport resize, and when handles extend outside the image.
+
+### Acceptance criteria
+
+- [x] Eight resize handles, rotation control, and center pivot derive from one geometry result.
+- [x] Drawn handles and hit targets cannot disagree.
+- [x] Handles remain a stable visual size from minimum to maximum zoom.
+- [x] Touch hit targets are at least 40 CSS pixels without oversized visuals.
+- [x] Outside-canvas handles remain interactive and visible when space permits.
+- [x] Hover and active cursors match each handle's current screen direction.
+- [x] Desktop and mobile Playwright tests grab every handle successfully.
+
+---
+
+## Phase 2: Correct Rotated Resize
+
+**User stories**: I can resize a rotated layer naturally. The opposite side or
+corner stays fixed, and the frame follows my pointer rather than drifting.
+
+### What to build
+
+Calculate drag movement in the frame's rotated local coordinate system. Anchor
+the opposite handle in document space and derive the new center from that
+anchor. Support crossing an axis as a deliberate flip instead of clamping to a
+one-pixel box. Apply the same geometry to one layer, multiple layers, and a
+selection transform.
+
+### Acceptance criteria
+
+- [x] Rotated corner and edge drags follow the pointer on the frame's local axes.
+- [x] The opposite anchor remains fixed within a sub-pixel tolerance.
+- [x] Crossing width or height zero produces a predictable horizontal or vertical flip.
+- [x] Shift locks the starting aspect ratio.
+- [x] Alt/Option scales around the transform center.
+- [x] Combined Shift+Alt/Option behavior is deterministic.
+- [x] Rotation snaps to 15-degree increments with Shift and remains smooth otherwise.
+- [x] Geometry tests cover 0, 45, 90, 135, and arbitrary-degree rotations.
+
+---
+
+## Phase 3: Transform Interaction Polish
+
+**User stories**: Transform behaves like a professional tool on mouse, pen, and
+touch. I can see exact values, snap precisely, and never lose a drag at the edge.
+
+### What to build
+
+Use a unified pointer session with pointer capture, live modifiers, and a small
+contextual transform readout. Add accurate rotated-frame interior hit testing,
+keyboard nudging, frame snapping, and clear Apply/Cancel behavior. Keep the
+existing compact Odysseus styling and make the numeric popup a precision surface
+rather than a competing transform implementation.
+
+### Acceptance criteria
+
+- [x] Pointer capture keeps a drag alive outside the canvas and browser viewport.
+- [x] Clicking inside a rotated frame moves it; clicking its empty bounding-box corner does not.
+- [x] Live X, Y, W, H, and angle values stay synchronized with direct manipulation.
+- [x] Arrow keys nudge, Shift+Arrow performs a larger nudge, Enter applies, and Escape cancels.
+- [x] Layer edges, document center/edges, guides, and grid participate in transform snapping.
+- [x] Snap guides clearly identify the active alignment without obscuring the photo.
+- [x] A complete gesture creates exactly one undo step.
+- [x] Touch gestures do not conflict with viewport pinch/pan behavior.
+
+---
+
+## Phase 4: Transform Content Correctness
+
+**User stories**: Transforming layers never unexpectedly damages masks, text,
+group layout, clipping, or image quality. Saving and reopening preserves it.
+
+### What to build
+
+Route raster layers, text layers, linked and unlinked masks, selections, clipped
+layers, and grouped multi-selection through the same transform contract. Keep
+immutable source data during previews and validate the final result through
+undo, cancel, autosave, project download, and reopen.
+
+### Acceptance criteria
+
+- [x] Raster previews are always derived from the session source, never a prior preview.
+- [x] Editable text remains editable after scaling, rotation, and flipping.
+- [x] Linked masks follow the layer while unlinked masks remain in document space.
+- [x] Multi-layer transforms preserve relative centers, order, clipping, and group membership.
+- [x] Transforming a selection changes only the selection mask unless content transform is explicitly chosen.
+- [x] Apply, Cancel, Undo, Redo, autosave reopen, and project-file reopen produce matching pixels and metadata.
+- [x] Large transforms cannot allocate beyond the editor's documented surface budget.
+
+---
+
+## Phase 5: Shared Direct-Manipulation Sessions
+
+**User stories**: Crop, selections, masks, text boxes, and shapes feel consistent
+with Transform instead of each behaving like a separate mini application.
+
+### What to build
+
+Generalize the proven transform pointer session into a reusable interaction
+contract. Migrate crop and selection movement first as a visible tracer bullet,
+including modifiers, snapping, pointer capture, cancel, and one-step history.
+
+### Acceptance criteria
+
+- [x] Transform, crop, and selection movement use the same gesture lifecycle.
+- [x] Tool switching safely commits, cancels, or prompts according to one policy.
+- [x] No stale pointer session can modify a newly selected tool or document.
+- [x] Mouse, pen, and touch event behavior is covered by shared tests.
+- [x] Adding a future frame-based tool does not require another global event stack.
+
+---
+
+## Phase 6: Non-Destructive Placed Layers
+
+**User stories**: I can import an image, resize it repeatedly without cumulative
+quality loss, replace its source, and choose when to rasterize it.
+
+### What to build
+
+Introduce a placed/smart layer kind containing source pixels and persistent
+transform metadata. Import-as-layer uses this kind by default. Rendering applies
+the transform at composite time, while Rasterize produces a normal raster layer.
+
+### Acceptance criteria
+
+- [x] Repeated transforms render from the original source rather than resampling the last result.
+- [x] A placed layer can be replaced while preserving its transform and masks.
+- [x] Rasterize produces a visually matching editable raster layer.
+- [x] Masks, clipping, groups, blend modes, and opacity work with placed layers.
+- [x] Version migration and recovery handle missing or corrupt placed sources.
+- [x] Existing raster projects open without changed output.
+
+---
+
+## Phase 7: Professional Selections And Masks
+
+**User stories**: I can build, inspect, refine, save, transform, and reuse precise
+selections without manually repainting every edge.
+
+### What to build
+
+Unify marquee, lasso, wand, SAM, Quick Mask, and saved selections around one
+selection-mask model. Add explicit replace/add/subtract/intersect modes, feather,
+expand, contract, smooth, border, and a focused refine-edge workflow.
+
+### Acceptance criteria
+
+- [x] Every selection tool supports replace, add, subtract, and intersect modes.
+- [x] Feather, expand, contract, smooth, and border preview before applying.
+- [x] Quick Mask edits the same canonical selection shown by marching ants.
+- [x] Selection-to-layer-mask and layer-mask-to-selection round-trip accurately.
+- [x] Saved selections retain names and pixels across reopen.
+- [x] Edge refinement works without requiring an AI dependency.
+
+---
+
+## Phase 8: Paint And Retouch Workflow
+
+**User stories**: I can paint and retouch photographs with predictable strokes,
+reusable presets, and the controls expected for a mouse, pen, or touch device.
+
+### What to build
+
+Promote brush behavior into a reusable brush engine. Add spacing, smoothing,
+pressure mapping, blend mode, sampled color, presets, and stroke preview. Build
+healing, dodge, and burn as complete retouching paths using that engine.
+
+### Acceptance criteria
+
+- [x] Brush, eraser, clone, masks, and inpaint share spacing and smoothing behavior.
+- [x] Pressure can independently affect size, opacity, or flow when supported.
+- [x] Eyedropper samples composite or active-layer color.
+- [x] Brush presets can be created, named, selected, and deleted.
+- [x] Healing, dodge, and burn create one undo entry per stroke.
+- [x] Long strokes remain smooth without blocking the main interface.
+
+---
+
+## Phase 9: Editable Text And Shapes
+
+**User stories**: I can design labels, cards, and overlays with text and vector
+shapes that remain editable after saving and reopening.
+
+### What to build
+
+Add on-canvas text-frame editing, selection, caret behavior, typography, and
+alignment. Introduce shape layers for rectangle, ellipse, line, and path-backed
+polygons with editable fill, stroke, corners, and transform metadata.
+
+### Acceptance criteria
+
+- [x] Text is edited directly on canvas without immediately rasterizing.
+- [x] Font, size, weight, line height, letter spacing, alignment, and color persist.
+- [x] Rectangle, ellipse, line, and polygon shapes remain editable.
+- [x] Shape fill, stroke, width, and corner radius can be changed after creation.
+- [x] Text and shape layers support masks, clipping, groups, blend modes, and transform.
+- [x] Missing fonts fall back predictably without corrupting the project.
+
+---
+
+## Phase 10: Adjustment Layers And Color
+
+**User stories**: I can correct a photograph non-destructively and return later
+to modify the correction without reconstructing the edit.
+
+### What to build
+
+Promote adjustments into first-class layers with masks and clipping. Deliver
+Levels and Curves first, then exposure, white balance, hue/saturation, color
+balance, selective color, gradients, and channel-aware controls.
+
+### Acceptance criteria
+
+- [ ] Adjustment layers affect content below them and can be clipped or grouped.
+- [ ] Every adjustment has live preview, reset, visibility, opacity, mask, Apply, and Cancel behavior.
+- [ ] Levels includes histogram, input range, gamma, and output range.
+- [ ] Curves supports RGB and channel curves with editable points.
+- [ ] Color results match flattened export and project reopen.
+- [ ] Large previews are throttled or worker-backed and remain cancellable.
+
+---
+
+## Phase 11: Layer Effects And Filters
+
+**User stories**: I can add common visual effects without permanently altering
+the layer and can reorder or disable those effects later.
+
+### What to build
+
+Create an ordered non-destructive filter/effect stack. Begin with Gaussian blur,
+sharpen, shadow, stroke, and color overlay; then add filter masks and reusable
+effect presets.
+
+### Acceptance criteria
+
+- [ ] Effects can be added, reordered, toggled, edited, masked, and removed.
+- [ ] Drop shadow, stroke, color overlay, blur, and sharpen survive project reopen.
+- [ ] Effects render correctly inside groups and clipping stacks.
+- [ ] Apply/rasterize produces a pixel-equivalent raster result.
+- [ ] Expensive filters expose progress and cancellation.
+
+---
+
+## Phase 12: Odysseus Professional Workspace
+
+**User stories**: I can work quickly without fighting floating windows or losing
+the active tool, layer, selection, or document context.
+
+### What to build
+
+Refine the existing shell into a consistent professional workspace: contextual
+tool options, properties inspector, panel persistence, command search, status
+information, multi-document switching, and compact touch sheets. Preserve the
+current Odysseus palette, typography, restrained borders, and frosted surfaces.
+
+### Acceptance criteria
+
+- [ ] Tool options appear in one predictable location and never duplicate popup state.
+- [ ] Panels remember size, collapsed state, and position per device class.
+- [ ] The properties inspector follows the active layer, mask, selection, or tool.
+- [ ] Command search exposes actions and shortcuts without adding toolbar clutter.
+- [ ] Switching documents preserves independent history, zoom, pan, and selection.
+- [ ] Mobile prioritizes canvas area while keeping all commands reachable.
+
+---
+
+## Phase 13: File Interchange And Export
+
+**User stories**: I can bring common assets into Odysseus and export predictable
+results without losing transparency, dimensions, or color intent.
+
+### What to build
+
+Strengthen image import/export first, then add layered interchange where a
+maintained parser makes it safe. Keep Odysseus project files as the lossless
+source of truth and clearly report what an external format cannot preserve.
+
+### Acceptance criteria
+
+- [ ] PNG, JPEG, WebP, and supported modern image imports honor orientation and transparency.
+- [ ] Export exposes format, dimensions, quality, metadata, and transparency choices.
+- [ ] Copy/paste and drag/drop preserve alpha and use placed layers when appropriate.
+- [ ] Layered imports report unsupported features instead of silently flattening them.
+- [ ] Exported pixels are covered by deterministic visual comparisons.
+
+---
+
+## Phase 14: Large-Document Performance And Recovery
+
+**User stories**: Large photos and layered projects remain responsive, autosave
+reliably, and recover after a crash or interrupted network connection.
+
+### What to build
+
+Move serialization, thumbnails, filters, and suitable pixel operations into
+workers. Add dirty-region rendering, reusable surfaces, measurable memory
+budgets, operation cancellation, autosave generations, and recovery diagnostics.
+
+### Acceptance criteria
+
+- [ ] Normal interactions remain responsive on the agreed 4K multi-layer benchmark.
+- [ ] Compositing avoids rebuilding unaffected layers and thumbnails.
+- [ ] History and document surfaces stay within explicit memory limits.
+- [ ] Closing or switching documents cancels stale work safely.
+- [ ] Autosave never lets an older request overwrite newer state.
+- [ ] Recovery can identify the last complete generation and explain skipped data.
+
+---
+
+## Phase 15: Odysseus-Native Assisted Editing
+
+**User stories**: I can use an available local or remote image capability as an
+editing assistant while retaining masks, layers, undo, privacy choices, and
+normal manual controls.
+
+### What to build
+
+Standardize image capability discovery and requests for generation, editing,
+inpainting, segmentation, restoration, and upscaling. Results enter the document
+as named layers with provenance and reusable masks. Add orchestration only after
+the manual operation it assists is dependable.
+
+### Acceptance criteria
+
+- [ ] The UI describes required capabilities rather than model or provider names.
+- [ ] Memory and unrelated chat context are not sent to image endpoints.
+- [ ] Requests show progress, support cancellation, and cannot update a closed document.
+- [ ] Generated results arrive as reversible layers with prompt/settings metadata.
+- [ ] A failed endpoint leaves the source document unchanged and offers a useful retry path.
+- [ ] Manual selection and masking remain available when assisted tools are absent.
+
+---
+
+## Phase 16: Professional Release Gate
+
+**User stories**: I can trust the editor for real work and understand what is
+unsupported before committing an edit.
+
+### What to build
+
+Create a release gate around complete user journeys rather than isolated button
+tests. Cover accessibility, keyboard-only operation, touch, browser differences,
+pixel correctness, persistence, failure recovery, and large-document behavior.
+
+### Acceptance criteria
+
+- [ ] Core workflows pass on current Chromium and Firefox desktop builds.
+- [ ] Mobile workflows pass at representative phone and tablet viewports.
+- [ ] Keyboard-only users can reach every command and escape every modal state.
+- [ ] Transform, masks, text, adjustments, export, and reopen have pixel/metadata regression tests.
+- [ ] No supported action silently flattens or discards editable document data.
+- [ ] The ALPHA badge can be removed based on explicit reliability metrics.
+
+---
+
+## Recommended delivery order
+
+The first four phases are one focused Transform 2.0 program and should ship in
+order. Phases 5 and 6 establish the interaction and document foundations needed
+for the remaining professional tools. After that, phases 7 through 13 can be
+prioritized by user value, while performance and release-gate work continue as
+part of every phase rather than being deferred entirely to the end.
+
+The recommended first milestone is complete when Phases 1 through 4 are live:
+transforming one layer, multiple layers, text, masks, and selections feels
+precise on desktop and mobile and remains correct through undo and reopen.

plans/photo-editor-remaining-scope.md

@@ -0,0 +1,159 @@
+# Photo Editor Remaining Scope
+
+Date: 2026-08-29
+
+## Current verdict
+
+Odysseus is now a credible layered everyday editor, not an editor mockup. The
+first nine roadmap phases are implemented: professional transform geometry,
+shared direct-manipulation sessions, retained placed content, unified
+selections and masks, a reusable brush/retouch engine, and retained text and
+shape layers.
+
+Phase 10 is functionally advanced but not closed. First-class adjustment layers
+now support Levels, Curves, Exposure, White Balance, Brightness/Contrast,
+Hue/Saturation/Lightness, Color Balance, Selective Color, and Gradient Map.
+They participate in clipping, groups, masks, visibility, opacity, history, the
+v14 document format, and flattening. Retained effects have since been added as
+a separate ordered stack with Gaussian Blur, Color Overlay, Drop Shadow, and
+Stroke, including editable colors, visibility, opacity, reorder, rasterize,
+history, persistence, and migration.
+
+Practical readiness estimate:
+
+- Everyday layered photo editing: **about 88%**
+- Dependable professional v1 described by the roadmap: **about 62%**
+- Broad Photoshop/Photopea feature parity: **about 50%**
+
+The remaining gap is dominated by large-document rendering outside the live
+composite path, workspace consolidation, interchange/color policy, and release
+proof rather than basic canvas tools.
+
+## Verification snapshot
+
+- The focused editor unit suite currently passes **31 tests** in Docker.
+- The full photo-editor browser suite currently has **41 passing workflows**;
+  the nested-group selection workflow initially exposed a row-hit regression,
+  which now passes on isolated rerun after the slider-selection fix. The new
+  group-effects workflow also passes.
+- The new adjustment tests exercise deterministic pixel math, nested parameter
+  normalization, retained metadata, undo/redo, clipping, masks, and draft
+  reopen.
+- The latest editor changes have not yet been rebuilt into the live `7011`
+  container.
+
+## Close Phase 10
+
+This is the immediate release slice.
+
+1. Finish the bounded preview path for large documents. Downsampled previews
+   now keep control movement responsive and full resolution is restored for
+   commit/export. Live worker composites now use generation checks, latest-only
+   coalescing, and close/reopen invalidation; extend the same guarantees to
+   remaining preview paths.
+2. Add flattened-export versus reopened-project pixel comparisons for every
+   adjustment family, including groups, clipping, masks, blend mode, and
+   partial opacity.
+3. Validate the color algorithms visually. White Balance and Selective Color
+   are currently deterministic approximations, not color-managed photographic
+   transforms.
+4. Test every adjustment popup on phone and desktop viewports, including tall
+   popups, color inputs, drag, Reset, Apply, Cancel, and Escape.
+5. Decide the migration path for the older per-raster `adjLayers` stack. It can
+   remain readable for compatibility, but new UI should converge on first-class
+   adjustment layers instead of maintaining two competing concepts.
+6. Bump static cache versions, rebuild the live container, and run a short
+   visual smoke test on `7011`.
+
+## Phase 11: Retained effects and filters
+
+The retained-effects slice is implemented for raster/placed/text/shape-compatible
+layer output: Gaussian Blur, Sharpen, Color Overlay, Drop Shadow, and Stroke
+have editable colors/parameters, visibility, opacity, reorder, rasterize,
+history, migration, and reopen support. Effect-specific masks, presets, and
+group-level effects are also implemented and covered by focused browser tests.
+Remaining work is:
+
+1. Extend worker coverage to serialization and remaining preview paths.
+   Thumbnail encoding, retained-effect rasterization, and live composite
+   rendering now use a worker where OffscreenCanvas is available, with
+   synchronous compatibility fallbacks. Generation invalidation, latest-only
+   coalescing, and CPU loop cancellation protect live rendering.
+2. Add explicit group-effect blend/ordering tests for nested groups and
+   non-default blend modes, plus visual comparisons for effect stacks.
+
+Introduce the renderer/worker cancellation boundary here rather than adding
+more synchronous full-canvas filters that Phase 14 must immediately replace.
+
+## Phase 12: Professional workspace
+
+Consolidate fragmented popups into one contextual properties surface. Persist
+panel layout by device class, add command search, expose stable document status,
+and support multiple open documents with independent history, zoom, pan, and
+selection. Mobile should use canvas-first sheets rather than compressed desktop
+panels.
+
+## Phase 13: Interchange and export
+
+Harden orientation, transparency, metadata, and color behavior for PNG, JPEG,
+and WebP first. Add copy/paste and drag/drop through placed layers. Treat
+layered formats as explicit compatibility projects: unsupported PSD/TIFF/HEIC
+features must be reported, never silently discarded. Odysseus project files
+remain the lossless source of truth.
+
+## Phase 14: Performance and recovery
+
+Move remaining preview/pixel paths into workers. Thumbnail encoding,
+autosave serialization, adjustment rendering, and retained-effect rendering
+now have worker-backed paths with compatibility fallbacks. Add
+dirty-region compositing, reusable render surfaces, cancellation tokens,
+operation telemetry, a documented surface/history budget, autosave generations,
+and a checked-in 4K multi-layer benchmark.
+
+This phase is the main architectural risk. Canvas 2D remains a valid
+compatibility renderer, but full-document synchronous passes will not scale to
+professional documents.
+
+## Phase 15: Assisted editing
+
+Normalize generation, editing, inpainting, segmentation, restoration, and
+upscaling behind capability-based endpoints. Keep model/provider names out of
+editor logic. Requests must exclude chat memory, show progress, cancel safely,
+and return named reversible layers with provenance. Manual tools remain fully
+usable without an endpoint.
+
+Much of the endpoint plumbing already exists; the remaining work is consistent
+capability discovery, lifecycle safety, and editor-native result handling.
+
+## Phase 16: Release gate
+
+Run complete user journeys on Chromium and Firefox desktop plus representative
+phone/tablet viewports. Add keyboard-only and accessibility coverage, mixed
+20-edit persistence/export tests, failure recovery, and large-document stress
+tests. No supported operation may silently flatten or discard retained state.
+
+## Architecture debt to control
+
+- `galleryEditor.js` is still a large orchestrator. Continue extracting domain
+  modules as visible features move, without a broad rewrite.
+- Legacy raster adjustment sublayers and first-class adjustment layers overlap.
+  Converge on the first-class model.
+- Pixel effects still rely heavily on synchronous full-canvas work.
+- `static/style.css` carries substantial editor-specific surface area and needs
+  clearer component boundaries before workspace customization expands.
+- The repository worktree contains many unrelated changes. Editor release and
+  merge decisions require a scoped diff or clean integration branch.
+
+## Recommended execution order
+
+1. Close and deploy Phase 10.
+2. Build Phase 11 through a cancellable render boundary.
+3. Consolidate the workspace in Phase 12.
+4. Define color/metadata policy and complete Phase 13.
+5. Finish worker rendering, stress, and recovery in Phase 14.
+6. Normalize assisted editing in Phase 15.
+7. Run the cross-browser professional release gate in Phase 16.
+
+Do not expand into full PSD fidelity, CMYK production, RAW development, 3D, or
+complete Photoshop parity before this critical path passes. Those are separate
+product decisions, not prerequisites for a strong Odysseus editor.

requirements-optional.txt

@@ -1,4 +1,7 @@
 # Optional dependencies — install only if you use the corresponding feature.
+# Local OCR for screenshots, scans, labels, and coordinate-grounded text extraction.
+rapidocr==3.9.2
+onnxruntime>=1.20,<2
 # The app handles their absence gracefully (clear error message on first use).
 #
 # Note: chromadb-client + fastembed moved to requirements.txt — RAG, semantic
@@ -44,3 +47,6 @@ PyMuPDF
 # [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
 # the dependency-age discussion in issue #485.
 markitdown[docx,pptx,xlsx,xls]==0.1.6
+
+# Photoshop PSD opening / flattened previews / layer inspection.
+psd-tools

requirements.txt

@@ -8,6 +8,11 @@ pydantic>=2.13.4
 pydantic-settings>=2.14.1
 SQLAlchemy
 pypdf
+pypdfium2
+Pillow
+faster-whisper
+PyPDF2
+pdfplumber
 beautifulsoup4
 charset-normalizer
 numpy
@@ -19,6 +24,7 @@ numpy
 chromadb-client
 fastembed
 youtube-transcript-api
+yt-dlp
 # Markdown rendering for research reports (src/visual_report.py).
 # Imported at module-top so it's a hard core dep, not optional.
 markdown

resources/skills/agent/artifact-completion/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: artifact-completion
+description: Create requested artifacts early, iterate from concrete output, and verify final deliverables
+version: 1.0.0
+category: agent
+tags: [artifacts, files, verification, workflow]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the task requires a file, patch, report, document, image, archive, configuration, or other persistent deliverable rather than only a text answer.
+
+## Procedure
+
+1. Extract the required deliverable path, format, content constraints, and acceptance criteria.
+2. Inspect the source material and existing target without delaying the first valid artifact.
+3. Create a minimal complete version at the required location, then iterate from that concrete output.
+4. Use the format's native parser, renderer, compiler, or test tool to inspect the artifact.
+5. Repair specific validation, content, or presentation failures while preserving correct portions.
+6. Confirm the final path, file type, required content, and usability before reporting completion.
+
+## Pitfalls
+
+- Do not spend the full task budget inspecting without creating the requested output.
+- Do not place the artifact at a convenient path when the task specifies another location.
+- Do not use a filename extension as proof that the file is valid in that format.
+- Do not report completion while placeholders, missing sections, parse errors, or failed checks remain.
+
+## Verification
+
+- The artifact exists at the required path and opens or parses successfully.
+- Required sections, fields, labels, or visual elements are present.
+- Relevant tests, render checks, or validators pass.

resources/skills/agent/terminal-recovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: terminal-recovery
+description: Recover from failed terminal commands using evidence-driven diagnosis and bounded retries
+version: 1.0.0
+category: agent
+tags: [terminal, shell, debugging, recovery]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a command fails, times out, produces incomplete output, or behaves differently from what the task requires.
+
+## Procedure
+
+1. Read the command, exit status, standard output, and standard error before choosing a response.
+2. Confirm the working directory, relevant files, executable availability, permissions, and environment assumptions with minimal read-only probes.
+3. Classify the failure as syntax, missing dependency, wrong path, permissions, resource pressure, timeout, service state, or task logic.
+4. Change one relevant condition and retry the narrowest command that can test the diagnosis.
+5. For a long-running command, use the returned session identifier to poll or provide input instead of launching duplicates.
+6. After recovery, run the original acceptance check and inspect the resulting files or service state.
+
+## Pitfalls
+
+- Do not rerun an unchanged failing command repeatedly.
+- Do not install packages or change global configuration before confirming they are missing and necessary.
+- Do not launch a second server or training job before checking for an existing process and port or device conflicts.
+- Do not treat partial output or a zero exit status as proof that the requested state was produced.
+
+## Verification
+
+- The diagnosed cause is supported by command output or environment state.
+- The corrected command exits as expected.
+- The requested artifact, process, or state passes an independent acceptance check.

resources/skills/agent/tool-discovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: tool-discovery
+description: Discover the smallest capable tool set and confirm argument schemas before acting
+version: 1.0.0
+category: agent
+tags: [tools, discovery, routing, schemas]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires tools whose names, capabilities, or argument shapes are not already clear. This is especially useful when many tools are available or a previous call failed because the wrong tool or parameters were selected.
+
+## Procedure
+
+1. Translate the request into required capabilities such as reading, searching, editing, executing, browsing, or verifying.
+2. Search the tool index for those capabilities and inspect the returned tool descriptions and schemas.
+3. Prefer one direct tool over a chain of indirect tools when it can complete the operation and provide evidence.
+4. Check required parameters, identifiers, path rules, side effects, and approval requirements before calling the tool.
+5. Make a small read-only probe when the environment or target is uncertain.
+6. Execute the selected action, inspect the result, and only broaden the tool search if the result shows a concrete capability gap.
+
+## Pitfalls
+
+- Do not guess tool names or argument keys from memory when the index or schema is available.
+- Do not load unrelated tool groups into context.
+- Do not repeat the same failed call without changing the arguments or strategy.
+- Do not use a broad shell or browser workaround when a scoped native tool already owns the operation.
+
+## Verification
+
+- The chosen tool directly matches the required capability.
+- Required arguments follow the exposed schema.
+- The result contains evidence of the requested effect or a specific error that guides the next step.

resources/skills/agent/verified-state-change/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: verified-state-change
+description: Make scoped state changes with target confirmation, minimal mutation, and read-back verification
+version: 1.0.0
+category: agent
+tags: [state, mutation, verification, safety]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when creating, editing, deleting, moving, sending, scheduling, or otherwise changing persistent state through an application, API, filesystem, or service.
+
+## Procedure
+
+1. Read the current state and identify the target using stable identifiers plus enough content to disambiguate it.
+2. Preserve fields the user did not ask to change and choose the narrowest supported mutation.
+3. For destructive or externally visible actions, confirm that the user's instruction authorizes the exact target and effect.
+4. Perform the mutation once and capture the returned identifier, status, or revision.
+5. Read the target again through an independent list, fetch, status, or content operation.
+6. Compare the observed state with the requested outcome and repair only the specific mismatch.
+
+## Pitfalls
+
+- Do not infer the target from a stale active item when a stable identifier can be fetched.
+- Do not report success from an accepted request alone; asynchronous or partial operations may not have completed.
+- Do not replace an entire object when a field-level update is supported and safer.
+- Do not silently broaden a mutation to adjacent files, records, accounts, or services.
+
+## Verification
+
+- The target identity was confirmed before mutation.
+- A read-back shows the intended values and preserves unrelated state.
+- Any external effect has a concrete status, identifier, or observable result.

resources/skills/communication/action-evidence-synthesis/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: action-evidence-synthesis
+description: "Turn messages, meeting notes, and documents into sourced decisions, actions, dependencies, and risks"
+version: 1.0.0
+category: communication
+tags: [messages, meetings, actions, status, evidence]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when information is fragmented across messages, meeting notes, transcripts, or documents and the user needs an action list, status summary, feasibility assessment, or executive brief.
+
+Do not use when the source material is unavailable or when the user only wants a verbatim transcript.
+
+## Procedure
+
+1. Identify the requested scope, audience, time window, and decision to support.
+2. Gather the relevant records in full and preserve stable source identifiers, authors, and timestamps.
+3. Extract explicit decisions, commitments, requests, owners, dates, dependencies, blockers, and changed facts.
+4. Reconcile revisions by preferring the newest authoritative record; keep unresolved conflicts visible instead of guessing.
+5. Separate observed facts from inferred owners, dates, urgency, feasibility, or recommendations, and label every inference as tentative.
+6. Produce the requested format with concise source references beside consequential claims and a final list of open questions.
+
+## Pitfalls
+
+- Do not turn discussion or speculation into a confirmed decision.
+- Do not invent owners or deadlines when none were assigned.
+- Do not silently discard older records that explain a changed commitment.
+- Do not send messages, create tasks, or update calendars unless the user separately authorizes those actions.
+
+## Verification
+
+- Every action has a source, status, and explicit or tentative owner and due date.
+- Conflicting values and revisions are resolved or visibly flagged.
+- The output covers decisions, actions, dependencies, risks, and open questions relevant to the request.
+

resources/skills/communication/reviewable-external-draft/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: reviewable-external-draft
+description: "Reconcile source evidence and prepare an accurate external-facing draft without bypassing review"
+version: 1.0.0
+category: communication
+tags: [drafting, email, messages, review, reconciliation]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when preparing a client, customer, partner, leadership, or other external-facing update from internal messages or documents.
+
+Do not use this procedure to send immediately unless the user explicitly authorizes the exact recipient and final content.
+
+## Procedure
+
+1. Confirm the audience, communication channel, requested tone, and whether the user asked for a draft or an immediate send.
+2. Gather the relevant source records and identify the latest values, dates, commitments, and unresolved discrepancies.
+3. Resolve recipient identity through the available contact source and avoid inferring internal versus external status from a display name alone.
+4. Draft only claims supported by the collected evidence; qualify uncertainty and omit internal-only detail that the audience should not receive.
+5. Save or present a reviewable draft through the native draft or document capability.
+6. Report the draft identifier or location plus any reconciliation notes that require human review.
+
+## Pitfalls
+
+- Do not send a draft merely because a send-capable tool is available.
+- Do not copy stale figures when a later correction exists.
+- Do not conceal unresolved discrepancies behind polished prose.
+- Do not expose private internal discussion, credentials, or unrelated personal data.
+
+## Verification
+
+- Recipient identity and communication mode match the request.
+- Dates, figures, status, and commitments map to current source evidence.
+- The result remains reviewable unless an explicit send-now instruction authorized delivery.
+

resources/skills/communication/scheduling-coordination/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: scheduling-coordination
+description: "Coordinate availability, confirmations, calendar changes, and participant notifications with read-back verification"
+version: 1.0.0
+category: communication
+tags: [calendar, scheduling, coordination, availability]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when arranging or changing a meeting across multiple participants, calendars, time zones, or communication channels.
+
+Do not create or modify an event when the user asked only for available options or a draft invitation.
+
+## Procedure
+
+1. Extract participants, duration, date range, time zones, location constraints, and required attendees.
+2. Resolve participant identities and inspect the relevant availability using declared calendar and contact capabilities.
+3. Compute candidate intervals in one explicit reference time zone and reject conflicts or insufficient travel buffers.
+4. Present or draft a small set of viable options when confirmation is still required.
+5. After authorization or recorded participant confirmation, create or update the event once with stable attendee identifiers.
+6. Read the event back and verify title, start, end, time zone, attendees, location, and conferencing details before drafting notifications.
+
+## Pitfalls
+
+- Do not overwrite or cancel unrelated events to manufacture availability.
+- Do not mix local times without naming the time zone.
+- Do not treat a proposed time as confirmed.
+- Do not create duplicates when an existing event can be updated safely.
+
+## Verification
+
+- The selected interval satisfies duration, availability, and time-zone constraints.
+- The calendar read-back matches the authorized event details.
+- Notifications describe the same confirmed event and remain drafts unless sending was explicitly authorized.
+

resources/skills/communication/support-triage-and-routing/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: support-triage-and-routing
+description: "Prioritize support requests, identify owners, route internally, and prepare safe customer drafts"
+version: 1.0.0
+category: communication
+tags: [support, triage, urgency, routing, drafts]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when reviewing a support backlog, identifying urgent incidents, assigning internal ownership, or drafting customer responses.
+
+Do not use when the request is merely to summarize an unrelated inbox or when sender identity cannot be established safely.
+
+## Procedure
+
+1. Read each in-scope request in full and retain its stable message or ticket identifier.
+2. Resolve whether the sender is internal or external and identify the responsible internal team from available contacts and service ownership data.
+3. Classify urgency from impact and time sensitivity: critical for outage, data loss, security exposure, or imminent contractual breach; high for a blocked user without a workaround; medium for degraded service with a workaround; low for non-blocking inquiries.
+4. Record a concise problem statement, evidence, affected scope, workaround, owner, next action, and response deadline.
+5. Route internally only when the user has authorized operational messaging; prepare external responses as reviewable drafts by default.
+6. Re-read created assignments or drafts and produce an escalation summary grouped by urgency.
+
+## Pitfalls
+
+- Do not infer severity from emotional language alone.
+- Do not expose one customer's data in another customer's response.
+- Do not send externally when the task calls for triage or drafting.
+- Do not mark an issue routed without a stable owner or observable routing result.
+
+## Verification
+
+- Every issue has a stable source identifier, urgency rationale, owner, and next action.
+- Critical and high items have explicit response targets and escalation state.
+- External communication is a draft unless the user explicitly authorized sending.

resources/skills/dev/developer-docs/SKILL.md

@@ -0,0 +1,37 @@
+---
+name: developer-docs
+description: Find, read, and apply authoritative developer documentation during implementation
+version: 1.0.0
+category: dev
+tags: [docs, documentation, api, software-development]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when the user asks how a library, framework, API, protocol, CLI, or SDK works, or when implementation depends on version-specific behavior. Prefer this skill over guessing from memory.
+
+## Procedure
+
+1. Identify the exact product, package, version, and task. Ask one focused clarification only when the target is genuinely ambiguous.
+2. Prefer the vendor's or project's primary documentation, source repository, release notes, and API reference. Use a general search only to locate those sources.
+3. Read the relevant page or reference section, then apply the documented behavior to the user's codebase and active workspace.
+4. Separate documented facts from inference, and call out version or environment assumptions.
+5. For code changes, add a focused regression test for the documented contract and run it before reporting completion.
+
+## Pitfalls
+
+- Do not present search snippets, stale cached knowledge, or a third-party tutorial as authoritative when primary documentation is available.
+- Do not silently mix instructions from different major versions.
+- Do not claim an API or option exists without confirming it in the relevant reference.
+- Do not use web search for a local project task when the active workspace and local tools can answer it.
+
+## Verification
+
+- The cited or retrieved documentation matches the target version.
+- The implementation or answer distinguishes source-backed facts from inference.
+- Any code change has a focused test or a concrete verification command.

resources/skills/general/test-driven-development/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: test-driven-development
+description: Build or fix software with a focused red-green-refactor loop
+version: 1.0.0
+category: general
+tags: [tdd, testing, debugging, red-green-refactor]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when implementing a feature, fixing a bug, or changing behavior where a regression test can define the expected result. Prefer this workflow for parser, routing, agent-loop, and UI behavior changes.
+
+## Procedure
+
+1. Inspect the relevant code, existing tests, and local conventions before editing.
+2. Write the smallest regression test that demonstrates the requested behavior or reproduces the bug.
+3. Run that test and confirm it fails for the expected reason, not because the test setup is broken.
+4. Make the smallest production change that makes the test pass.
+5. Run the focused test again, then run the surrounding module suite.
+6. Review the diff for unrelated changes, brittle assertions, hidden state, and missing error paths.
+7. Report the tests run and any remaining coverage or environment limits.
+
+## Pitfalls
+
+- Do not write a test that only mirrors the implementation; assert the user-visible contract.
+- Do not weaken an assertion just to make a failing test pass.
+- Do not skip the focused failing-test step when the behavior is observable in a local test.
+- Keep network, filesystem, and model calls deterministic with fakes or fixtures unless the integration itself is under test.
+
+## Verification
+
+- The new regression test fails before the fix and passes after it.
+- The relevant focused suite passes.
+- The broader suite passes or its failure is explained with evidence.
+- The final diff contains the test and the production change needed for the same behavior.

resources/skills/media/multimodal-evidence/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: multimodal-evidence
+description: Extract and verify evidence from images, documents, and video without redundant inspection
+version: 1.0.1
+category: media
+tags: [image, video, document, evidence, ocr]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the answer or requested artifact depends on visual, temporal, tabular, or textual evidence contained in images, documents, or video.
+
+## Procedure
+
+1. Identify the evidence required: objects, text, values, ordering, timestamps, labels, or visual relationships.
+2. Inspect the whole input or a broad representative sample first to establish structure and likely evidence locations.
+3. Narrow to relevant pages, frames, regions, or time intervals and record observations with their locations.
+4. Use the format's native parser for exact text and numbers: for example `python-docx` or ZIP/XML inspection for DOCX, `pdftotext` or a PDF library for PDF, spreadsheet readers for XLSX, and OCR only when the source is image-based. Do not search binary office files with plain `grep` or `cat`.
+5. Resolve conflicts with one targeted reinspection at better scale or a nearby frame rather than repeating the same crop.
+6. Build the answer or artifact from the evidence ledger and perform a final coverage check against every requested item.
+
+## Pitfalls
+
+- Do not infer unseen content from filenames, surrounding text, or a single thumbnail.
+- Do not repeatedly inspect nearly identical regions without a new hypothesis.
+- Do not trust OCR blindly for small labels, punctuation, or numeric values.
+- Do not finalize before checking that every requested item has supporting evidence.
+
+## Verification
+
+- Each factual output can be traced to a page, frame, region, or timestamp.
+- Exact labels and numbers were visually checked after extraction.
+- The final response or artifact covers all requested evidence categories.

resources/skills/research/web-research-fallback/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: web-research-fallback
+description: Research current web information with source-first search and controlled browser fallback
+version: 1.0.0
+category: research
+tags: [web, search, browser, sources, research]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires current public information, primary sources, multiple pages, or a site that cannot be reliably read from search results alone.
+
+## Procedure
+
+1. Define the facts needed and the preferred primary source for each fact.
+2. Search with a focused query and use result metadata to select likely authoritative pages.
+3. Open the source directly and extract the relevant passage, date, and URL rather than relying on a search snippet.
+4. Use the private browser when the page requires interaction, client-side rendering, navigation, or visual inspection.
+5. If a page fails, try a primary-source alternative or a narrower route before broadening to secondary sources.
+6. Cross-check unstable or consequential claims and distinguish source-backed facts from inference.
+
+## Pitfalls
+
+- Do not treat snippets as evidence for claims not visible on the source page.
+- Do not browse repeatedly without recording what each page established.
+- Do not use a secondary summary when an accessible primary source answers the question.
+- Do not claim freshness without checking publication or update dates.
+
+## Verification
+
+- Each important claim maps to a source that directly supports it.
+- Time-sensitive facts include an observed date or version.
+- Browser interaction produced the needed page state or a documented fallback was used.

routes/auth_routes.py

@@ -736,6 +736,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         _INT_RANGES = {
             "agent_max_rounds": (1, 200),
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
+            "auto_compact_threshold_percent": (50, 95),
         }
         for key in DEFAULT_SETTINGS:
             if key in RETIRED_SETTING_KEYS:

routes/calendar_routes.py

@@ -4,7 +4,7 @@ import logging
 import json
 import re
 import uuid
-from datetime import datetime, date, timedelta
+from datetime import datetime, date, timedelta, timezone
 from typing import Optional, List
 
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
@@ -13,7 +13,7 @@ from sqlalchemy import or_, and_
 from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
-from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
+from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent, Note
 from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
 from src.upload_handler import reserve_upload_references
@@ -207,6 +207,7 @@ class EventCreate(BaseModel):
     calendar_href: Optional[str] = None  # calendar id
     rrule: Optional[str] = None
     color: Optional[str] = None  # per-event color override
+    reminder_minutes: Optional[int] = None
 
 
 class EventUpdate(BaseModel):
@@ -218,6 +219,7 @@ class EventUpdate(BaseModel):
     location: Optional[str] = None
     rrule: Optional[str] = None
     color: Optional[str] = None
+    reminder_minutes: Optional[int] = None
 
 
 # ── Helpers ──
@@ -621,7 +623,133 @@ def _parse_dt(s: str) -> datetime:
         raise ValueError(f"could not parse datetime: {s!r}")
 
 
-def _event_to_dict(ev: CalendarEvent) -> dict:
+def _note_due_datetime(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        text = str(value).strip()
+        if text.endswith("Z"):
+            text = text[:-1] + "+00:00"
+        due = datetime.fromisoformat(text)
+        if due.tzinfo is not None:
+            return due.astimezone(timezone.utc).replace(tzinfo=None)
+        return due
+    except Exception:
+        return None
+
+
+def _calendar_reminder_for_event(db, owner: str, ev: CalendarEvent) -> dict | None:
+    """Return the closest Notes reminder that belongs to this calendar event.
+
+    Calendar alarms are currently stored as Notes rows. Older rows do not carry
+    an event UID, so match conservatively by the generated title plus due_date
+    before the event start. This keeps existing reminder notes visible on the
+    calendar without a schema migration.
+    """
+    if not db or not owner or not ev or not ev.dtstart:
+        return None
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return None
+
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    if not notes:
+        return None
+
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    best = None
+    best_minutes = None
+    for note in notes:
+        due = _note_due_datetime(note.due_date)
+        if due is None:
+            continue
+        minutes = round((start - due).total_seconds() / 60)
+        if minutes < 0 or minutes > 7 * 24 * 60:
+            continue
+        if best is None or minutes < best_minutes:
+            best = note
+            best_minutes = minutes
+    if best is None:
+        return None
+    return {
+        "note_id": best.id,
+        "due_date": best.due_date,
+        "minutes": best_minutes,
+    }
+
+
+def _delete_calendar_reminders_for_event(db, owner: str, ev: CalendarEvent) -> int:
+    if not db or not owner or not ev:
+        return 0
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return 0
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    for note in notes:
+        db.delete(note)
+    return len(notes)
+
+
+def _create_calendar_reminder_for_event(db, owner: str, ev: CalendarEvent, minutes_before: int) -> dict:
+    if not owner or not ev or not ev.dtstart:
+        return {"note_id": None, "skipped_reason": "missing event"}
+    minutes_before = max(0, int(minutes_before))
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    remind_at = start - timedelta(minutes=minutes_before)
+    now = datetime.utcnow() if getattr(ev, "is_utc", False) else datetime.now()
+    if start <= now:
+        return {"note_id": None, "skipped_reason": "event already passed"}
+    if remind_at <= now:
+        remind_at = now
+
+    summary = (ev.summary or "(no title)").strip() or "(no title)"
+    location = (ev.location or "").strip()
+    start_fmt = start.strftime("%a %b %d") if ev.all_day else start.strftime("%a %b %d %H:%M")
+    loc = f" @ {location}" if location else ""
+    due_date = remind_at.isoformat() + ("Z" if getattr(ev, "is_utc", False) and not ev.all_day else "")
+    note = Note(
+        id=str(uuid.uuid4()),
+        owner=owner,
+        title=f"Calendar reminder: {summary}",
+        items=json.dumps([{"text": f"{summary}{loc} — {start_fmt}", "done": False, "checked": False}]),
+        note_type="todo",
+        label="calendar",
+        due_date=due_date,
+        source="calendar",
+    )
+    db.add(note)
+    return {"note_id": note.id, "due_date": due_date, "minutes": minutes_before, "skipped_reason": None}
+
+
+def _event_to_dict(ev: CalendarEvent, db=None, owner: str | None = None) -> dict:
     """Convert a CalendarEvent model to the API dict format.
 
     Timed events whose stored datetimes represent UTC (is_utc=True) are
@@ -637,6 +765,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         suffix = "Z" if getattr(ev, "is_utc", False) else ""
         start_str = ev.dtstart.isoformat() + suffix
         end_str = ev.dtend.isoformat() + suffix
+    reminder = _calendar_reminder_for_event(db, owner, ev) if db and owner else None
     return {
         "uid": ev.uid,
         "summary": ev.summary or "",
@@ -653,6 +782,10 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
         "event_type": getattr(ev, "event_type", None),
         "importance": getattr(ev, "importance", None) or "normal",
+        "has_reminder": bool(reminder),
+        "reminder_note_id": reminder["note_id"] if reminder else None,
+        "reminder_due_date": reminder["due_date"] if reminder else None,
+        "reminder_minutes": reminder["minutes"] if reminder else None,
     }
 
 
@@ -684,7 +817,7 @@ def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
 
 
 def _expand_rrule(
-    ev: CalendarEvent, start: datetime, end: datetime
+    ev: CalendarEvent, start: datetime, end: datetime, db=None, owner: str | None = None
 ) -> List[dict]:
     """Expand a single recurring CalendarEvent into occurrence dicts.
 
@@ -702,7 +835,7 @@ def _expand_rrule(
         # Non-recurring — return the base event as-is. list_events
         # already filters non-recurring rows with the overlap check
         # in SQL, so we don't re-check here.
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -728,7 +861,7 @@ def _expand_rrule(
         logger.warning(
             "Failed to parse rrule=%r for event %s: %s", ev.rrule, ev.uid, ex
         )
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -746,7 +879,7 @@ def _expand_rrule(
     expand_start = start - duration
     results = []
     truncated = False
-    base = _event_to_dict(ev)
+    base = _event_to_dict(ev, db=db, owner=owner)
     exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
@@ -1185,7 +1318,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             # Expand recurring events into individual occurrences.
             expanded = []
             for e in events:
-                expanded.extend(_expand_rrule(e, start_dt, end_dt))
+                expanded.extend(_expand_rrule(e, start_dt, end_dt, db=db, owner=owner))
 
             # Sort by occurrence start time for consistent frontend ordering.
             truncated = any(e.get("truncated") for e in expanded)
@@ -1251,10 +1384,19 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 caldav_sync_pending="create" if cal.source == "caldav" else None,
             )
             db.add(ev)
+            reminder = None
+            if data.reminder_minutes is not None:
+                reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             db.commit()
+            db.refresh(ev)
             if cal.source == "caldav":
                 await _push_caldav_event_after_commit(owner, uid, "create")
-            return {"ok": True, "uid": uid}
+            return {
+                "ok": True,
+                "uid": uid,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1264,6 +1406,17 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    @router.get("/events/{uid}")
+    async def get_event(request: Request, uid: str):
+        owner = _require_user(request)
+        db = SessionLocal()
+        try:
+            base_uid = _resolve_base_uid(uid)
+            ev = _get_or_404_event(db, base_uid, owner)
+            return {"event": _event_to_dict(ev, db=db, owner=owner)}
+        finally:
+            db.close()
+
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
@@ -1300,13 +1453,24 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 ev.rrule = data.rrule
             if data.color is not None:
                 ev.color = data.color if data.color else None
+            reminder = None
+            reminder_fields = getattr(data, "model_fields_set", getattr(data, "__fields_set__", set()))
+            if "reminder_minutes" in reminder_fields:
+                _delete_calendar_reminders_for_event(db, owner, ev)
+                if data.reminder_minutes is not None:
+                    reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"
             db.commit()
+            db.refresh(ev)
             if is_caldav:
                 await _push_caldav_event_after_commit(owner, base_uid, "update")
-            return {"ok": True}
+            return {
+                "ok": True,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1328,6 +1492,8 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             ev = _get_or_404_event(db, base_uid, owner)
             is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if scope in {"occurrence", "instance"} and not is_occurrence_delete:
+                raise HTTPException(400, "Occurrence delete requires a recurring occurrence uid")
             if is_occurrence_delete:
                 key = _occurrence_exdate_key(uid, ev)
                 if not key:
@@ -1344,6 +1510,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
+            _delete_calendar_reminders_for_event(db, owner, ev)
             db.delete(ev)
             db.commit()
             if is_caldav:
@@ -1423,7 +1590,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 raise HTTPException(400, f"Invalid ICS file: {e}")
 
             # Sanitize display name — length cap + strip control chars
-            raw_name = calendar_name.strip() or (file.filename or "").replace(".ics", "").replace("_", " ").strip() or "Imported"
+            raw_name = calendar_name.strip() or re.sub(r"\.(?:calendar|ics|ical)$", "", file.filename or "", flags=re.IGNORECASE).replace("_", " ").strip() or "Imported"
             cal_display = "".join(c for c in raw_name if c.isprintable())[:120] or "Imported"
 
             target_cal = db.query(CalendarCal).filter(

routes/chat_helpers.py

@@ -3,6 +3,7 @@
 import asyncio
 import json
 import logging
+import math
 import os
 import re
 import time
@@ -25,6 +26,56 @@ from fastapi import HTTPException
 
 logger = logging.getLogger(__name__)
 
+_INVISIBLE_RESPONSE_CHARS = "\u2063\u200b\u200c\u200d\ufeff"
+
+
+def _skill_run_is_complex(agent_rounds: int, agent_tool_calls: int) -> bool:
+    """Keep one-off TUI edit loops out of automatic skill extraction."""
+    return agent_tool_calls >= 4 or (agent_rounds >= 5 and agent_tool_calls >= 3)
+
+
+def clean_repeated_assistant_content(text: object) -> str:
+    """Collapse repeated terminal assistant prose before history/SFT storage."""
+    value = str(text or "")
+    for char in _INVISIBLE_RESPONSE_CHARS:
+        value = value.replace(char, "")
+    value = value.strip()
+    if not value:
+        return ""
+
+    # Stream rejoin/finalization races can concatenate the same complete
+    # answer without separators. Collapse only exact 2-4x repetitions.
+    for copies in range(4, 1, -1):
+        if len(value) % copies == 0:
+            width = len(value) // copies
+            unit = value[:width]
+            if unit and unit * copies == value:
+                value = unit.strip()
+                break
+
+    # Interrupted/rejoined streams can leave a short suffix before a closing
+    # think tag at the edge of visible prose, e.g. "ls.\n</think>\n\nHere's...".
+    edge_close_re = re.compile(r"(?is)^\s*(?!<\s*think\b)[^<\n]{0,120}\s*</\s*think\s*>\s*")
+    while True:
+        cleaned = edge_close_re.sub("", value, count=1).strip()
+        if cleaned == value:
+            break
+        value = cleaned
+
+    first_line = next((line.strip() for line in value.splitlines() if line.strip()), "")
+    if 8 <= len(first_line) <= 180:
+        matches = list(re.finditer(r"(?m)^" + re.escape(first_line) + r"\s*$", value))
+        if len(matches) >= 2:
+            value = value[matches[0].start():matches[1].start()].strip()
+
+    value = re.sub(
+        r"(?is)(?<=[.!?])(?:[a-z]{1,12}\.)\s*</\s*think\s*>\s*$",
+        "",
+        value,
+    ).strip()
+    value = re.sub(r"(?is)\s*</\s*think\s*>\s*$", "", value).strip()
+    return value
+
 _CASUAL_OPENING_RE = re.compile(
     r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
     r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
@@ -36,6 +87,14 @@ _CASUAL_BLOCKLIST_RE = re.compile(
     r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
     re.IGNORECASE,
 )
+_PERSONAL_TOOL_CONTEXT_RE = re.compile(
+    r"\b(?:"
+    r"email|emails|mail|inbox|gmail|"
+    r"calendar|events?|meetings?|appointments?|schedule|"
+    r"notes?|todo|checklist|reminders?|tasks?"
+    r")\b",
+    re.IGNORECASE,
+)
 
 
 def _is_casual_low_signal(text: str) -> bool:
@@ -51,6 +110,14 @@ def _is_casual_low_signal(text: str) -> bool:
     return len(tail_words) <= 2
 
 
+def _truthy_request_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
 # Strong references to in-flight fire-and-forget tasks scheduled from this
 # module. asyncio only keeps weak references to tasks created via
 # create_task, so without this the GC can collect a task mid-execution and
@@ -60,6 +127,197 @@ _BG_TASKS: set[asyncio.Task] = set()
 _INCOGNITO_CONTEXTS: dict[str, dict[str, Any]] = {}
 _INCOGNITO_CONTEXT_TTL_SECONDS = 6 * 60 * 60
 _INCOGNITO_CONTEXT_MAX_MESSAGES = 80
+_SFT_TRACE_CAPTURE_ENV = "ODYSSEUS_SFT_TRACE_CAPTURE"
+_SFT_TRACE_DIR_ENV = "ODYSSEUS_SFT_TRACE_DIR"
+_RUNTIME_REVISION_ENV = "ODYSSEUS_RUNTIME_REVISION"
+
+
+def _sft_trace_capture_enabled(owner: str | None) -> bool:
+    flag = os.getenv(_SFT_TRACE_CAPTURE_ENV, "1").strip().lower()
+    return flag not in {"0", "false", "no", "off"} and str(owner or "").startswith("sft_")
+
+
+def _json_safe(value: Any) -> Any:
+    try:
+        json.dumps(value)
+        return value
+    except TypeError:
+        return str(value)
+
+
+def _last_user_message_for_trace(sess) -> str:
+    for msg in reversed(getattr(sess, "history", []) or []):
+        if getattr(msg, "role", None) == "user":
+            return str(getattr(msg, "content", "") or "").strip()
+    return ""
+
+
+def _append_sft_trace_record(
+    *,
+    owner: str | None,
+    session_id: str,
+    sess,
+    assistant_content: str,
+    metadata: dict,
+    message_id: Any = None,
+) -> None:
+    """Append one training-ready trace record for synthetic SFT users."""
+    if not _sft_trace_capture_enabled(owner):
+        return
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        os.makedirs(trace_dir, exist_ok=True)
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        runtime_revision = os.getenv(_RUNTIME_REVISION_ENV, "").strip()
+        record = {
+            "format": "odysseus_sft_trace_turn_v1",
+            "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "owner": owner,
+            "session_id": session_id,
+            "session_name": getattr(sess, "name", "") or "",
+            "message_id": message_id,
+            "user": _last_user_message_for_trace(sess),
+            "assistant": str(assistant_content or "").strip(),
+            "thinking": str((metadata or {}).get("thinking") or "").strip(),
+            "tool_events": _json_safe((metadata or {}).get("tool_events") or []),
+            "round_texts": _json_safe((metadata or {}).get("round_texts") or []),
+            "runtime_revision": runtime_revision,
+            "metadata": {
+                "model": (metadata or {}).get("model"),
+                "requested_model": (metadata or {}).get("requested_model"),
+                "endpoint_label": (metadata or {}).get("endpoint_label"),
+                "endpoint_id": (metadata or {}).get("endpoint_id"),
+                "response_time": (metadata or {}).get("response_time"),
+                "input_tokens": (metadata or {}).get("input_tokens"),
+                "output_tokens": (metadata or {}).get("output_tokens"),
+                "usage_buckets": _json_safe((metadata or {}).get("usage_buckets") or []),
+                "runtime_revision": runtime_revision,
+            },
+        }
+        _prune_sft_retry_rows_before_append(path, record)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception as exc:
+        logger.warning("Failed to append SFT trace record for %s/%s: %s", owner, session_id, exc)
+
+
+def remove_session_sft_trace_rows(owner: str | None, session_id: str) -> int:
+    """Remove every captured training row for a deleted synthetic session."""
+    if not _sft_trace_capture_enabled(owner) or not str(session_id or "").strip():
+        return 0
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        if not os.path.exists(path):
+            return 0
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as source:
+            for line in source:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if str(row.get("session_id") or "") != session_id:
+                    kept.append(raw)
+                    continue
+                row["deleted_from_training"] = True
+                removed.append(json.dumps(row, ensure_ascii=False))
+        if not removed:
+            return 0
+        tmp_path = f"{path}.{os.getpid()}.{time.time_ns()}.tmp"
+        with open(tmp_path, "w", encoding="utf-8") as target:
+            for raw in kept:
+                target.write(raw + "\n")
+        os.replace(tmp_path, path)
+        with open(path + ".trash", "a", encoding="utf-8") as trash:
+            for raw in removed:
+                trash.write(raw + "\n")
+        logger.info("Removed %d SFT trace row(s) for deleted session %s", len(removed), session_id)
+        return len(removed)
+    except Exception as exc:
+        logger.warning("Failed to remove SFT trace rows for session %s: %s", session_id, exc)
+        return 0
+
+
+def _prune_sft_retry_rows_before_append(path: str, record: dict[str, Any]) -> None:
+    """For SFT traces, keep only the latest retry for a repeated user send.
+
+    The browser resend flow can append a second identical user turn without
+    first calling the delete endpoint. Training wants the final attempt, not
+    both sends, so remove prior trailing rows in the same session with the same
+    user prompt before appending the replacement.
+    """
+    current_session = str(record.get("session_id") or "")
+    current_user = str(record.get("user") or "").strip()
+    if not current_session or not current_user or not os.path.exists(path):
+        return
+    kept: list[str] = []
+    parsed: list[tuple[str, dict | None]] = []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    parsed.append((raw, json.loads(raw)))
+                except json.JSONDecodeError:
+                    parsed.append((raw, None))
+
+        last_different_same_session = -1
+        for idx, (_raw, row) in enumerate(parsed):
+            if not isinstance(row, dict) or row.get("session_id") != current_session:
+                continue
+            if str(row.get("user") or "").strip() != current_user:
+                last_different_same_session = idx
+
+        removed: list[str] = []
+        for idx, (raw, row) in enumerate(parsed):
+            should_remove = (
+                idx > last_different_same_session
+                and isinstance(row, dict)
+                and row.get("session_id") == current_session
+                and str(row.get("user") or "").strip() == current_user
+            )
+            if should_remove:
+                tombstone = dict(row)
+                tombstone["deleted_from_training"] = True
+                tombstone["delete_reason"] = "sft_retry_replaced"
+                removed.append(json.dumps(tombstone, ensure_ascii=False))
+            else:
+                kept.append(raw)
+
+        if not removed:
+            return
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        with open(path + ".trash", "a", encoding="utf-8") as f:
+            for raw in removed:
+                f.write(raw + "\n")
+        logger.info(
+            "Removed %d prior SFT retry row(s) before appending replacement for session %s",
+            len(removed),
+            current_session,
+        )
+    except Exception as exc:
+        logger.warning("Failed to prune prior SFT retry rows for %s: %s", current_session, exc)
+
+
+def strip_tui_local_context(content: Any) -> Any:
+    """Remove client-only workspace metadata before persistence/display."""
+    if not isinstance(content, str):
+        return content
+    return re.sub(r"\s*<local_context\b[^>]*>.*?</local_context>\s*", "", content, flags=re.IGNORECASE | re.DOTALL).strip()
 
 
 def _spawn_bg(coro) -> asyncio.Task:
@@ -113,6 +371,8 @@ class PresetInfo:
     max_tokens: Optional[int]
     system_prompt: Optional[str]
     character_name: Optional[str]
+    persona_memory: Optional[str] = None
+    persona_memory_schema: str = "general"
 
 
 @dataclass
@@ -251,6 +511,14 @@ def needs_auto_name(name: str) -> bool:
     return False
 
 
+def fallback_session_title(text: str, *, max_words: int = 6) -> str:
+    words = re.findall(r"[A-Za-z0-9@._'-]+", text)
+    if not words:
+        return "New chat"
+    title = " ".join(words[:max_words]).strip()
+    return title[:60] or "New chat"
+
+
 async def auto_name_session(session_manager, sess):
     """Generate a short title for a session from its first user message."""
     try:
@@ -273,6 +541,17 @@ async def auto_name_session(session_manager, sess):
         if not first_msg:
             return
 
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model_name = str(getattr(sess, "model", "") or "")
+        if (
+            "ttft" in model_name.lower()
+            or re.search(r":18\d{3}\b", endpoint_url)
+        ):
+            title = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, title)
+            logger.info(f"Auto-named session {sess.id} deterministically: {title}")
+            return
+
         owner = getattr(sess, "owner", None)
         t_url, t_model, t_headers = resolve_task_endpoint(
             sess.endpoint_url, sess.model, sess.headers, owner=owner
@@ -294,9 +573,9 @@ async def auto_name_session(session_manager, sess):
                 {"role": "user", "content": first_msg},
             ],
             temperature=0.3,
-            max_tokens=4096,
+            max_tokens=64,
             headers=t_headers,
-            timeout=60,
+            timeout=15,
         )
 
         title = title.strip().strip('"\'').strip()
@@ -304,18 +583,47 @@ async def auto_name_session(session_manager, sess):
         # via the central helper.
         from src.text_helpers import strip_think
         title = strip_think(title, prose=False, prompt_echo=False)
-        if title and len(title) < 80:
-            session_manager.update_session_name(sess.id, title)
-            logger.info(f"Auto-named session {sess.id}: {title}")
+        if not title or len(title) >= 80 or "\n" in title:
+            fallback = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, fallback)
+            logger.info(
+                "Auto-named session %s with fallback title after unusable model title: %s",
+                sess.id,
+                fallback,
+            )
+            return
+
+        session_manager.update_session_name(sess.id, title)
+        logger.info(f"Auto-named session {sess.id}: {title}")
 
     except Exception as e:
         import traceback
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
+async def auto_name_session_after_stream(session_id: str, session_manager, sess):
+    """Delay chat title generation until the first response stream is settled."""
+    try:
+        waited = 0.0
+        while _is_session_stream_active(session_id) and waited < 30.0:
+            await asyncio.sleep(0.25)
+            waited += 0.25
+        # Let the final SSE chunk/message_saved bookkeeping clear before any
+        # title model call can contend with the user's visible response.
+        await asyncio.sleep(0.5)
+        try:
+            sess = session_manager.get_session(session_id)
+        except Exception as e:
+            logger.warning("[auto-name] Could not reload session %s before naming: %s", session_id, e)
+        await auto_name_session(session_manager, sess)
+    except Exception as e:
+        import traceback
+        logger.error(f"Deferred auto-name failed for {session_id}: {e}\n{traceback.format_exc()}")
+
+
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
-    temperature, max_tokens, system_prompt, char_name = (
+    temperature, max_tokens, system_prompt, char_name, persona_memory, persona_memory_schema = (
         chat_handler.validate_and_extract_preset(preset_id)
     )
     return PresetInfo(
@@ -323,6 +631,8 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo:
         max_tokens=max_tokens,
         system_prompt=system_prompt,
         character_name=char_name,
+        persona_memory=persona_memory,
+        persona_memory_schema=persona_memory_schema,
     )
 
 
@@ -406,14 +716,28 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
     return manifest
 
 
-def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
+def add_user_message(
+    sess,
+    chat_handler,
+    preprocessed: PreprocessedMessage,
+    incognito: bool = False,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
+):
     """Add user message to session history and update session name.
     Incognito messages must not mutate persistent session history, even in
     memory, because a later normal turn can persist the same session object."""
     if incognito:
         return
-    user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
-    sess.add_message(ChatMessage("user", preprocessed.user_content, metadata=user_meta))
+    user_meta = {}
+    if preprocessed.attachment_meta:
+        user_meta["attachments"] = preprocessed.attachment_meta
+    if interaction_mode in {"chat", "agent", "research"}:
+        user_meta["interaction_mode"] = interaction_mode
+    if auto_escalated:
+        user_meta["auto_escalated"] = True
+    clean_content = strip_tui_local_context(preprocessed.user_content)
+    sess.add_message(ChatMessage("user", clean_content, metadata=user_meta or None))
     chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
 
 
@@ -624,6 +948,10 @@ async def build_chat_context(
     agent_mode: bool = False,
     allow_tool_preprocessing: bool = True,
     defer_context_shaping: bool = False,
+    continuation_context_message: str | None = None,
+    persist_user_message: bool = True,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -647,14 +975,27 @@ async def build_chat_context(
     # Add user message to history. Nobody/incognito uses a request-local
     # transcript store instead of session history so stale saved chats cannot
     # bleed into context and the turn is not persisted.
-    if incognito:
-        user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
+    if persist_user_message and incognito:
+        user_meta = {}
+        if preprocessed.attachment_meta:
+            user_meta["attachments"] = preprocessed.attachment_meta
+        if interaction_mode in {"chat", "agent", "research"}:
+            user_meta["interaction_mode"] = interaction_mode
+        if auto_escalated:
+            user_meta["auto_escalated"] = True
         _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
-    else:
-        add_user_message(sess, chat_handler, preprocessed, incognito=False)
+    elif persist_user_message:
+        add_user_message(
+            sess,
+            chat_handler,
+            preprocessed,
+            incognito=False,
+            interaction_mode=interaction_mode,
+            auto_escalated=auto_escalated,
+        )
 
     # Fire events
-    if not incognito:
+    if persist_user_message and not incognito:
         fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
 
     # Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
@@ -666,13 +1007,22 @@ async def build_chat_context(
         getattr(chat_handler, "upload_handler", None),
         getattr(sess, "owner", None),
     )
-    casual_low_signal = _is_casual_low_signal(message)
+    context_message = (
+        str(continuation_context_message).strip()
+        if continuation_context_message
+        else message
+    )
+    casual_low_signal = _is_casual_low_signal(context_message)
 
     # Memory enabled?
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
     # Skills injection respects its own enable toggle (mirrors memory_enabled).
     # When off, the "Available skills" index is not added to the prompt.
-    skills_enabled = not incognito and uprefs.get("skills_enabled", True)
+    skills_enabled = (
+        not incognito
+        and uprefs.get("skills_enabled", True)
+        and getattr(sess, "skill_injection_enabled", True) is not False
+    )
     if not allow_tool_preprocessing:
         mem_enabled = False
         skills_enabled = False
@@ -697,22 +1047,40 @@ async def build_chat_context(
     if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
         use_rag_val = False
 
-    # If pre-fetched search context was provided (compare mode), skip live web search
-    skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
+    use_web_val = _truthy_request_flag(use_web)
+    # If pre-fetched search context was provided (compare mode), skip live web
+    # search. Personal app requests should be served by their tools; pre-search
+    # here caused calendar/email turns with use_web="false" to run irrelevant
+    # web searches before the agent even saw the tool surface.
+    skip_web = (
+        bool(search_context)
+        or not allow_tool_preprocessing
+        or casual_low_signal
+        or bool(agent_mode and _PERSONAL_TOOL_CONTEXT_RE.search(context_message or ""))
+    )
 
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
     # the sync path uses text_for_context.
-    _ctx_msg = preprocessed.enhanced_message if use_enhanced_message else preprocessed.text_for_context
+    _ctx_msg = (
+        context_message
+        if continuation_context_message
+        else (
+            preprocessed.enhanced_message
+            if use_enhanced_message
+            else preprocessed.text_for_context
+        )
+    )
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
-        use_web=use_web and not skip_web,
+        use_web=use_web_val and not skip_web,
         use_memory=mem_enabled,
         time_filter=time_filter,
         preset_system_prompt=preset.system_prompt,
         owner=user,
         character_name=preset.character_name,
+        persona_memory=preset.persona_memory,
         agent_mode=agent_mode,
         incognito=incognito,
         use_skills=skills_enabled,
@@ -811,10 +1179,17 @@ async def build_chat_context(
 
 
 def accumulate_token_usage(session_id: str, metrics: dict):
-    """Add input/output token counts to the session's running totals."""
+    """Add input/output token counts (and USD cost) to the session's totals."""
     in_t = metrics.get("input_tokens", 0)
     out_t = metrics.get("output_tokens", 0)
-    if not (in_t or out_t):
+    cost = metrics.get("cost_usd")
+    try:
+        cost = float(cost) if cost is not None else 0.0
+        if not math.isfinite(cost) or cost < 0:
+            cost = 0.0
+    except (TypeError, ValueError):
+        cost = 0.0
+    if not (in_t or out_t or cost):
         return
     db = SessionLocal()
     try:
@@ -822,6 +1197,8 @@ def accumulate_token_usage(session_id: str, metrics: dict):
         if db_s:
             db_s.total_input_tokens = (db_s.total_input_tokens or 0) + in_t
             db_s.total_output_tokens = (db_s.total_output_tokens or 0) + out_t
+            if cost:
+                db_s.total_cost_usd = (db_s.total_cost_usd or 0.0) + cost
             db.commit()
     except Exception:
         db.rollback()
@@ -874,6 +1251,21 @@ def _normalize_thinking(text: str) -> str:
 
     # Qwen3.5: "Thinking Process:" or "Thinking:" prefix
     if thinking_prefix_re.match(text.lstrip()):
+        # Tool-router checkpoints sometimes narrate several drafts and then
+        # emit an explicit final marker near the end. Prefer the last marker;
+        # the first ordinary-looking paragraph can still be internal review.
+        final_markers = list(re.finditer(
+            r"(?im)^\s*Final\s+(?:decision|answer|output(?:\s+generation)?)\s*:\s*",
+            text,
+        ))
+        if final_markers:
+            marker = final_markers[-1]
+            think = thinking_prefix_re.sub('', text[:marker.start()]).strip()
+            reply = text[marker.end():].strip()
+            if len(reply) >= 2 and reply[0] in {'\"', '\u201c'} and reply[-1] in {'\"', '\u201d'}:
+                reply = reply[1:-1].strip()
+            if reply:
+                return '<think>' + think + '</think>\n\n' + reply
         # Try clean boundary first
         m = re.match(
             r'^(Thinking(?:\s+Process)?:[\s\S]*?)(\n\n(?=[A-Z]|Hey|Yo|Hi|Sure|I |What|Here|Let|The |This |OK|Ok|Yes|No |So |Well |Thank|Alright|Of course|Absolutely|Great|Hello|As ))',
@@ -1002,6 +1394,23 @@ def clean_thinking_for_save(content: str, metadata: dict | None = None) -> tuple
         if info.get("time"):
             md["thinking_time"] = info["time"]
         return info["reply"], md
+    # A stopped stream can end before producing any answer prose. Preserve its
+    # partial reasoning as structured metadata so history rendering and the
+    # next Resume request can both recover it. Normal reasoning-only completed
+    # turns retain the legacy raw-content behavior.
+    if md.get("stopped"):
+        raw = str(content or "")
+        partial = re.match(
+            r'^\s*<think(?:ing)?(?:\s+time="([\d.]+)")?>([\s\S]*?)(?:</think(?:ing)?>\s*)?$',
+            raw,
+            re.IGNORECASE,
+        )
+        if partial and partial.group(2).strip():
+            md["thinking"] = partial.group(2).strip()
+            md["thinking_interrupted"] = True
+            if partial.group(1):
+                md["thinking_time"] = partial.group(1)
+            return "", md
     return content, md
 
 
@@ -1056,6 +1465,16 @@ def save_assistant_response(
     if tool_events:
         md["tool_events"] = tool_events
 
+    # The streaming route may have forwarded textual DSML/XML tool calls as
+    # deltas before the agent loop parsed them. Strip them again at the
+    # persistence boundary so raw tool markup cannot survive in history.
+    try:
+        from src.tool_parsing import strip_tool_blocks
+        full_response = strip_tool_blocks(str(full_response or "")).strip()
+    except Exception:
+        full_response = str(full_response or "")
+    full_response = clean_repeated_assistant_content(full_response)
+
     # Extract thinking into metadata (don't pollute message content with <think> tags)
     _think_info = _extract_thinking_meta(full_response)
     if _think_info:
@@ -1081,10 +1500,25 @@ def save_assistant_response(
     try:
         _last = sess.history[-1]
         _meta = getattr(_last, "metadata", None)
+        _message_id = _meta.get("_db_id") if isinstance(_meta, dict) else None
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+            message_id=_message_id,
+        )
         if isinstance(_meta, dict):
-            return _meta.get("_db_id")
+            return _message_id
     except (IndexError, AttributeError):
-        pass
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+        )
     return None
 
 
@@ -1157,6 +1591,8 @@ def run_post_response_tasks(
     owner: str = None,
     extract_skills: bool = True,
     allow_background_extraction: bool = True,
+    preset_manager=None,
+    persona_memory_schema: str = "general",
 ):
     """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.
 
@@ -1177,7 +1613,8 @@ def run_post_response_tasks(
     # Memory extraction — only every 4th message pair to avoid excess LLM calls
     _msg_count = len(sess.history) if hasattr(sess, 'history') else 0
     _should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0)
-    if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
+    _chat_memory_extract = getattr(sess, "memory_extraction_enabled", True) is not False
+    if allow_background_extraction and not incognito and not compare_mode and _chat_memory_extract and _should_extract and uprefs.get("auto_memory", True):
         from services.memory.memory_extractor import extract_and_store
         from src.task_endpoint import resolve_task_endpoint
         t_url, t_model, t_headers = resolve_task_endpoint(
@@ -1188,6 +1625,27 @@ def run_post_response_tasks(
             t_url, t_model, t_headers,
         )))
 
+    if (
+        allow_background_extraction
+        and not incognito
+        and not compare_mode
+        and _chat_memory_extract
+        and _should_extract
+        and uprefs.get("auto_memory", True)
+        and character_name
+    ):
+        if preset_manager is not None:
+            from services.memory.memory_extractor import update_persona_memory
+            from src.task_endpoint import resolve_task_endpoint
+            p_url, p_model, p_headers = resolve_task_endpoint(
+                sess.endpoint_url, sess.model, sess.headers, owner=owner,
+            )
+            _extraction_jobs.append(("persona-memory", update_persona_memory(
+                sess, preset_manager, character_name,
+                p_url, p_model, p_headers,
+                schema=persona_memory_schema,
+            )))
+
     # Skill extraction from complex agent runs. Only when the user actually
     # chose agent mode — not a chat we auto-escalated for a notes/calendar
     # intent, and never in incognito/compare.
@@ -1202,13 +1660,17 @@ def run_post_response_tasks(
         extract_skills, auto_skills_enabled, incognito, compare_mode,
         agent_rounds, agent_tool_calls, "set" if skills_manager else "MISSING",
     )
+    # A normal inspect/edit/verify turn is commonly three calls. Treating that
+    # as a reusable skill creates one-off titles and makes the skill library
+    # noisy. Automatic extraction is reserved for runs that demonstrate a
+    # genuinely longer procedure; explicit skill tools remain unaffected.
     if (
         extract_skills
         and allow_background_extraction
         and auto_skills_enabled
         and not incognito
         and not compare_mode
-        and (agent_rounds >= 2 or agent_tool_calls >= 2)
+        and _skill_run_is_complex(agent_rounds, agent_tool_calls)
     ):
         if skills_manager is None:
             logger.warning(
@@ -1245,4 +1707,4 @@ def run_post_response_tasks(
 
     # Auto-name
     if needs_auto_name(sess.name):
-        _spawn_bg(auto_name_session(session_manager, sess))
+        _spawn_bg(auto_name_session_after_stream(session_id, session_manager, sess))

routes/contacts/contacts_routes.py

@@ -5,8 +5,10 @@ CardDAV contacts integration. Reads from local Radicale, supports
 search and adding new contacts.
 """
 
+import asyncio
 import re
 import logging
+import threading
 import uuid
 import json
 import csv
@@ -19,10 +21,11 @@ from datetime import datetime
 from urllib.parse import urljoin, urlparse, urlunparse
 
 from core.log_safety import redact_url
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from fastapi import APIRouter, Query, Depends, Request, Response, HTTPException
 from typing import List, Dict, Optional
 
 from core.middleware import require_admin
+from src.auth_helpers import effective_user
 from src.url_safety import check_outbound_url
 
 logger = logging.getLogger(__name__)
@@ -93,22 +96,37 @@ def _normalize_contact(contact: Dict) -> Dict:
     if not name and emails:
         name = emails[0].split("@")[0]
     address = str(contact.get("address") or "").strip()
-    return {
+    out = {
         "uid": str(contact.get("uid") or uuid.uuid4()),
         "name": name,
         "emails": emails,
         "phones": phones,
         "address": address,
     }
+    owner = str(contact.get("owner") or "").strip()
+    if owner:
+        out["owner"] = owner
+    return out
 
 
-def _load_local_contacts() -> List[Dict]:
+def _contact_visible_to_owner(contact: Dict, owner: Optional[str]) -> bool:
+    owner = str(owner or "").strip()
+    row_owner = str(contact.get("owner") or "").strip()
+    if owner:
+        if row_owner:
+            return row_owner == owner
+        return not owner.startswith("sft_")
+    return True
+
+
+def _load_local_contacts(owner: Optional[str] = None) -> List[Dict]:
     try:
         if not LOCAL_CONTACTS_FILE.exists():
             return []
         data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
         rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        contacts = [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        return [c for c in contacts if _contact_visible_to_owner(c, owner)]
     except Exception as e:
         logger.error(f"Failed to load local contacts: {e}")
         return []
@@ -119,7 +137,9 @@ def _save_local_contacts(contacts: List[Dict]) -> None:
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
     _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["by_owner"] = {}
     _contact_cache["fetched_at"] = datetime.utcnow()
+    _contact_cache["failed_at"] = None
 
 
 # ── vCard parsing ──
@@ -264,7 +284,58 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None,
 
 # ── In-memory cache ──
 
-_contact_cache = {"contacts": [], "fetched_at": None}
+_CONTACT_CACHE_TTL_SECONDS = 60
+_CONTACT_FAILURE_BACKOFF_SECONDS = 120
+_CARDDAV_TIMEOUT = httpx.Timeout(5.0, connect=2.0)
+
+# CardDAV can be unavailable for a while. Keep the UI responsive by serving
+# the last known result (or an empty list on first use) while a single worker
+# attempts a refresh in the background.
+_contact_cache = {
+    "contacts": [],
+    "fetched_at": None,
+    "failed_at": None,
+    "by_owner": {},
+}
+_contact_fetch_lock = threading.Lock()
+
+
+def _cached_contacts(owner_key: str) -> List[Dict]:
+    cached = (_contact_cache.get("by_owner") or {}).get(owner_key) or {}
+    if owner_key and cached:
+        return cached.get("contacts") or []
+    return _contact_cache.get("contacts") or []
+
+
+def _mark_contact_fetch_failure(owner_key: str) -> List[Dict]:
+    now = datetime.utcnow()
+    stale_contacts = _cached_contacts(owner_key)
+    _contact_cache["failed_at"] = now
+    if owner_key:
+        _contact_cache.setdefault("by_owner", {})[owner_key] = {
+            "contacts": stale_contacts,
+            "fetched_at": now,
+        }
+    else:
+        _contact_cache["fetched_at"] = now
+    return stale_contacts
+
+
+def _contact_sync_status() -> Dict[str, str]:
+    """Return a safe, user-facing summary for contact autocomplete clients."""
+    if not _carddav_configured():
+        return {"state": "local", "message": "No contact sync is configured."}
+    if _contact_fetch_lock.locked():
+        return {"state": "syncing", "message": "Syncing contacts..."}
+    failed_at = _contact_cache.get("failed_at")
+    if failed_at:
+        age = (datetime.utcnow() - failed_at).total_seconds()
+        if age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return {
+                "state": "unavailable",
+                "message": "Contacts sync is unavailable. Try again later.",
+            }
+    return {"state": "ready", "message": ""}
 
 
 def _abs_url(href: str) -> str:
@@ -306,7 +377,7 @@ def _fetch_via_report(cfg, auth):
             "REPORT", cfg["url"],
             content=_ADDRESSBOOK_QUERY.encode("utf-8"),
             headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
+            auth=auth, timeout=_CARDDAV_TIMEOUT,
         )
         if r.status_code not in (207, 200):
             return None
@@ -337,20 +408,51 @@ def _fetch_via_report(cfg, auth):
         return None
 
 
-def _fetch_contacts(force=False):
+def _fetch_contacts(force=False, owner: Optional[str] = None):
     """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
+    owner_key = str(owner or "").strip()
+    by_owner = _contact_cache.setdefault("by_owner", {})
+    if owner_key and not force and owner_key in by_owner:
+        cached = by_owner.get(owner_key) or {}
+        fetched_at = cached.get("fetched_at")
+        if fetched_at:
+            age = (datetime.utcnow() - fetched_at).total_seconds()
+            if age < _CONTACT_CACHE_TTL_SECONDS:
+                return cached.get("contacts") or []
+
+    if not owner_key and not force and _contact_cache["fetched_at"]:
         age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
+        if age < _CONTACT_CACHE_TTL_SECONDS:
             return _contact_cache["contacts"]
 
+    failed_at = _contact_cache.get("failed_at")
+    if not force and failed_at:
+        failure_age = (datetime.utcnow() - failed_at).total_seconds()
+        if failure_age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return _cached_contacts(owner_key)
+
+    # SFT users must not see the operator's personal/CardDAV contact book.
+    # Their training contacts are seeded as owner-scoped local rows.
+    if owner_key.startswith("sft_"):
+        contacts = _load_local_contacts(owner_key)
+        by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        return contacts
+
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        contacts = _load_local_contacts(owner_key or None)
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        else:
+            _contact_cache["contacts"] = contacts
+            _contact_cache["fetched_at"] = datetime.utcnow()
         return contacts
 
+    # Do not let a burst of typeahead requests start parallel CardDAV timeouts.
+    # A caller that arrives during a refresh gets the most recent cache instead.
+    if not _contact_fetch_lock.acquire(blocking=False):
+        return _cached_contacts(owner_key)
+
     try:
         cfg["url"] = _carddav_base_url(cfg)
         auth = None
@@ -360,17 +462,23 @@ def _fetch_contacts(force=False):
         contacts = _fetch_via_report(cfg, auth)
         if contacts is None:
             # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            r = httpx.get(cfg["url"], auth=auth, timeout=_CARDDAV_TIMEOUT)
             if r.status_code != 200:
                 logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
+                return _mark_contact_fetch_failure(owner_key)
             contacts = _parse_vcards(r.text)
+        fetched_at = datetime.utcnow()
         _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        _contact_cache["fetched_at"] = fetched_at
+        _contact_cache["failed_at"] = None
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": fetched_at}
         return contacts
     except Exception as e:
         logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
+        return _mark_contact_fetch_failure(owner_key)
+    finally:
+        _contact_fetch_lock.release()
 
 
 def _resolve_resource_url(uid: str) -> str:
@@ -394,25 +502,31 @@ def _resolve_resource_url(uid: str) -> str:
     return _lookup() or _vcard_url(uid)
 
 
-def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
+def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None, owner: Optional[str] = None) -> bool:
     """Add a new contact via CardDAV or local contacts."""
     email = (email or "").strip()
     phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         email_l = email.lower()
         for c in contacts:
+            if owner_key and not _contact_visible_to_owner(c, owner_key):
+                continue
             if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
                 return True
             if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
                 return True
-        contacts.append(_normalize_contact({
+        row = {
             "name": name,
             "emails": [email] if email else [],
             "phones": phone_list,
             "address": address,
-        }))
+        }
+        if owner_key:
+            row["owner"] = owner_key
+        contacts.append(_normalize_contact(row))
         _save_local_contacts(contacts)
         return True
 
@@ -650,24 +764,34 @@ def _contacts_to_csv(contacts: List[Dict]) -> str:
     return out.getvalue()
 
 
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "", owner: Optional[str] = None) -> bool:
     """Rewrite an existing contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         found = False
         out = []
         for c in contacts:
             if c.get("uid") == uid:
+                if owner_key and not _contact_visible_to_owner(c, owner_key):
+                    out.append(c)
+                    continue
                 # Preserve existing address when caller passes "" (only
                 # updating name/emails/phones, not touching address).
                 addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}
+                if owner_key:
+                    row["owner"] = owner_key
+                out.append(_normalize_contact(row))
                 found = True
             else:
                 out.append(c)
         if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+            row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}
+            if owner_key:
+                row["owner"] = owner_key
+            out.append(_normalize_contact(row))
         _save_local_contacts(out)
         return True
 
@@ -694,12 +818,16 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], a
         return False
 
 
-def _delete_contact(uid: str) -> bool:
+def _delete_contact(uid: str, owner: Optional[str] = None) -> bool:
     """Delete a contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
+        remaining = [
+            c for c in contacts
+            if c.get("uid") != uid or (owner_key and not _contact_visible_to_owner(c, owner_key))
+        ]
         _save_local_contacts(remaining)
         return True
 
@@ -739,17 +867,17 @@ def setup_contacts_routes():
     router = APIRouter(prefix="/api/contacts", tags=["contacts"])
 
     @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
+    async def list_contacts(request: Request, _admin: str = Depends(require_admin)):
         """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
+        return {"contacts": contacts, "count": len(contacts), "sync": _contact_sync_status()}
 
     @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+    async def search_contacts(request: Request, q: str = Query(""), _admin: str = Depends(require_admin)):
         """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
         if not q:
-            return {"results": []}
+            return {"results": [], "sync": _contact_sync_status()}
         q_lower = q.lower()
         results = []
         for c in contacts:
@@ -760,11 +888,12 @@ def setup_contacts_routes():
                 if q_lower in em.lower():
                     results.append(c)
                     break
-        return {"results": results[:10]}
+        return {"results": results[:10], "sync": _contact_sync_status()}
 
     @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+    async def add_contact(data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Add a new contact."""
+        owner = effective_user(request)
         name = (data.get("name") or "").strip()
         email = (data.get("email") or "").strip()
         phone = (data.get("phone") or "").strip()
@@ -778,17 +907,20 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, phone, or address required"}
         if not name:
             name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-        contacts = _fetch_contacts()
+        contacts = _fetch_contacts(owner=owner)
         for c in contacts:
             if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
                 return {"success": True, "message": "Already exists", "contact": c}
             if phones and any(p in (c.get("phones") or []) for p in phones):
                 return {"success": True, "message": "Already exists", "contact": c}
         create_params = inspect.signature(_create_contact).parameters
-        if "phones" in create_params:
-            ok = _create_contact(name, email, address, phones=phones)
-        elif len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
+        if len(create_params) >= 3:
+            create_kwargs = {}
+            if "phones" in create_params:
+                create_kwargs["phones"] = phones
+            if "owner" in create_params:
+                create_kwargs["owner"] = owner
+            ok = _create_contact(name, email, address, **create_kwargs)
         else:
             ok = _create_contact(name, email)
         # If a phone was provided, do an immediate update to thread it
@@ -796,7 +928,7 @@ def setup_contacts_routes():
         # email + address; phones happen via update).
         if ok and phones and "phones" not in create_params:
             try:
-                fresh = _fetch_contacts(force=True)
+                fresh = _fetch_contacts(force=True, owner=owner)
                 created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
                 if created:
                     _update_contact(
@@ -804,6 +936,7 @@ def setup_contacts_routes():
                         created.get("emails", []),
                         phones,
                         address,
+                        owner=owner,
                     )
             except Exception:
                 pass
@@ -830,11 +963,16 @@ def setup_contacts_routes():
 
     @router.get("/export")
     async def export_contacts(
+        request: Request,
         format: str = Query("vcf", pattern="^(vcf|csv)$"),
         _admin: str = Depends(require_admin),
     ):
         """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
+        contacts = await asyncio.to_thread(
+            _fetch_contacts,
+            force=True,
+            owner=effective_user(request),
+        )
         if format == "csv":
             content = _contacts_to_csv(contacts)
             media_type = "text/csv; charset=utf-8"
@@ -876,19 +1014,28 @@ def setup_contacts_routes():
         _save_settings(settings)
         # Force re-fetch
         _contact_cache["fetched_at"] = None
+        _contact_cache["failed_at"] = None
         return {"success": True}
 
     @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
+    async def clear_contacts(request: Request, _admin: str = Depends(require_admin)):
         """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
+        owner = effective_user(request)
+        if owner:
+            remaining = [
+                c for c in _load_local_contacts()
+                if not _contact_visible_to_owner(c, owner)
+            ]
+            _save_local_contacts(remaining)
+        else:
+            _save_local_contacts([])
         return {"success": True}
 
     # NOTE: the /{uid} routes are declared LAST so the literal paths above
     # (/list, /search, /add, /config) win — otherwise PUT /config would
     # match PUT /{uid} with uid="config".
     @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+    async def edit_contact(uid: str, data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Edit an existing contact — name / emails / phones / address."""
         name = (data.get("name") or "").strip()
         emails = data.get("emails")
@@ -902,15 +1049,15 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, or address required"}
         if not name and emails:
             name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
+        ok = _update_contact(uid, name, emails, phones, address, owner=effective_user(request))
         return {"success": ok}
 
     @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+    async def delete_contact(uid: str, request: Request, _admin: str = Depends(require_admin)):
         """Delete a contact by UID."""
         if not uid:
             return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
+        ok = _delete_contact(uid, owner=effective_user(request))
         return {"success": ok}
 
     return router

routes/cookbook_helpers.py

@@ -1085,6 +1085,10 @@ class ServeRequest(BaseModel):
     hf_token: str | None = None
     gpus: str | None = None
     platform: str | None = None    # "linux", "termux", or "windows"
+    # Optional explicit image runtime adapter. "auto" preserves compatibility
+    # with older callers; catalog-backed launches can set this without relying
+    # on model-name heuristics in the generated runner.
+    runtime_adapter: str | None = None
 
 
 def _parse_serve_phase(snapshot: str, task_type: str = "serve") -> dict:
@@ -1204,6 +1208,41 @@ def _safe_env_prefix(ep: str | None) -> str | None:
     return f'[ -f "{path}" ] && source "{path}" || true'
 
 
+def _local_windows_bash_env_prefix(ep: str | None) -> str | None:
+    """Convert a frontend PowerShell venv prefix for the local Git Bash runner."""
+    if not ep:
+        return ep
+
+    prefix = ep.strip()
+    if not prefix.startswith("&"):
+        return ep
+
+    raw_path = prefix[1:].lstrip()
+    if not raw_path:
+        return ep
+    if raw_path.startswith("'"):
+        if len(raw_path) < 2 or not raw_path.endswith("'"):
+            return ep
+        quoted_path = raw_path[1:-1]
+        if "'" in quoted_path.replace("''", ""):
+            return ep
+        path = quoted_path.replace("''", "'")
+    else:
+        path = raw_path.rstrip()
+        if "'" in path or '"' in path:
+            return ep
+    if any(c in path for c in "\r\n;&|`$<>"):
+        return ep
+    if not path.replace("\\", "/").casefold().endswith("/scripts/activate.ps1"):
+        return ep
+
+    bash_path = _git_bash_path(path)
+    if "\\" in bash_path:
+        return ep
+    bash_path = bash_path[: -len("Activate.ps1")] + "activate"
+    return "source " + shlex.quote(bash_path)
+
+
 def _ssh_ps(host, script_path, port=None):
     """Build SSH command to run a PowerShell script on a Windows remote."""
     pf = f"-p {port} " if port and port != "22" else ""

routes/cookbook_routes.py

@@ -50,7 +50,7 @@ from routes.cookbook_helpers import (
     _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
     _validate_local_dir, _validate_gpus, _shell_path,
     _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
-    _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
+    _safe_env_prefix, _local_windows_bash_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
     load_stored_hf_token,
     _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
@@ -114,6 +114,16 @@ def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     runner_lines.append('chmod +x scripts/mlx_image_server.py 2>/dev/null || true')
 
 
+def _normalize_runtime_adapter(value: str | None) -> str:
+    """Return a shell-safe explicit image adapter name."""
+    value = (value or "auto").strip().lower()
+    if not value:
+        return "auto"
+    if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,39}", value):
+        raise HTTPException(400, "Invalid runtime adapter")
+    return value
+
+
 def _venv_root_from_serve_cmd(cmd: str) -> str:
     """Best-effort venv root from an absolute venv python in a serve command."""
     try:
@@ -1336,7 +1346,7 @@ def setup_cookbook_routes() -> APIRouter:
             # Local: run hf download in the background (tmux on POSIX, a detached
             # process + logfile on Windows where tmux doesn't exist).
             if req.env_prefix:
-                lines.append(_safe_env_prefix(req.env_prefix))
+                lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 lines.append("deactivate 2>/dev/null; hash -r")
             # Show whether the HF token reached this run (masked) — tells a gated
@@ -1411,7 +1421,6 @@ def setup_cookbook_routes() -> APIRouter:
         # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection.
         host = validate_remote_host(host)
         ssh_port = validate_ssh_port(ssh_port)
-        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
 
         model_dirs = []
         if model_dir:
@@ -1423,20 +1432,17 @@ def setup_cookbook_routes() -> APIRouter:
                     model_dirs.append(d)
         paths_code = _cached_model_scan_script(model_dirs)
 
-        scan_py = TMUX_LOG_DIR / "scan_cache.py"
-        scan_py.write_text(paths_code, encoding="utf-8")
-
         async def _run_cached_scan_once():
+            # Each request owns its script bytes. A shared scan_cache.py races
+            # when the tool scans several hosts/directories concurrently.
             if host:
-                _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
-                _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
-                if platform == "windows":
-                    # Windows: use 'python' and pipe via stdin with double-quote wrapping
-                    cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
-                else:
-                    cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
-                proc = await asyncio.create_subprocess_shell(
-                    cmd,
+                ssh_args = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
+                            '-o', 'ServerAliveInterval=4', '-o', 'ServerAliveCountMax=1']
+                if ssh_port and ssh_port != '22':
+                    ssh_args.extend(['-p', ssh_port])
+                proc = await asyncio.create_subprocess_exec(
+                    *ssh_args, host, 'python -' if platform == 'windows' else 'python3 -',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
@@ -1454,12 +1460,31 @@ def setup_cookbook_routes() -> APIRouter:
                     or which_tool("py") or "python"
                 )
                 proc = await asyncio.create_subprocess_exec(
-                    local_py, str(scan_py),
+                    local_py, '-',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
                 )
-            return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode
+            try:
+                output = await asyncio.wait_for(proc.communicate(paths_code.encode('utf-8')), timeout=60)
+                return output, proc.returncode
+            finally:
+                # A timed-out/cancelled request must not abandon its scanner.
+                # This handle belongs only to this request, never a model job.
+                if proc.returncode is None:
+                    try:
+                        proc.terminate()
+                    except ProcessLookupError:
+                        pass
+                    try:
+                        await asyncio.wait_for(proc.wait(), timeout=2)
+                    except asyncio.TimeoutError:
+                        try:
+                            proc.kill()
+                        except ProcessLookupError:
+                            pass
+                        await asyncio.wait_for(proc.wait(), timeout=2)
 
         (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
         stderr_txt = stderr_b.decode(errors="replace").strip()
@@ -1974,6 +1999,7 @@ def setup_cookbook_routes() -> APIRouter:
         validate_remote_host(req.remote_host)
         req.ssh_port = validate_ssh_port(req.ssh_port)
         req.gpus = _validate_gpus(req.gpus)
+        req.runtime_adapter = _normalize_runtime_adapter(req.runtime_adapter)
         req.hf_token = req.hf_token or _load_stored_hf_token()
         _validate_token(req.hf_token)
         # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
@@ -2166,7 +2192,7 @@ def setup_cookbook_routes() -> APIRouter:
             if req.gpus:
                 runner_lines.append(f"export CUDA_VISIBLE_DEVICES='{req.gpus}'")
             if req.env_prefix:
-                runner_lines.append(_safe_env_prefix(req.env_prefix))
+                runner_lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 runner_lines.append("deactivate 2>/dev/null; hash -r")
             _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)
@@ -2602,19 +2628,20 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('print(model)')
                 runner_lines.append('PY')
                 runner_lines.append(')"')
-                runner_lines.append('if printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; then')
+                runner_lines.append(f"export ODYSSEUS_MLX_IMAGE_ADAPTER='{_bash_squote(req.runtime_adapter or 'auto')}'")
+                runner_lines.append('if [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "hidream" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import mlx, mlx_vlm, transformers, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: HiDream MLX serving needs the model requirements in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart mlx mlx-vlm \'transformers>=4.57.0,<6.0\' huggingface_hub safetensors numpy pillow tqdm sentencepiece hf_transfer"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "boogu" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import boogu_image_mlx, mlx, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: Boogu MLX serving needs boogu-image-mlx in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U git+https://github.com/xocialize/boogu-image-mlx.git fastapi uvicorn python-multipart pillow"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "ddcolor" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: DDColor MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2634,7 +2661,7 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "inpaint" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2654,10 +2681,12 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
-                runner_lines.append('  echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
-                runner_lines.append('  echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "mflux" ] || [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ]; then')
+                runner_lines.append('  if ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
                 runner_lines.append('fi')
             elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
@@ -3516,12 +3545,19 @@ def setup_cookbook_routes() -> APIRouter:
             return {"ok": False, "error": str(e)}
 
     @router.get("/api/cookbook/hf-latest")
-    async def hf_latest(vram_gb: float = 0, limit: int = 10, pipeline: str = "text-generation", owner: str = Depends(require_user)):
+    async def hf_latest(
+        vram_gb: float = 0,
+        limit: int = 10,
+        pipeline: str = "text-generation",
+        official_only: bool = False,
+        owner: str = Depends(require_user),
+    ):
         """Fetch latest HuggingFace models, filtered by what fits in available VRAM.
 
         vram_gb: total available VRAM in GB. 0 = no filter (return everything).
         limit:   how many models to return (default 10).
         pipeline: HF pipeline_tag filter (text-generation, text-to-image, etc.).
+        official_only: restrict results to recognized first-party provider namespaces.
         """
         import re
         import httpx
@@ -3587,6 +3623,20 @@ def setup_cookbook_routes() -> APIRouter:
                     return True
             return False
 
+        # HF does not expose a universal "first-party" flag. Keep this as a
+        # namespace policy rather than a model-name list, so newly published
+        # provider models are included without recommending community forks.
+        OFFICIAL_NAMESPACES = {
+            "apple", "black-forest-labs", "deepseek-ai", "google", "lightricks",
+            "meta-llama", "microsoft", "mistralai", "nvidia", "openai", "qwen",
+            "stabilityai", "tencent", "runwayml",
+        }
+
+        def _is_official(entry: dict, repo_id: str) -> bool:
+            namespace = repo_id.split("/", 1)[0].strip().lower() if "/" in repo_id else ""
+            author = str(entry.get("author") or "").strip().lower()
+            return namespace in OFFICIAL_NAMESPACES and (not author or author == namespace)
+
         out = []
         for entry in raw:
             repo_id = entry.get("modelId") or entry.get("id") or ""
@@ -3601,6 +3651,8 @@ def setup_cookbook_routes() -> APIRouter:
             # Skip adapters, LoRAs, datasets, etc.
             if _is_excluded(repo_id, tags):
                 continue
+            if official_only and not _is_official(entry, repo_id):
+                continue
 
             est_fp16 = _est_vram_fp16(repo_id)
             quant_mult = _quant_factor(repo_id, tags)
@@ -3614,7 +3666,11 @@ def setup_cookbook_routes() -> APIRouter:
                     # if we cannot estimate size from the repo id/tags, do not
                     # present it as runnable on this hardware.
                     continue
-                if needed_vram > vram_gb:
+                # Leave allocator/runtime headroom instead of treating the
+                # reported total as a safe load budget. This keeps the
+                # official-only list honest on tight GPUs as well.
+                usable_vram = vram_gb * 0.90
+                if needed_vram > usable_vram:
                     continue
 
             out.append({
@@ -4412,6 +4468,7 @@ def setup_cookbook_routes() -> APIRouter:
 
             progress_text = ""
             full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""
+            _persisted_terminal = False
 
             if local_win_task:
                 # File-based liveness + output for the detached-process model.
@@ -4445,9 +4502,10 @@ def setup_cookbook_routes() -> APIRouter:
                     and bool(full_snapshot)
                     and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
                 )
-                if _task_status in {"stopped", "done", "completed",
+                _persisted_terminal = _task_status in {"stopped", "done", "completed",
                                     "crashed", "error", "failed",
-                                    "ended", "killed"} and not _persisted_serve_ready:
+                                    "ended", "killed"} and not _persisted_serve_ready
+                if _persisted_terminal:
                     is_alive = False
                     # Keep the persisted output_tail for the UI — it's
                     # what the agent uses to diagnose past failures.
@@ -4486,7 +4544,9 @@ def setup_cookbook_routes() -> APIRouter:
                 and (
                     ".incomplete" in full_snapshot
                     or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
-                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
+                    or (not _persisted_terminal and _download_cache_incomplete(
+                        _payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or ""
+                    ))
                 )
             )
             if is_alive or (local_win_task and full_snapshot):
@@ -4538,6 +4598,7 @@ def setup_cookbook_routes() -> APIRouter:
                         progress_text = "Download complete"
                 elif (
                     task_type == "download"
+                    and not _persisted_terminal
                     and not download_has_incomplete_evidence
                     and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 ):

routes/document/document_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 from typing import Dict, Any, List, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form
+from fastapi.responses import HTMLResponse
 
 from sqlalchemy import case, func, or_
 from core.database import SessionLocal, Document, DocumentVersion
@@ -479,6 +480,32 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    # ---- GET /api/document/{doc_id}/visual-report ----
+    @router.get("/api/document/{doc_id}/visual-report", response_class=HTMLResponse)
+    async def document_visual_report(request: Request, doc_id: str) -> HTMLResponse:
+        """Render a Markdown document with the same standalone report UI used by Deep Research."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            doc = db.query(Document).filter(Document.id == doc_id).first()
+            if not doc:
+                raise HTTPException(404, "Document not found")
+            _verify_doc_owner(db, doc, user)
+            if (doc.language or "").lower() != "markdown":
+                raise HTTPException(400, "Visual reports are available for Markdown documents")
+
+            from src.visual_report import generate_visual_report
+
+            html_content = generate_visual_report(
+                question=doc.title or "Document",
+                report_markdown=doc.current_content or "",
+                sources=[],
+                stats={},
+            )
+            return HTMLResponse(content=html_content)
+        finally:
+            db.close()
+
     # ---- POST /api/document/{doc_id}/archive — soft-archive / restore ----
     @router.post("/api/document/{doc_id}/archive")
     async def archive_document(request: Request, doc_id: str, archived: bool = Query(True)) -> Dict[str, Any]:
@@ -575,7 +602,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh",
             "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c",
             "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php",
-            "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini",
+            "text": ".txt", "email": ".eml", "xml": ".xml", "toml": ".toml", "ini": ".ini",
         }
         db = SessionLocal()
         try:
@@ -602,7 +629,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                         name = f"{base}-{i}" + ("" if "." in base else ext)
                         i += 1
                     used.add(name)
-                    zf.writestr(name, doc.current_content or "")
+                    content = doc.current_content or ""
+                    if (doc.language or "").lower() == "email":
+                        content = re.sub(r"\r?\n---\r?\n", "\r\n\r\n", content, count=1)
+                    zf.writestr(name, content)
                     wrote += 1
             if not wrote:
                 raise HTTPException(404, "No documents found")

routes/editor_draft_routes.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel
 
 from core.database import EditorDraft, SessionLocal
 from src.auth_helpers import get_current_user
+from src.upload_limits import EDITOR_DRAFT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +76,16 @@ def _load_payload(raw: Optional[str]) -> Dict[str, Any]:
     return payload if isinstance(payload, dict) else {}
 
 
+def _dump_payload(payload: Dict[str, Any]) -> str:
+    raw = json.dumps(payload or {}, separators=(",", ":"))
+    if len(raw.encode("utf-8")) > EDITOR_DRAFT_MAX_BYTES:
+        raise HTTPException(
+            413,
+            f"Editor draft exceeds the {EDITOR_DRAFT_MAX_BYTES // (1024 * 1024)} MB safety limit",
+        )
+    return raw
+
+
 def setup_editor_draft_routes() -> APIRouter:
     router = APIRouter(tags=["editor-drafts"])
 
@@ -120,13 +131,15 @@ def setup_editor_draft_routes() -> APIRouter:
                 source_image_id=body.source_image_id,
                 width=body.width,
                 height=body.height,
-                payload=json.dumps(body.payload or {}),
+                payload=_dump_payload(body.payload),
                 thumbnail=body.thumbnail,
             )
             db.add(d)
             db.commit()
             db.refresh(d)
             return _summary(d)
+        except HTTPException:
+            raise
         except Exception as e:
             db.rollback()
             logger.warning(f"editor-draft create failed: {e}")
@@ -151,7 +164,7 @@ def setup_editor_draft_routes() -> APIRouter:
             if body.height is not None:
                 d.height = body.height
             if body.payload is not None:
-                d.payload = json.dumps(body.payload)
+                d.payload = _dump_payload(body.payload)
             if body.thumbnail is not None:
                 d.thumbnail = body.thumbnail
             db.commit()

routes/email_helpers.py

@@ -886,10 +886,16 @@ def _init_scheduled_db():
             size INTEGER DEFAULT 0,
             flags TEXT DEFAULT '',
             has_attachments INTEGER DEFAULT 0,
+            attachment_names TEXT DEFAULT '',
             updated_at TEXT NOT NULL,
             PRIMARY KEY (owner, account_key, folder, uid)
         )
     """)
+    _message_index_cols = {
+        row[1] for row in conn.execute("PRAGMA table_info(email_message_index)").fetchall()
+    }
+    if "attachment_names" not in _message_index_cols:
+        conn.execute("ALTER TABLE email_message_index ADD COLUMN attachment_names TEXT DEFAULT ''")
     conn.execute("""
         CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
         ON email_message_index(owner, account_key, folder, date_epoch DESC)
@@ -1667,7 +1673,15 @@ def _extract_text(msg):
         payload = msg.get_payload(decode=True)
         if payload:
             charset = msg.get_content_charset() or "utf-8"
-            return payload.decode(charset, errors="replace")
+            text = payload.decode(charset, errors="replace")
+            if msg.get_content_type() == "text/html":
+                text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
+                text = re.sub(r"</(?:p|div|li|tr|h[1-6])\s*>", "\n", text, flags=re.I)
+                text = re.sub(r"<[^>]+>", "", text)
+                text = html.unescape(text)
+                text = re.sub(r"[ \t]+\n", "\n", text)
+                text = re.sub(r"\n{3,}", "\n\n", text)
+            return text.strip()
     return ""
 
 
@@ -1998,6 +2012,9 @@ class SendEmailRequest(BaseModel):
     # answered after successful delivery so it leaves undone/reply-soon views.
     source_uid: Optional[str] = None
     source_folder: Optional[str] = None
+    # Exact IMAP draft to remove after successful delivery.
+    draft_uid: Optional[str] = None
+    draft_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -228,6 +228,9 @@ def _ensure_away_reply_table():
 
 
 def _sender_is_automated(msg, sender_addr: str) -> bool:
+    subject = str(msg.get("Subject") or "").lower()
+    if re.search(r"automatic\s+reply|auto(?:matic)?[- ]?reply|out\s+of\s+office|\booo\b|r[ée]ponse\s+automatique", subject):
+        return True
     auto_submitted = (msg.get("Auto-Submitted") or "").strip().lower()
     if auto_submitted and auto_submitted != "no":
         return True
@@ -244,6 +247,31 @@ def _sender_is_automated(msg, sender_addr: str) -> bool:
     }
 
 
+def _remove_urgent_tag_from_cache(message_id: str, owner: str, account_id: str) -> None:
+    """Remove stale urgent tags from messages identified as automated."""
+    import sqlite3 as _sql3
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        owner_clause, owner_params = _email_cache_owner_clause(owner)
+        rows = conn.execute(
+            f"SELECT rowid, tags FROM email_tags WHERE message_id=? AND {owner_clause} "
+            "AND (account_id=? OR account_id='' OR account_id IS NULL)",
+            (message_id, *owner_params, account_id or ""),
+        ).fetchall()
+        for rowid, raw_tags in rows:
+            try:
+                tags = json.loads(raw_tags or "[]")
+            except Exception:
+                tags = []
+            if not isinstance(tags, list) or "urgent" not in tags:
+                continue
+            cleaned = [tag for tag in tags if str(tag).strip().lower() != "urgent"]
+            conn.execute("UPDATE email_tags SET tags=? WHERE rowid=?", (json.dumps(cleaned), rowid))
+        conn.commit()
+    finally:
+        conn.close()
+
+
 def _away_reply_already_sent(settings: dict, account_owner: str, account_id: str | None,
                              message_id: str, sender_addr: str) -> bool:
     import sqlite3 as _sql3
@@ -712,6 +740,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     _, _from_addr_only = email.utils.parseaddr(_from_raw)
                 except Exception:
                     _from_addr_only = ""
+                _is_automated = _sender_is_automated(msg, _from_addr_only)
+                if _is_automated and auto_tag:
+                    _remove_urgent_tag_from_cache(message_id, account_owner or "", account_id or "")
                 _is_self_mail = bool(_self_self_addr) and _from_addr_only.lower() == _self_self_addr
                 need_sum = auto_sum and message_id not in _sum_existing
                 need_reply = auto_reply_draft and message_id not in _reply_existing
@@ -1286,6 +1317,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
                             tags = ["marketing" if t == "promo" else t for t in tags]
                             tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                            if _is_automated:
+                                tags = [t for t in tags if t != "urgent"]
                             is_spam = bool(parsed.get("spam"))
                             spam_reason = str(parsed.get("reason") or "")[:200]
 

routes/history/history_routes.py

@@ -1,6 +1,7 @@
 """History routes — session history, truncation, fork, conversation topics."""
 
 import json
+import os
 import uuid
 import logging
 import re
@@ -19,6 +20,7 @@ from routes.session_routes import (
     _reject_compact_during_active_run,
     _verify_session_owner,
 )
+from routes.chat_helpers import strip_tui_local_context
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +28,105 @@ _HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
 _DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
 
 
+def _sft_trace_file_for_owner(owner: str | None) -> str | None:
+    if not str(owner or "").startswith("sft_"):
+        return None
+    flag = os.getenv("ODYSSEUS_SFT_TRACE_CAPTURE", "1").strip().lower()
+    if flag in {"0", "false", "no", "off"}:
+        return None
+    try:
+        from src.constants import DATA_DIR
+        trace_dir = os.getenv("ODYSSEUS_SFT_TRACE_DIR") or os.path.join(DATA_DIR, "sft_traces")
+        return os.path.join(trace_dir, f"{owner}.jsonl")
+    except Exception:
+        return None
+
+
+def _remove_deleted_sft_trace_rows(
+    *,
+    owner: str | None,
+    session_id: str,
+    deleted_pairs: list[dict[str, str]],
+) -> None:
+    """Keep the training JSONL aligned with user-deleted chat attempts."""
+    path = _sft_trace_file_for_owner(owner)
+    if not path or not deleted_pairs or not os.path.exists(path):
+        return
+    try:
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if row.get("session_id") != session_id:
+                    kept.append(raw)
+                    continue
+                row_user = str(row.get("user") or "").strip()
+                row_assistant = str(row.get("assistant") or "").strip()
+                should_remove = any(
+                    row_user == pair.get("user", "").strip()
+                    and row_assistant == pair.get("assistant", "").strip()
+                    for pair in deleted_pairs
+                )
+                if should_remove:
+                    tombstone = dict(row)
+                    tombstone["deleted_from_training"] = True
+                    removed.append(json.dumps(tombstone, ensure_ascii=False))
+                else:
+                    kept.append(raw)
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        if removed:
+            trash_path = path + ".trash"
+            with open(trash_path, "a", encoding="utf-8") as f:
+                for raw in removed:
+                    f.write(raw + "\n")
+            logger.info(
+                "Removed %d SFT trace row(s) for deleted messages in session %s",
+                len(removed),
+                session_id,
+            )
+    except Exception as exc:
+        logger.warning("Failed to prune SFT trace rows for %s: %s", session_id, exc)
+
+
+def _deleted_sft_pairs_from_db_rows(rows: list[DbChatMessage]) -> list[dict[str, str]]:
+    """Build user/assistant pairs affected by deleted messages.
+
+    The SFT trace row is one assistant turn paired with the nearest preceding
+    user turn. If the user deletes either side of a failed attempt before
+    retrying, remove that pair from the training JSONL.
+    """
+    pairs: list[dict[str, str]] = []
+    last_user = ""
+    pending_deleted_user = ""
+    for row in rows:
+        role = str(getattr(row, "role", "") or "")
+        content = str(getattr(row, "content", "") or "").strip()
+        will_delete = bool(getattr(row, "_will_delete_for_sft", False))
+        if role == "user":
+            last_user = content
+            if will_delete:
+                pending_deleted_user = content
+            continue
+        if role != "assistant":
+            continue
+        if will_delete and last_user:
+            pairs.append({"user": last_user, "assistant": content})
+        elif pending_deleted_user:
+            pairs.append({"user": pending_deleted_user, "assistant": content})
+            pending_deleted_user = ""
+    return pairs
+
+
 def _history_display_content(content: Any) -> Any:
     """Return a lightweight browser-display copy of stored message content.
 
@@ -100,6 +201,41 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
     return to_delete
 
 
+def _is_continue_interruption_message(message: Any) -> bool:
+    if isinstance(message, ChatMessage):
+        role = message.role
+        content = message.content
+    elif isinstance(message, dict):
+        role = message.get("role", "")
+        content = message.get("content", "")
+    else:
+        role = getattr(message, "role", "")
+        content = getattr(message, "content", "")
+    normalized = " ".join(str(content or "").strip().lower().split())
+    return role == "user" and (
+        "previous response was interrupted" in normalized
+        or normalized in {
+            "continue from where you left off.",
+            "continue from where you left off",
+        }
+    )
+
+
+def _has_immediate_continue_marker(messages: list[Any], idx1: int, idx2: int) -> bool:
+    return idx2 - idx1 == 2 and _is_continue_interruption_message(messages[idx1 + 1])
+
+
+def _keep_count_before_message(db_messages, before_msg_id: str | None) -> int | None:
+    """Return the durable-history keep count before a DB message id."""
+    wanted = str(before_msg_id or "").strip()
+    if not wanted:
+        return None
+    for pos, row in enumerate(db_messages):
+        if str(getattr(row, "id", "")) == wanted:
+            return pos
+    return None
+
+
 def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
     router = APIRouter(tags=["history"])
 
@@ -124,13 +260,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             )
 
     def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
-        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        entry = {"role": m.role, "content": strip_tui_local_context(_history_display_content(m.content))}
         meta = {}
         if m.meta_data:
             try:
                 meta = json.loads(m.meta_data) or {}
             except (json.JSONDecodeError, ValueError):
                 meta = {}
+        meta["_db_id"] = m.id
         if m.timestamp and "timestamp" not in meta:
             meta["timestamp"] = m.timestamp.isoformat() + "Z"
         if meta:
@@ -199,7 +336,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Skip hidden messages (e.g. compaction summaries for AI context)
                 if msg.metadata and msg.metadata.get("hidden"):
                     continue
-                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
+                entry = {"role": msg.role, "content": strip_tui_local_context(_history_display_content(msg.content))}
                 if msg.metadata:
                     entry["metadata"] = msg.metadata
                 history_dict.append(entry)
@@ -208,7 +345,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     continue
                 entry = {
                     "role": msg.get("role", ""),
-                    "content": _history_display_content(msg.get("content", "")),
+                    "content": strip_tui_local_context(_history_display_content(msg.get("content", ""))),
                 }
                 if msg.get("metadata"):
                     entry["metadata"] = msg["metadata"]
@@ -249,11 +386,36 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
         _verify_session_owner(request, session_id)
         try:
             body = await request.json()
-            keep_count = body.get("keep_count", 0)
+            keep_count = int(body.get("keep_count", 0))
+            before_msg_id = str(body.get("before_msg_id") or body.get("message_id") or "").strip()
+            deleted_sft_pairs: list[dict[str, str]] = []
+            if keep_count >= 0:
+                db = SessionLocal()
+                try:
+                    all_db_messages = db.query(DbChatMessage).filter(
+                        DbChatMessage.session_id == session_id
+                    ).order_by(DbChatMessage.timestamp).all()
+                    if before_msg_id:
+                        resolved_keep_count = _keep_count_before_message(all_db_messages, before_msg_id)
+                        if resolved_keep_count is None:
+                            raise HTTPException(404, "Message not found")
+                        keep_count = resolved_keep_count
+                    for pos, row in enumerate(all_db_messages):
+                        row._will_delete_for_sft = pos >= keep_count
+                    deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+                finally:
+                    db.close()
             result = session_manager.truncate_messages(session_id, keep_count)
+            _remove_deleted_sft_trace_rows(
+                owner=effective_user(request),
+                session_id=session_id,
+                deleted_pairs=deleted_sft_pairs,
+            )
             return {"status": "ok", "kept": keep_count, "truncated": result}
         except KeyError:
             raise HTTPException(404, "Session not found")
+        except HTTPException:
+            raise
         except Exception as e:
             logger.error(f"Truncate error {session_id}: {e}")
             raise HTTPException(500, str(e))
@@ -288,6 +450,18 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             session = session_manager.get_session(session_id)
             db = SessionLocal()
             try:
+                all_db_messages = db.query(DbChatMessage).filter(
+                    DbChatMessage.session_id == session_id
+                ).order_by(DbChatMessage.timestamp).all()
+                delete_id_set = set(msg_ids or [])
+                delete_index_set = set(indices or [])
+                for pos, row in enumerate(all_db_messages):
+                    row._will_delete_for_sft = (
+                        (bool(delete_id_set) and row.id in delete_id_set)
+                        or (not delete_id_set and bool(delete_index_set) and pos in delete_index_set)
+                    )
+                deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+
                 if msg_ids:
                     # New ID-based delete
                     deleted = 0
@@ -330,6 +504,11 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     db_session.updated_at = datetime.now(timezone.utc)
 
                 db.commit()
+                _remove_deleted_sft_trace_rows(
+                    owner=effective_user(request),
+                    session_id=session_id,
+                    deleted_pairs=deleted_sft_pairs,
+                )
                 return {"status": "ok", "deleted": deleted}
             finally:
                 db.close()
@@ -520,6 +699,9 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 return {"status": "ok", "merged": False}
 
             idx1, idx2 = ai_indices[-2], ai_indices[-1]
+            if not _has_immediate_continue_marker(session.history, idx1, idx2):
+                return {"status": "ok", "merged": False, "reason": "no_continue_marker"}
+
             msg1, msg2 = session.history[idx1], session.history[idx2]
 
             content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
@@ -530,7 +712,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
             meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
             merged_meta = {**meta1, **meta2}
+            thinking1 = str(meta1.get('thinking') or '').strip()
+            thinking2 = str(meta2.get('thinking') or '').strip()
+            if thinking1 and thinking2:
+                merged_meta['thinking'] = thinking1 + "\n\n(continued)\n\n" + thinking2
+            elif thinking1:
+                merged_meta['thinking'] = thinking1
             merged_meta.pop('stopped', None)  # no longer stopped after continue
+            merged_meta.pop('thinking_interrupted', None)
 
             # Update first message, remove second
             if isinstance(msg1, ChatMessage):
@@ -542,13 +731,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
 
             # Also remove the hidden "continue" user message between them if present
             # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
+            remove_indices = [idx2, idx1 + 1]
 
             for ri in sorted(remove_indices, reverse=True):
                 session.history.pop(ri)
@@ -566,19 +749,20 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Find last two assistant messages in DB
                 ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
                 if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
+                    (db_idx1, db1), (db_idx2, db2) = ai_db[-2], ai_db[-1]
+                    if _has_immediate_continue_marker(db_messages, db_idx1, db_idx2):
+                        db1.content = merged_content
+                        db1.meta_data = _json.dumps(merged_meta)
 
-                    # Mirror the in-memory deletion: remove the second assistant
-                    # message and ONLY the "continue" user message between them
-                    # (not arbitrary tool/system/user rows). The old
-                    # range-delete destroyed every row between the two assistant
-                    # messages, desyncing the DB from the in-memory history.
-                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
-                        db.delete(_row)
+                        # Mirror the in-memory deletion: remove the second assistant
+                        # message and ONLY the "continue" user message between them
+                        # (not arbitrary tool/system/user rows). The old
+                        # range-delete destroyed every row between the two assistant
+                        # messages, desyncing the DB from the in-memory history.
+                        for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
+                            db.delete(_row)
 
-                    db.commit()
+                        db.commit()
             finally:
                 db.close()
             session_manager.save_sessions()
@@ -672,6 +856,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             raise HTTPException(404, "Session not found")
 
         try:
+            from src.context_compactor import auto_compact_threshold_percent
             from src.model_context import estimate_tokens, get_context_length
 
             messages = session.get_context_messages()
@@ -679,6 +864,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             ctx_len = int(get_context_length(session.endpoint_url, session.model) or 0)
             pct = round((used / ctx_len) * 100, 1) if ctx_len else 0.0
             pct = max(0.0, min(100.0, pct))
+            auto_threshold = auto_compact_threshold_percent()
             visible_messages = sum(
                 1 for m in session.history
                 if not (getattr(m, "metadata", None) or {}).get("hidden")
@@ -699,13 +885,119 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 "context_messages": len(messages),
                 "compacted_messages": compacted_messages,
                 "can_compact": can_compact,
-                "should_compact": pct >= 70,
-                "auto_compact_threshold": 85,
+                "should_compact": pct >= auto_threshold,
+                "auto_compact_threshold": auto_threshold,
+                "memory_extraction_enabled": getattr(session, "memory_extraction_enabled", True) is not False,
+                "skill_injection_enabled": getattr(session, "skill_injection_enabled", True) is not False,
+                "thinking_mode": getattr(session, "thinking_mode", "") or "off",
+                "temperature_override": getattr(session, "temperature_override", None),
+                "max_tokens_override": getattr(session, "max_tokens_override", None),
             }
         except Exception as e:
             logger.error(f"Context usage error {session_id}: {e}")
             raise HTTPException(500, str(e))
 
+    @router.post("/api/session/{session_id}/memory-extraction")
+    async def set_session_memory_extraction(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle automatic memory extraction for one chat session."""
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                raise HTTPException(404, "Session not found")
+            db_session.memory_extraction_enabled = enabled
+            db.commit()
+            session.memory_extraction_enabled = enabled
+            return {"status": "success", "memory_extraction_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Memory extraction toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update memory extraction")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/skill-injection")
+    async def set_session_skill_injection(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle skill injection for one chat session."""
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                # Some active chats exist only in the in-memory manager until
+                # their first persisted write. Keep the toggle usable there.
+                session.skill_injection_enabled = enabled
+                session_manager.save_sessions()
+                return {"status": "success", "skill_injection_enabled": enabled}
+            db_session.skill_injection_enabled = enabled
+            db.commit()
+            session.skill_injection_enabled = enabled
+            return {"status": "success", "skill_injection_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Skill injection toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update skill injection")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/generation-settings")
+    async def set_session_generation_settings(request: Request, session_id: str) -> Dict[str, Any]:
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+            body = await request.json()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        mode = str(body.get("thinking_mode") or "").lower()
+        if mode not in {"", "on", "off"}:
+            raise HTTPException(400, "Invalid thinking mode")
+        temperature = body.get("temperature_override")
+        temperature = None if temperature in (None, "") else max(0.0, min(2.0, float(temperature)))
+        max_tokens = body.get("max_tokens_override")
+        max_tokens = None if max_tokens in (None, "", 0) else max(256, min(32768, int(max_tokens)))
+        db = SessionLocal()
+        try:
+            row = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not row:
+                raise HTTPException(404, "Session not found")
+            row.thinking_mode, row.temperature_override, row.max_tokens_override = mode, temperature, max_tokens
+            db.commit()
+            session.thinking_mode, session.temperature_override, session.max_tokens_override = mode, temperature, max_tokens
+            return {"status": "success", "thinking_mode": mode, "temperature_override": temperature, "max_tokens_override": max_tokens}
+        finally:
+            db.close()
+
     @router.post("/api/session/{session_id}/compact")
     async def compact_session(request: Request, session_id: str):
         """Manually trigger context compaction for a session."""

routes/hwfit_routes.py

@@ -16,6 +16,24 @@ from routes._validators import validate_remote_host, validate_ssh_port
 # "metal" routes through the Apple-Silicon path (GGUF-only, llama.cpp/Ollama),
 # the CPU backends through the RAM/offload path, cuda/rocm through vLLM.
 _MANUAL_BACKENDS = {"cuda", "rocm", "metal", "cpu_x86", "cpu_arm"}
+_OFFICIAL_NAMESPACES = {
+    "apple", "allenai", "black-forest-labs", "cohere", "deepseek-ai",
+    "google", "ibm", "lightricks", "meta-llama", "microsoft", "mistralai",
+    "nvidia", "openai", "qwen", "stabilityai", "tencent", "tiiuae",
+    "upstage", "zai-org", "runwayml",
+}
+
+
+def _is_official_model(model: dict) -> bool:
+    """Recognize first-party namespaces without maintaining model-name lists."""
+    # Image rows expose a friendly `name` without its namespace, while regular
+    # rows may use `name`. Prefer whichever field still contains `owner/repo`.
+    model_id = str(model.get("id") or model.get("name") or "")
+    namespace = model_id.split("/", 1)[0].strip().lower() if "/" in model_id else ""
+    # `provider` is a display label for image rows (for example, "Stability AI")
+    # and is not a stable repository namespace. The model id is the canonical
+    # source for this filter, so a recognized namespace is sufficient.
+    return namespace in _OFFICIAL_NAMESPACES
 
 
 def _validate_detection_target(host: str = "", ssh_port: str = "") -> tuple[str, str]:
@@ -191,7 +209,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False, official_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -310,6 +328,8 @@ def setup_hwfit_routes():
             rank_kwargs.pop("target_context", None)
             rank_kwargs.pop("fit_only", None)
         results = rank_models(system, **rank_kwargs)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         payload = {"system": system, "models": results}
         if catalog_refresh is not None:
             payload["catalog_refresh"] = catalog_refresh
@@ -410,7 +430,7 @@ def setup_hwfit_routes():
         }
 
     @router.get("/image-models")
-    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False):
+    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, official_only: bool = False):
         """Rank image generation models against detected hardware."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.image_models import rank_image_models
@@ -451,6 +471,8 @@ def setup_hwfit_routes():
             system["gpu_count"] = 1 if single_vram > 0 else 0
             system["gpu_only"] = True if single_vram > 0 else False
         results = rank_image_models(system, search=search or None, sort=sort)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         return {"system": system, "models": results}
 
     return router

routes/model_routes.py

@@ -17,7 +17,20 @@ from fastapi import APIRouter, HTTPException, Form, Query, Body, Request, Respon
 from pydantic import BaseModel
 from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
-from core.log_safety import redact_url as _redact_url_for_log
+try:
+    from core.log_safety import redact_url as _redact_url_for_log
+except ModuleNotFoundError:
+    def _redact_url_for_log(url: str) -> str:
+        try:
+            parsed = urlparse(url or "")
+            host = parsed.hostname or ""
+            if ":" in host:
+                host = f"[{host}]"
+            if parsed.port:
+                host = f"{host}:{parsed.port}"
+            return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
+        except Exception:
+            return "<endpoint>"
 from core.middleware import require_admin
 from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
@@ -455,6 +468,7 @@ def _truthy(value: str | None) -> bool:
 
 _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
 _REFRESH_MODES = {"auto", "manual", "disabled"}
+_MODEL_TOOL_MODES = {"none", "compact", "full"}
 
 
 def _normalize_endpoint_kind(value: Any) -> str:
@@ -462,6 +476,30 @@ def _normalize_endpoint_kind(value: Any) -> str:
     return kind if kind in _ENDPOINT_KINDS else "auto"
 
 
+def _normalize_model_tool_mode(value: Any) -> str:
+    mode = str(value or "").strip().lower()
+    return mode if mode in _MODEL_TOOL_MODES else ""
+
+
+def _model_tool_modes(ep: Any) -> Dict[str, str]:
+    raw = getattr(ep, "model_tool_modes", None)
+    if not raw:
+        return {}
+    try:
+        data = json.loads(raw) if isinstance(raw, str) else raw
+    except Exception:
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    modes: Dict[str, str] = {}
+    for key, value in data.items():
+        model_id = str(key or "").strip()
+        mode = _normalize_model_tool_mode(value)
+        if model_id and mode:
+            modes[model_id] = mode
+    return modes
+
+
 def _normalize_refresh_mode(value: Any, endpoint_kind: str = "auto") -> str:
     mode = str(value or "").strip().lower()
     kind = _normalize_endpoint_kind(endpoint_kind)
@@ -1973,6 +2011,7 @@ def setup_model_routes(model_discovery):
                     "ping_error": (ping or {}).get("error") if ping else None,
                     "model_type": getattr(r, "model_type", None) or "llm",
                     "supports_tools": getattr(r, "supports_tools", None),
+                    "model_tool_modes": _model_tool_modes(r),
                     "endpoint_kind": kind,
                     "category": _classify_endpoint(base, kind),
                     "model_refresh_mode": _endpoint_refresh_mode(r, kind),
@@ -2344,6 +2383,7 @@ def setup_model_routes(model_discovery):
                     response.headers["X-Model-Refresh-Warning"] = "Model refresh failed or returned no models; kept cached models."
             _, pinned = _picker_models_for_endpoint(ep, base, kind)
             pinned_set = set(pinned)
+            tool_modes = _model_tool_modes(ep)
             return [
                 {
                     "id": m,
@@ -2351,6 +2391,7 @@ def setup_model_routes(model_discovery):
                     "is_hidden": m in hidden,
                     "is_pinned": m in pinned_set,
                     "picker_requires_pinning": picker_requires_pinning,
+                    "tool_mode": tool_modes.get(m, ""),
                 }
                 for m in _merge_model_ids(all_models, pinned)
             ]
@@ -2401,11 +2442,31 @@ def setup_model_routes(model_discovery):
                     ep.hidden_models = None
                 else:
                     ep.pinned_models = json.dumps(pinned) if pinned else None
+            if "model_tool_modes" in body:
+                raw_modes = body.get("model_tool_modes")
+                if not isinstance(raw_modes, dict):
+                    raise HTTPException(400, "model_tool_modes must be an object")
+                modes = _model_tool_modes(ep)
+                for model_id, mode in raw_modes.items():
+                    model_id = str(model_id or "").strip()
+                    if not model_id:
+                        continue
+                    normalized = _normalize_model_tool_mode(mode)
+                    if normalized:
+                        modes[model_id] = normalized
+                    else:
+                        modes.pop(model_id, None)
+                ep.model_tool_modes = json.dumps(modes) if modes else None
             db.commit()
             _invalidate_models_cache()
             hidden_count = len(json.loads(ep.hidden_models)) if ep.hidden_models else 0
             pinned_count = len(json.loads(ep.pinned_models)) if ep.pinned_models else 0
-            return {"id": ep_id, "hidden_count": hidden_count, "pinned_count": pinned_count}
+            return {
+                "id": ep_id,
+                "hidden_count": hidden_count,
+                "pinned_count": pinned_count,
+                "model_tool_modes": _model_tool_modes(ep),
+            }
         finally:
             db.close()
 
@@ -2572,6 +2633,7 @@ def setup_model_routes(model_discovery):
                 "model_type": ep.model_type,
                 "base_url": ep.base_url,
                 "pinned_models": _normalize_model_ids(getattr(ep, "pinned_models", None)),
+                "model_tool_modes": _model_tool_modes(ep),
                 "endpoint_kind": getattr(ep, "endpoint_kind", None) or "auto",
                 "model_refresh_mode": getattr(ep, "model_refresh_mode", None) or "auto",
                 "model_refresh_interval": getattr(ep, "model_refresh_interval", None),

routes/note/note_routes.py

@@ -35,6 +35,7 @@ class NoteCreate(BaseModel):
     source: str = "user"
     session_id: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = "none"
     sort_order: Optional[int] = None
 
@@ -50,6 +51,7 @@ class NoteUpdate(BaseModel):
     archived: Optional[bool] = None
     due_date: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = None
     sort_order: Optional[int] = None
     agent_session_id: Optional[str] = None
@@ -89,6 +91,7 @@ def _note_to_dict(note: Note) -> Dict[str, Any]:
         "session_id": note.session_id,
         "sort_order": note.sort_order or 0,
         "image_url": note.image_url,
+        "gallery_id": getattr(note, "gallery_id", None),
         "repeat": note.repeat or "none",
         "ai_classification": ai_cls,
         "ai_content_hash": getattr(note, "ai_content_hash", None),
@@ -674,6 +677,7 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 source=body.source,
                 session_id=body.session_id,
                 image_url=body.image_url,
+                gallery_id=body.gallery_id,
                 repeat=body.repeat or "none",
                 sort_order=body.sort_order if body.sort_order is not None else 0,
             )
@@ -743,6 +747,8 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 note.due_date = body.due_date
             if body.image_url is not None:
                 note.image_url = body.image_url
+            if body.gallery_id is not None:
+                note.gallery_id = body.gallery_id
             if body.repeat is not None:
                 note.repeat = body.repeat
             if body.sort_order is not None:

routes/preset_routes.py

@@ -21,6 +21,8 @@ class UserTemplateRequest(BaseModel):
     system_prompt: str = Field("", max_length=10000)
     temperature: float = Field(1.0, ge=0.0, le=2.0)
     max_tokens: int = Field(0, ge=0, le=65536)
+    persona_memory: str = Field("", max_length=6000)
+    persona_memory_schema: str = Field("general", pattern="^(general|health)$")
 
 
 def setup_preset_routes(preset_manager) -> APIRouter:
@@ -41,6 +43,10 @@ def setup_preset_routes(preset_manager) -> APIRouter:
                 preset_update.enabled,
                 preset_update.inject_prefix,
                 preset_update.inject_suffix,
+                preset_update.persona_memory,
+                preset_update.persona_memory_schema,
+                preset_update.thinking_mode,
+                preset_update.show_persona_name,
             )
             if success:
                 return {"success": True, "message": "Custom preset updated"}

routes/research/research_routes.py

@@ -7,7 +7,7 @@ import re
 import uuid
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Literal, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request
 from fastapi.responses import HTMLResponse, StreamingResponse
@@ -271,7 +271,13 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "query": entry.get("query", ""),
                     "status": "running",
                     "progress": entry.get("progress", {}),
+                    "source_state": research_handler.get_source_state(sid),
+                    "source_coverage": research_handler.get_source_coverage(sid),
+                    "navigation_trace": research_handler.get_navigation_trace(sid),
+                    "action_trace": research_handler.get_action_trace(sid),
                     "started_at": entry.get("started_at", 0),
+                    "category": research_handler.get_category(sid),
+                    "mode": research_handler.get_mode(sid),
                 })
         return {"active": active}
 
@@ -284,6 +290,24 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         status = research_handler.get_status(session_id)
         if status is None:
             raise HTTPException(404, "No research found for this session")
+        try:
+            source_state = research_handler.get_source_state(session_id)
+            if isinstance(source_state, str) and source_state:
+                status["source_state"] = source_state
+            source_coverage = research_handler.get_source_coverage(session_id)
+            if isinstance(source_coverage, dict) and source_coverage:
+                status["source_coverage"] = source_coverage
+        except Exception:
+            pass
+        try:
+            navigation_trace = research_handler.get_navigation_trace(session_id)
+            if isinstance(navigation_trace, list) and navigation_trace:
+                status["navigation_trace"] = navigation_trace
+            action_trace = research_handler.get_action_trace(session_id)
+            if isinstance(action_trace, list) and action_trace:
+                status["action_trace"] = action_trace
+        except Exception:
+            pass
         return status
 
     @router.post("/api/research/cancel/{session_id}")
@@ -306,8 +330,26 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        category = research_handler.get_category(session_id)
+        mode = research_handler.get_mode(session_id)
         research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": category,
+            "mode": mode,
+        }
 
     def _assert_owns_research(session_id: str, user: str) -> None:
         """404-not-403 ownership gate for a research session's on-disk JSON.
@@ -394,6 +436,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "id": p.stem,
                     "query": query,
                     "category": d.get("category") or "",
+                    "mode": d.get("mode") or "research",
+                    "mode": d.get("mode") or "research",
                     "source_count": len(sources),
                     "status": d.get("status", "done"),
                     "duration": d.get("stats", {}).get("Duration", ""),
@@ -479,6 +523,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
 
     class ResearchStartRequest(BaseModel):
         query: str
+        origin_chat_id: Optional[str] = None
         # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
         max_rounds: int = Field(default=0, ge=0, le=20)
         search_provider: Optional[str] = None
@@ -487,7 +532,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         max_time: int = Field(default=300, ge=60, le=1800)
         extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
         extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
+        category: Optional[Literal["product", "comparison", "howto", "factcheck"]] = None
 
     @router.post("/api/research/start")
     async def research_start(body: ResearchStartRequest, request: Request):
@@ -509,6 +554,15 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                         pass
                 user = tool_owner
         session_id = f"rp-{uuid.uuid4().hex[:12]}"
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        if body.origin_chat_id:
+            from core.database import SessionLocal, Session as DbSession
+            with SessionLocal() as db:
+                origin = db.get(DbSession, body.origin_chat_id)
+                if origin is None or origin.owner != user:
+                    raise HTTPException(404, 'Origin chat not found')
+            if delivery is None:
+                raise HTTPException(503, 'Background chat delivery is unavailable')
 
         if body.endpoint_id:
             from src.database import SessionLocal
@@ -558,8 +612,12 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             if body.model:
                 ep_model = body.model
 
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        # 0 = auto research capped at 20.
+        effective_max_rounds = body.max_rounds if body.max_rounds != 0 else 20
+        if body.origin_chat_id and 'max_rounds' not in body.model_fields_set:
+            effective_max_rounds = 2
+        if body.origin_chat_id:
+            delivery.register(session_id, body.origin_chat_id, user, 'research', body.query, effective_max_rounds)
         research_handler.start_research(
             session_id=session_id,
             query=body.query,
@@ -573,8 +631,22 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             extraction_timeout=body.extraction_timeout,
             extraction_concurrency=body.extraction_concurrency,
             owner=user,
+            on_complete=(lambda sid, result, sources, findings: delivery.complete(sid, result, sources))
+                if body.origin_chat_id else None,
         )
-        return {"session_id": session_id, "status": "running", "query": body.query}
+        return {
+            "session_id": session_id,
+            "status": "running",
+            "query": body.query,
+            "category": body.category or "",
+            "mode": "research",
+        }
+
+    @router.get('/api/research/chat-jobs/{chat_id}')
+    async def chat_research_jobs(chat_id: str, request: Request):
+        user = _require_user(request)
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        return {'jobs': delivery.list_for_chat(chat_id, user) if delivery else []}
 
     @router.get("/api/research/stream/{session_id}")
     async def research_stream(session_id: str, request: Request):
@@ -584,7 +656,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         if not _owns_in_memory(session_id, user):
             raise HTTPException(404, "No research found for this session")
         async def _generate():
-            last_progress = None
+            last_payload = None
             while True:
                 status = research_handler.get_status(session_id)
                 if status is None:
@@ -592,9 +664,30 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     return
                 st = status.get("status", "")
                 progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                payload = {
+                    **progress,
+                    'status': st,
+                    'category': research_handler.get_category(session_id),
+                    'mode': research_handler.get_mode(session_id),
+                }
+                try:
+                    source_state = research_handler.get_source_state(session_id)
+                    if source_state:
+                        payload["source_state"] = source_state
+                    source_coverage = research_handler.get_source_coverage(session_id)
+                    if source_coverage:
+                        payload["source_coverage"] = source_coverage
+                    navigation_trace = research_handler.get_navigation_trace(session_id)
+                    if navigation_trace:
+                        payload["navigation_trace"] = navigation_trace
+                    action_trace = research_handler.get_action_trace(session_id)
+                    if action_trace:
+                        payload["action_trace"] = action_trace
+                except Exception:
+                    pass
+                if payload != last_payload:
+                    last_payload = payload
+                    yield f"data: {json.dumps(payload)}\n\n"
                 if st != "running":
                     final = {'status': st, 'final': True}
                     task = research_handler._active_tasks.get(session_id, {})
@@ -625,12 +718,33 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "result": d.get("result", ""),
                     "sources": d.get("sources", []),
                     "raw_findings": d.get("raw_findings", []),
+                    "analyzed_urls": d.get("analyzed_urls", []),
+                    "source_state": d.get("source_state", ""),
+                    "source_coverage": d.get("source_coverage", {}),
+                    "navigation_trace": d.get("navigation_trace", []),
+                    "action_trace": d.get("action_trace", []),
                     "category": d.get("category") or "",
                 }
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": research_handler.get_category(session_id),
+            "mode": research_handler.get_mode(session_id),
+        }
 
     @router.post("/api/research/spinoff/{session_id}")
     async def research_spinoff(session_id: str, request: Request):

routes/search/search_routes.py

@@ -1,9 +1,12 @@
 """Search routes — /api/search/config GET, /api/search POST."""
 
+import html
+import json
 import logging
 from typing import Dict, Any
 
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, Query, Request
+from fastapi.responses import HTMLResponse
 
 import time
 
@@ -39,6 +42,91 @@ async def _request_values(request: Request) -> Dict[str, Any]:
 def setup_search_routes(config) -> APIRouter:
     router = APIRouter(tags=["search"])
 
+    @router.get("/search/web", response_class=HTMLResponse)
+    async def web_search_page(q: str = Query("", min_length=0)) -> HTMLResponse:
+        """Browser-facing search results page for clickable agent web_search rows."""
+        safe_q = str(q or "").strip()
+        title = html.escape(safe_q or "Web search")
+        q_json = json.dumps(safe_q)
+        page = f"""<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>{title} - Odysseus Search</title>
+  <style>
+    :root {{ color-scheme: dark; --bg:#111; --fg:#eee; --muted:#999; --border:#333; --accent:#e05252; }}
+    body {{ margin:0; font:14px/1.45 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; background:var(--bg); color:var(--fg); }}
+    main {{ max-width:900px; margin:0 auto; padding:22px 18px 40px; }}
+    form {{ display:flex; gap:8px; margin:0 0 16px; }}
+    input {{ flex:1; min-width:0; height:34px; padding:0 10px; border:1px solid var(--border); border-radius:7px; background:#181818; color:var(--fg); font:inherit; }}
+    button {{ height:34px; padding:0 13px; border:1px solid color-mix(in srgb,var(--accent) 45%,var(--border)); border-radius:7px; background:color-mix(in srgb,var(--accent) 14%,transparent); color:var(--fg); font:inherit; cursor:pointer; }}
+    h1 {{ margin:0 0 14px; font-size:16px; font-weight:650; }}
+    .status {{ color:var(--muted); font-size:12px; margin:8px 0 14px; }}
+    .result {{ display:block; padding:11px 0; border-top:1px solid var(--border); text-decoration:none; color:inherit; }}
+    .result-title {{ color:var(--fg); font-weight:650; }}
+    .result-url {{ margin-top:3px; color:color-mix(in srgb,var(--accent) 78%,var(--fg)); font-size:12px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }}
+    .result-snippet {{ margin-top:5px; color:color-mix(in srgb,var(--fg) 72%,transparent); font-size:13px; }}
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Web Search</h1>
+    <form id="search-form">
+      <input id="query" value="{html.escape(safe_q, quote=True)}" autocomplete="off">
+      <button type="submit">Search</button>
+    </form>
+    <div class="status" id="status">Loading...</div>
+    <div id="results"></div>
+  </main>
+  <script>
+    const initialQuery = {q_json};
+    const input = document.getElementById('query');
+    const statusEl = document.getElementById('status');
+    const resultsEl = document.getElementById('results');
+    function esc(value) {{
+      return String(value || '').replace(/[&<>"']/g, ch => ({{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}}[ch]));
+    }}
+    async function runSearch(query) {{
+      query = String(query || '').trim();
+      if (!query) {{ statusEl.textContent = 'Enter a search query.'; resultsEl.innerHTML = ''; return; }}
+      statusEl.textContent = 'Searching...';
+      resultsEl.innerHTML = '';
+      const fd = new FormData();
+      fd.append('query', query);
+      const res = await fetch('/api/search', {{ method: 'POST', credentials: 'same-origin', body: fd }});
+      const data = await res.json().catch(() => ({{}}));
+      const sources = Array.isArray(data.sources) ? data.sources : [];
+      if (!res.ok || data.error) {{
+        statusEl.textContent = data.error || `Search failed (${{res.status}})`;
+        return;
+      }}
+      statusEl.textContent = sources.length ? `${{sources.length}} results` : 'No results';
+      resultsEl.innerHTML = sources.map(s => {{
+        const url = s.url || s.link || '';
+        const title = s.title || url || 'Untitled';
+        const snippet = s.snippet || s.content || '';
+        return `<a class="result" href="${{esc(url)}}" target="_blank" rel="noopener noreferrer">
+          <div class="result-title">${{esc(title)}}</div>
+          <div class="result-url">${{esc(url)}}</div>
+          <div class="result-snippet">${{esc(snippet)}}</div>
+        </a>`;
+      }}).join('');
+    }}
+    document.getElementById('search-form').addEventListener('submit', ev => {{
+      ev.preventDefault();
+      const q = input.value.trim();
+      const url = new URL(window.location.href);
+      url.searchParams.set('q', q);
+      history.replaceState(null, '', url);
+      runSearch(q);
+    }});
+    runSearch(initialQuery);
+  </script>
+</body>
+</html>"""
+        return HTMLResponse(page)
+
     @router.get("/api/search/config")
     async def get_search_settings() -> Dict[str, Any]:
         return get_search_config()

routes/session_routes.py

@@ -3,8 +3,10 @@ import re
 import html
 import json
 import uuid
+import time
+from pathlib import Path
 from datetime import datetime
-from fastapi import APIRouter, Form, HTTPException, Response, Request
+from fastapi import APIRouter, Form, HTTPException, Response, Request, Query
 import logging
 
 from core.session_manager import SessionManager
@@ -60,6 +62,114 @@ def _content_to_text(content) -> str:
     return ""
 
 
+def _context_info_skill_inventory(
+    skills_manager, owner: str | None, limit: int = 80
+) -> list[dict]:
+    """Compact skill metadata for TUI context/status/autocomplete.
+
+    This intentionally exposes only the skill index fields. Full SKILL.md
+    bodies remain behind manage_skills/view so context_info cannot become a
+    prompt/body dump path.
+    """
+    if not skills_manager:
+        return []
+    try:
+        indexed = skills_manager.index_for(owner=owner, active_toolsets=None)
+    except Exception:
+        return []
+    try:
+        loaded = skills_manager.load(owner=owner)
+    except Exception:
+        loaded = []
+    paths_by_name = {
+        str(skill.get("name") or ""): str(skill.get("path") or "").strip()
+        for skill in loaded
+        if isinstance(skill, dict)
+    }
+    out: list[dict] = []
+    seen: set[str] = set()
+    for row in indexed:
+        if not isinstance(row, dict):
+            continue
+        name = str(row.get("name") or "").strip()
+        if not name or name in seen:
+            continue
+        item = {"name": name}
+        description = str(row.get("description") or "").strip()
+        if description:
+            item["description"] = description
+        path = paths_by_name.get(name, "")
+        if path:
+            item["source"] = f"file: {path}"
+        out.append(item)
+        seen.add(name)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_tool_inventory(limit: int = 80) -> list[dict]:
+    """Compact built-in tool metadata for TUI context/status/autocomplete."""
+    try:
+        from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS
+    except Exception:
+        return []
+    out: list[dict] = []
+    for name, description in BUILTIN_TOOL_DESCRIPTIONS.items():
+        clean_name = str(name or "").strip()
+        if not clean_name:
+            continue
+        item = {"name": clean_name, "source": "backend"}
+        clean_description = re.sub(r"\s+", " ", str(description or "")).strip()
+        if clean_description:
+            item["description"] = clean_description[:280]
+        out.append(item)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_agents_md_inventory(workspace: str | None, limit: int = 8) -> list[dict]:
+    """Compact AGENTS.md path metadata for the active workspace.
+
+    Bodies intentionally stay on disk. The TUI can read a selected file only
+    when the user asks for `/agent <path> --show`.
+    """
+    try:
+        from src.tool_execution import vet_workspace
+        root = vet_workspace(workspace or "")
+    except Exception:
+        root = None
+    if not root:
+        return []
+
+    start = Path(root).resolve()
+    candidates = []
+    current = start
+    while True:
+        candidate = current / "AGENTS.md"
+        if candidate.is_file():
+            candidates.append(candidate)
+        if current.parent == current:
+            break
+        current = current.parent
+        if len(candidates) >= limit:
+            break
+
+    # Codex-style precedence reads parent instructions before child overrides.
+    out: list[dict] = []
+    seen: set[str] = set()
+    for candidate in reversed(candidates):
+        path = str(candidate)
+        if path in seen:
+            continue
+        out.append({"path": path, "source": "workspace"})
+        seen.add(path)
+        if len(out) >= limit:
+            break
+    return out
+
+
 def _message_role(message) -> str:
     if isinstance(message, ChatMessage):
         return message.role or ""
@@ -157,20 +267,36 @@ def _reject_raw_endpoint_url_for_non_admin(
         raise HTTPException(403, "Choose a registered model endpoint")
 
 
-def _persist_session_headers(session_id: str, headers: dict | None) -> None:
+def _persist_session_headers(session_id: str, headers: dict | None) -> bool:
     """Persist endpoint auth headers for DB-backed session metadata."""
-    db = SessionLocal()
-    try:
-        db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-        if db_session:
-            db_session.headers = headers or {}
-            db_session.updated_at = utcnow_naive()
-            db.commit()
-    except Exception:
-        db.rollback()
-        raise
-    finally:
-        db.close()
+    delays = (0.05, 0.15, 0.35)
+    last_exc: Exception | None = None
+    for attempt in range(len(delays) + 1):
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if db_session:
+                db_session.headers = headers or {}
+                db_session.updated_at = utcnow_naive()
+                db.commit()
+            return True
+        except Exception as exc:
+            db.rollback()
+            last_exc = exc
+            if attempt >= len(delays):
+                break
+            if "database is locked" not in str(exc).lower():
+                break
+            time.sleep(delays[attempt])
+        finally:
+            db.close()
+
+    logger.warning(
+        "Failed to persist headers for session %s; continuing with in-memory headers: %s",
+        session_id,
+        last_exc,
+    )
+    return False
 
 
 _HIDDEN_SYSTEM_SESSION_NAMES = {
@@ -184,6 +310,16 @@ _HIDDEN_SYSTEM_SESSION_NAMES = {
 }
 
 
+def _is_hidden_session_name(name: str | None) -> bool:
+    """Return whether a session should be omitted from the sidebar list."""
+    clean = (name or "").strip()
+    return (
+        clean in ("Nobody", "Incognito")
+        or clean in _HIDDEN_SYSTEM_SESSION_NAMES
+        or clean.startswith("SFT trace batch ")
+    )
+
+
 def _pick_endpoint_for_sort(owner=None):
     """Pick model endpoint for auto-sort LLM call — uses utility endpoint setting, falls back to default."""
     from src.endpoint_resolver import resolve_endpoint
@@ -210,6 +346,7 @@ def setup_session_routes(
     config: dict,
     webhook_manager=None,
     upload_handler=None,
+    skills_manager=None,
 ):
     """Setup session routes with the provided manager and config"""
 
@@ -258,35 +395,22 @@ def setup_session_routes(
         except Exception:
             pass
         user_sessions = session_manager.get_sessions_for_user(user)
-        # Fetch folder info from DB for each session
+        # The sidebar must be backed by persisted DB rows. SessionManager only
+        # hydrates a bounded recent cache at startup, so older-but-valid
+        # conversations can disappear after refresh if this endpoint trusts
+        # memory as the source of truth.
         db = SessionLocal()
         try:
-            folder_map = {}
-            token_map = {}
-            important_map = {}
-            created_map = {}
-            updated_map = {}
-            last_msg_map = {}
-            mode_map = {}
-            msg_count_map = {}
-            q = db.query(DbSession.id, DbSession.folder, DbSession.total_input_tokens, DbSession.total_output_tokens, DbSession.is_important, DbSession.created_at, DbSession.updated_at, DbSession.last_message_at, DbSession.mode, DbSession.message_count).filter(DbSession.archived == False)
+            q = (
+                db.query(DbSession)
+                .filter(DbSession.archived == False)
+                .order_by(DbSession.is_important.desc(), DbSession.updated_at.desc())
+            )
             q = owner_filter(q, DbSession, user)
-            rows = q.all()
-            for row in rows:
-                folder_map[row.id] = row.folder
-                token_map[row.id] = (row.total_input_tokens or 0) + (row.total_output_tokens or 0)
-                important_map[row.id] = row.is_important or False
-                created_map[row.id] = row.created_at.isoformat() if row.created_at else None
-                updated_map[row.id] = row.updated_at.isoformat() if row.updated_at else None
-                # Fall back to updated_at then created_at so sessions that
-                # predate the column (or have no messages) still sort sanely.
-                last_msg_map[row.id] = (
-                    row.last_message_at.isoformat() if row.last_message_at
-                    else (row.updated_at.isoformat() if row.updated_at
-                          else (row.created_at.isoformat() if row.created_at else None))
-                )
-                mode_map[row.id] = row.mode
-                msg_count_map[row.id] = row.message_count or 0
+            rows = [
+                row for row in q.all()
+                if not _is_hidden_session_name(row.name)
+            ]
             # Sessions with active documents that have content
             from sqlalchemy import func
             doc_session_ids = set(
@@ -305,26 +429,58 @@ def setup_session_routes(
                     GalleryImage, user)
                 .distinct().all()
             )
+
+            # Resolve saved routes without waiting for the frontend model catalog.
+            from core.database import ModelEndpoint
+            from src.endpoint_resolver import build_chat_url, normalize_base
+            endpoint_routes = {}
+            endpoint_query = owner_filter(db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), ModelEndpoint, user)
+            for endpoint in endpoint_query.all():
+                route_url = build_chat_url(normalize_base(endpoint.base_url or '')).rstrip('/')
+                endpoint_routes.setdefault(route_url, []).append(endpoint)
+            sessions = []
+            for s in rows:
+                if (
+                    (s.message_count or 0) <= 0
+                    and s.id not in doc_session_ids
+                    and s.id not in img_session_ids
+                    and s.id not in user_sessions
+                ):
+                    continue
+                # Fall back to updated_at then created_at so sessions that
+                # predate the column (or have no messages) still sort sanely.
+                last_message_at = (
+                    s.last_message_at.isoformat() if s.last_message_at
+                    else (s.updated_at.isoformat() if s.updated_at
+                          else (s.created_at.isoformat() if s.created_at else None))
+                )
+                matches = endpoint_routes.get((s.endpoint_url or '').rstrip('/'), [])
+                selected_endpoint = matches[0] if len(matches) == 1 else None
+                sessions.append({
+                    "id": s.id,
+                    "name": s.name,
+                    "model": _public_model(s.name, s.model),
+                    "endpoint_url": s.endpoint_url,
+                    "endpoint_id": selected_endpoint.id if selected_endpoint else None,
+                    "endpoint_name": selected_endpoint.name if selected_endpoint else None,
+                    "rag": s.rag,
+                    "archived": s.archived,
+                    "folder": s.folder,
+                    "cwd": s.cwd,
+                    "total_tokens": (s.total_input_tokens or 0) + (s.total_output_tokens or 0),
+                    "total_cost_usd": s.total_cost_usd or 0.0,
+                    "is_important": s.is_important or False,
+                    "created_at": s.created_at.isoformat() if s.created_at else None,
+                    "updated_at": s.updated_at.isoformat() if s.updated_at else None,
+                    "last_message_at": last_message_at,
+                    "has_documents": s.id in doc_session_ids,
+                    "has_images": s.id in img_session_ids,
+                    "mode": s.mode,
+                    "message_count": s.message_count or 0,
+                })
         finally:
             db.close()
 
-        sessions = [{"id": s.id, "name": s.name, "model": _public_model(s.name, s.model),
-                     "endpoint_url": s.endpoint_url, "rag": s.rag,
-                     "archived": s.archived, "folder": folder_map.get(s.id),
-                     "total_tokens": token_map.get(s.id, 0),
-                     "is_important": important_map.get(s.id, False),
-                     "created_at": created_map.get(s.id),
-                     "updated_at": updated_map.get(s.id),
-                     "last_message_at": last_msg_map.get(s.id),
-                     "has_documents": s.id in doc_session_ids,
-                     "has_images": s.id in img_session_ids,
-                     "mode": mode_map.get(s.id),
-                     "message_count": msg_count_map.get(s.id, 0)}
-                    for s in user_sessions.values()
-                    if not s.archived
-                    and (s.name or "").strip() not in ("Nobody", "Incognito")
-                    and (s.name or "").strip() not in _HIDDEN_SYSTEM_SESSION_NAMES]
-
         return sessions
     
     @router.post("/session", response_model=SessionResponse)
@@ -337,6 +493,7 @@ def setup_session_routes(
         skip_validation: str = Form(None),
         api_key: str = Form(""),
         endpoint_id: str = Form(""),
+        cwd: str = Form(None),
     ):
         skip_val = str(skip_validation).lower() == "true"
         user = effective_user(request)
@@ -432,6 +589,7 @@ def setup_session_routes(
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
             owner=user,
+            cwd=cwd or None,
         )
         # Set auth headers for custom API-key endpoints
         resolved_key = request_api_key
@@ -456,7 +614,8 @@ def setup_session_routes(
             name=session.name,
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
-            archived=False
+            archived=False,
+            cwd=session.cwd,
         )    
     @router.patch("/session/{sid}")
     def rename_session(
@@ -464,6 +623,7 @@ def setup_session_routes(
         name: str = Form(None), folder: str = Form(None),
         model: str = Form(None), endpoint_url: str = Form(None),
         endpoint_id: str = Form(None),
+        cwd: str = Form(None),
     ):
         _verify_session_owner(request, sid)
         try:
@@ -486,6 +646,19 @@ def setup_session_routes(
                     result["folder"] = folder if folder else None
             finally:
                 db.close()
+        if cwd is not None:
+            clean_cwd = cwd.strip() or None
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == sid).first()
+                if db_session:
+                    db_session.cwd = clean_cwd
+                    db_session.updated_at = utcnow_naive()
+                    db.commit()
+                    session.cwd = clean_cwd
+                    result["cwd"] = clean_cwd
+            finally:
+                db.close()
         # Switch model/endpoint mid-session
         if model is not None and endpoint_url is not None:
             user = effective_user(request)
@@ -597,6 +770,8 @@ def setup_session_routes(
                     db.close()
 
                 if session_manager.delete_session(sid):
+                    from routes.chat_helpers import remove_session_sft_trace_rows
+                    remove_session_sft_trace_rows(effective_user(request), sid)
                     deleted_count += 1
             except Exception:
                 pass
@@ -621,6 +796,8 @@ def setup_session_routes(
 
             # Delete the session and all its messages
             if session_manager.delete_session(sid):
+                from routes.chat_helpers import remove_session_sft_trace_rows
+                remove_session_sft_trace_rows(effective_user(request), sid)
                 return {"status": "deleted"}
             else:
                 raise HTTPException(404, "Session not found")
@@ -1034,11 +1211,19 @@ def setup_session_routes(
         if not session_manager.replace_messages(session_id, new_history):
             raise HTTPException(500, "Failed to save compacted history")
 
+        # Rough token estimate of the compacted history so clients can
+        # refresh their context-pressure display without waiting for the
+        # next turn's metrics event.
+        context_tokens_estimate = sum(
+            len(_message_text(m) or "") // 4 + 8 for m in new_history
+        )
+
         return {
             "ok": True,
             "summarized": len(older),
             "kept": len(recent),
             "message_count": len(new_history),
+            "context_tokens_estimate": context_tokens_estimate,
         }
 
     @router.post("/sessions/auto-sort")
@@ -1328,19 +1513,74 @@ def setup_session_routes(
         }
 
     @router.get("/session/{session_id}/context_info")
-    async def get_context_info(request: Request, session_id: str):
+    async def get_context_info(
+        request: Request,
+        session_id: str,
+        cwd: str | None = Query(default=None),
+    ):
         """Get the real context length for a session's model from the endpoint."""
         _verify_session_owner(request, session_id)
+        owner = effective_user(request)
         session = session_manager.get_session(session_id)
         if not session:
             raise HTTPException(404, "Session not found")
+        skills = _context_info_skill_inventory(skills_manager, owner=owner)
+        tools = _context_info_tool_inventory()
+        agents_md = _context_info_agents_md_inventory(cwd)
+        # Workspace visibility: lets the TUI answer "can the backend actually
+        # see this directory?" (mounted vs bridge-only) without probing.
+        from src.workspace_paths import backend_workspace_path, workspace_mount_pairs
+
+        _raw_cwd = str(cwd or getattr(session, "cwd", "") or "").strip()
+        _backend_cwd = backend_workspace_path(_raw_cwd)[:400] if _raw_cwd else ""
+        # Server-side tool policy: non-admin owners silently lose the computer
+        # tools (src/tool_security); surface that so the TUI can show it.
+        try:
+            from src.tool_security import blocked_tools_for_owner
+
+            _blocked = blocked_tools_for_owner(owner)
+        except Exception:
+            _blocked = set()
+        _computer = {"bash", "python", "read_file", "write_file", "host_shell"}
+        _policy = {
+            "computer_tools": "restricted" if _computer & _blocked else "full",
+            "reason": "non-admin owner" if _blocked else "single-user or admin",
+        }
+        _workspace = {
+            "backend_path": _backend_cwd,
+            "exists_in_backend": bool(_backend_cwd) and Path(_backend_cwd).is_dir(),
+            "mount_configured": bool(workspace_mount_pairs()),
+            "via_mount": bool(_raw_cwd) and backend_workspace_path(_raw_cwd) != _raw_cwd,
+        }
         if not session.endpoint_url or not session.model:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         try:
             from src.model_context import get_context_length
             ctx = get_context_length(session.endpoint_url, session.model)
-            return {"context_length": ctx, "model": session.model}
+            return {
+                "context_length": ctx,
+                "model": session.model,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         except Exception:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
 
     return router

routes/shell_routes.py

@@ -11,6 +11,7 @@ import shutil
 import subprocess
 import uuid
 import tempfile
+import time
 from collections import namedtuple
 from pathlib import Path
 from typing import Dict, Any
@@ -22,6 +23,7 @@ from src.host_docker_access import (
     running_in_container as _running_in_container,
 )
 from src.optional_deps import prepare_optional_dependency_import
+from src.auth_helpers import _auth_disabled
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
 # on Windows, so importing them unconditionally crashed app startup there
@@ -53,6 +55,11 @@ from core.platform_compat import (
 def _require_admin(request: Request):
     """Reject non-admin callers. Shell exec is admin-only — never expose to
     regular users; that's RCE-after-signup."""
+    # In the explicitly single-user, auth-disabled deployment the middleware
+    # does not attach a current user. AuthManager is still instantiated by the
+    # app, so checking only for its presence incorrectly returns 403 here.
+    if _auth_disabled():
+        return
     auth_manager = getattr(request.app.state, "auth_manager", None)
     if not auth_manager:
         # No auth at all — only safe in fully-trusted localhost dev mode
@@ -78,6 +85,13 @@ def _reject_cross_site(request: Request):
 _SSH_PORT_RE = re.compile(r"^\d{1,5}$")
 _SAFE_VENV_RE = re.compile(r"^[A-Za-z0-9_./~-]+$")
 
+# Dependency probes can involve several SSH/import checks. Keep the result
+# briefly so the Dependencies tab and a pre-launch check arriving together do
+# not repeat the same expensive work. Installation clears this cache.
+_PACKAGE_STATUS_CACHE: dict[tuple[str, ...], tuple[float, dict[str, Any]]] = {}
+_PACKAGE_STATUS_CACHE_TTL = 3.0
+_PACKAGE_STATUS_CACHE_MAX = 64
+
 
 def _ssh_base_argv(host: str, ssh_port: str | None) -> list[str]:
     """Build an ssh argv prefix for remote probes without local-shell parsing."""
@@ -204,6 +218,19 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
             (dists.get("transformers") or modules.get("transformers", {}).get("real_module"))
             and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
         )
+    if name == "office_docs":
+        return bool(
+            dists.get("markitdown")
+            or modules.get("markitdown", {}).get("real_module")
+            or dists.get("python-docx")
+            or modules.get("docx", {}).get("real_module")
+        )
+    if name == "psd_tools":
+        return bool(dists.get("psd-tools") or modules.get("psd_tools", {}).get("real_module"))
+    if name == "pymupdf":
+        return bool(dists.get("PyMuPDF") or modules.get("fitz", {}).get("real_module"))
+    if name == "libreoffice":
+        return bool(binaries.get("soffice") or binaries.get("libreoffice"))
     if name == "hf_transfer":
         return bool(
             dists.get("hf-transfer")
@@ -254,6 +281,28 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"SAM object masks: transformers {dists.get('transformers', 'available')} with torch {dists.get('torch', 'available')}"
         return "SAM click/object mask selection needs transformers and torch."
+    if name == "office_docs":
+        if _package_installed_from_probe(name, probe):
+            if dists.get("markitdown"):
+                return f"Office document extraction: markitdown {dists['markitdown']}"
+            if dists.get("python-docx"):
+                return f"Word document extraction: python-docx {dists['python-docx']}"
+            return "Office document extraction available"
+        return "Office attachments need MarkItDown for full fidelity; DOCX has a basic built-in fallback."
+    if name == "psd_tools":
+        if _package_installed_from_probe(name, probe):
+            return f"PSD support: psd-tools {dists.get('psd-tools', 'available')}"
+        return "PSD files need psd-tools for layer/image parsing."
+    if name == "pymupdf":
+        if _package_installed_from_probe(name, probe):
+            return f"PDF forms/rendering: PyMuPDF {dists.get('PyMuPDF', 'available')}"
+        return "Advanced PDF open/render/form features need PyMuPDF."
+    if name == "libreoffice":
+        if binaries.get("soffice"):
+            return f"DOCX signable preview converter: {binaries['soffice']}"
+        if binaries.get("libreoffice"):
+            return f"DOCX signable preview converter: {binaries['libreoffice']}"
+        return "DOCX signing preview needs LibreOffice/soffice to convert Word files to PDF."
     if name == "mlx_lm":
         if _package_installed_from_probe(name, probe):
             return f"MLX LM {dists.get('mlx-lm', 'available')}"
@@ -399,16 +448,21 @@ dist_names={{
     'diffusers':['diffusers','torch'],
     'krea_diffusers':['diffusers','torch'],
     'sam_mask':['transformers','torch'],
-    'hf_transfer':['hf-transfer','hf_transfer'],
-}}
-bin_names={{
+    'office_docs':['markitdown','python-docx'],
+	    'psd_tools':['psd-tools'],
+	    'pymupdf':['PyMuPDF'],
+	    'libreoffice':[],
+	    'hf_transfer':['hf-transfer','hf_transfer'],
+	}}
+	bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
     'mflux':['mflux-generate-qwen', 'mflux-generate'],
-    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
-    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
-    'tmux':['tmux'],
-}}
+	    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
+	    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
+	    'libreoffice':['soffice', 'libreoffice'],
+	    'tmux':['tmux'],
+	}}
 
 def add_user_install_bins_to_path():
     candidates = []
@@ -457,6 +511,13 @@ def probe(n):
     mods = {{n: mod_status(n)}}
     if n == 'diffusers':
         mods['torch'] = mod_status('torch')
+    if n == 'office_docs':
+        mods['markitdown'] = mod_status('markitdown')
+        mods['docx'] = mod_status('docx')
+    if n == 'psd_tools':
+        mods['psd_tools'] = mod_status('psd_tools')
+    if n == 'pymupdf':
+        mods['fitz'] = mod_status('fitz')
     dists = dist_status(dist_names.get(n, [n]))
     bins = {{b: shutil.which(b) for b in bin_names.get(n, [])}}
     files = {{}}
@@ -1145,6 +1206,7 @@ def setup_shell_routes() -> APIRouter:
         "make":            {"debian": ["make"], "arch": ["make"], "fedora": ["make"], "alpine": ["make"], "suse": ["make"], "macos": []},
         "git":             {"debian": ["git"], "arch": ["git"], "fedora": ["git"], "alpine": ["git"], "suse": ["git"], "macos": ["git"]},
         "tmux":            {"debian": ["tmux"], "arch": ["tmux"], "fedora": ["tmux"], "alpine": ["tmux"], "suse": ["tmux"], "macos": ["tmux"]},
+        "libreoffice":     {"debian": ["libreoffice"], "arch": ["libreoffice-fresh"], "fedora": ["libreoffice"], "alpine": ["libreoffice"], "suse": ["libreoffice"], "macos": ["--cask", "libreoffice"]},
     }
     _BACKEND_EXTRAS = {
         "cuda":   {"debian": ["nvidia-cuda-toolkit"], "arch": ["cuda"], "fedora": ["cuda-toolkit"], "alpine": [], "suse": ["cuda"], "macos": []},
@@ -1206,13 +1268,16 @@ def setup_shell_routes() -> APIRouter:
         import sys
 
         platform_l = (platform or "").strip().lower()
-        model_hint_l = (model_hint or "").strip().lower()
-        has_krea_model = "krea" in model_hint_l
-        has_lama_mlx_model = any(
-            key in model_hint_l
-            for key in ("lama", "mi-gan", "migan", "inpainting-mlx")
+        package_cache_key = (
+            (host or "").strip(),
+            (ssh_port or "").strip(),
+            (venv or "").strip(),
+            (backend or "").strip().lower(),
+            platform_l,
         )
-        has_ddcolor_mlx_model = "ddcolor" in model_hint_l
+        cached_status = _PACKAGE_STATUS_CACHE.get(package_cache_key)
+        if cached_status and time.monotonic() - cached_status[0] < _PACKAGE_STATUS_CACHE_TTL:
+            return cached_status[1]
         _prepend_user_install_bins_to_path()
         importlib.invalidate_caches()
         try:
@@ -1396,6 +1461,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Image",
                 "target": "local",
             },
+            {
+                "name": "psd_tools",
+                "pip": "psd-tools",
+                "desc": "Open Photoshop PSD files and inspect flattened/layered image data",
+                "category": "Image",
+                "target": "local",
+            },
             # ── Tools ──
             {
                 "name": "playwright",
@@ -1404,6 +1476,31 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Tools",
                 "target": "local",
             },
+            {
+                "name": "office_docs",
+                "pip": "markitdown[docx,pptx,xlsx,xls]",
+                "desc": "Open Office attachments and documents (.docx, .pptx, .xlsx, .xls) as readable Markdown",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "pymupdf",
+                "pip": "PyMuPDF",
+                "desc": "Advanced PDF opening, rendering, forms, annotations, and signatures",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "libreoffice",
+                "pip": "",
+                "desc": "Convert DOCX attachments to signable PDF previews",
+                "category": "Tools",
+                "target": "local",
+                "kind": "system",
+                "system_prereqs": ["libreoffice"],
+                "install_cmd": "sudo apt install -y libreoffice || brew install --cask libreoffice",
+                "install_hint": "Install LibreOffice/soffice where Odysseus runs to open DOCX attachments as signable PDF previews. Without it, DOCX opens as readable Markdown.",
+            },
         ]
 
         # Most packages should not be installed through external means. Hence, set the default of the
@@ -1411,21 +1508,10 @@ def setup_shell_routes() -> APIRouter:
         for pkg in packages:
             pkg.setdefault("install_cmd", None)
             pkg.setdefault("update_cmd", None)
-        if not has_krea_model:
-            packages = [
-                p for p in packages
-                if p.get("name") not in {"krea_diffusers", "transformers"}
-            ]
-        if not has_lama_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_lama_swift"
-            ]
-        if not has_ddcolor_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_ddcolor_swift"
-            ]
+        # Keep the Image section complete. Dependency visibility is a product
+        # capability decision, not a substring test against a model id. Model
+        # catalogs may declare an explicit runtime package, while the generic
+        # backend preflight handles ordinary models.
         # Remote check: for remote-target packages, probe the selected server's
         # venv over SSH so a remote `pip install` actually reflects here.
         remote_status: dict = {}
@@ -1596,6 +1682,14 @@ def setup_shell_routes() -> APIRouter:
                         if IS_APPLE_SILICON
                         else "Requires a native Apple Silicon Mac with Apple Foundational Models support."
                     )
+                elif pkg["name"] == "libreoffice":
+                    soffice_path = shutil.which("soffice") or shutil.which("libreoffice")
+                    pkg["installed"] = soffice_path is not None
+                    pkg["status_note"] = (
+                        f"DOCX signable preview converter: {soffice_path}"
+                        if soffice_path
+                        else "DOCX signing preview needs LibreOffice/soffice."
+                    )
                 else:
                     pkg["installed"] = shutil.which(pkg["name"]) is not None
             elif pkg["name"] == "llama_cpp" and shutil.which("llama-server"):
@@ -1757,7 +1851,12 @@ def setup_shell_routes() -> APIRouter:
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint
-        return {"packages": packages}
+        result = {"packages": packages}
+        if len(_PACKAGE_STATUS_CACHE) >= _PACKAGE_STATUS_CACHE_MAX:
+            oldest_key = min(_PACKAGE_STATUS_CACHE, key=lambda key: _PACKAGE_STATUS_CACHE[key][0])
+            _PACKAGE_STATUS_CACHE.pop(oldest_key, None)
+        _PACKAGE_STATUS_CACHE[package_cache_key] = (time.monotonic(), result)
+        return result
 
     @router.post("/api/cookbook/packages/install")
     async def install_package(request: Request):
@@ -1802,6 +1901,7 @@ def setup_shell_routes() -> APIRouter:
             *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
         )
         stdout, stderr = await proc.communicate()
+        _PACKAGE_STATUS_CACHE.clear()
         if proc.returncode == 0:
             return {"ok": True, "output": stdout.decode()[-200:]}
         return {"ok": False, "error": stderr.decode()[-300:]}
@@ -1824,7 +1924,7 @@ def setup_shell_routes() -> APIRouter:
         ssh_port = body.get("ssh_port")
         # Names users can request — must match canonical names used in the
         # deps catalog's `system_prereqs` field and on the System rows.
-        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make"}
+        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make", "libreoffice"}
         pkgs = [str(p).strip() for p in raw if str(p).strip() in ALLOWED]
         if not pkgs:
             return {"ok": False, "error": "no installable packages requested (allowlist: " + ", ".join(sorted(ALLOWED)) + ")"}
@@ -1854,7 +1954,15 @@ def setup_shell_routes() -> APIRouter:
                 else: out.append(n)
             return out
         def _brew(names):
-            return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
+            out = []
+            for n in names:
+                if n in ("build-essential", "g++", "gcc", "make"):
+                    continue
+                if n == "libreoffice":
+                    out += ["--cask", "libreoffice"]
+                else:
+                    out.append(n)
+            return out
         # Build a single shell snippet that detects the package manager and
         # runs the right install. Non-interactive sudo (-n) only — if sudo
         # asks for a password the script reports it instead of hanging.
@@ -1920,6 +2028,7 @@ def setup_shell_routes() -> APIRouter:
             combined = err_txt or tail_out or f"exit code {proc.returncode}"
         else:
             combined = None
+        _PACKAGE_STATUS_CACHE.clear()
         return {
             "ok": ok,
             "exit_code": proc.returncode,

routes/task/task_routes.py

@@ -10,7 +10,7 @@ from typing import Optional, Dict, Any
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
-from core.database import SessionLocal, ScheduledTask, TaskRun
+from core.database import SessionLocal, ScheduledTask, TaskRun, NotificationLog
 from core.constants import internal_api_base
 from src.auth_helpers import get_current_user
 from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
@@ -569,6 +569,57 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         notes = task_scheduler.pop_notifications(owner=user)
         return {"notifications": notes}
 
+    @router.get("/notification-logs")
+    async def get_notification_logs(request: Request, limit: int = 200):
+        """Return persisted task notifications without consuming them."""
+        user = _owner(request)
+        if not user:
+            return {"notifications": []}
+        limit = max(1, min(int(limit or 200), 1000))
+        db = SessionLocal()
+        try:
+            rows = (db.query(NotificationLog)
+                    .filter(NotificationLog.owner == user)
+                    .order_by(NotificationLog.timestamp.desc())
+                    .limit(limit)
+                    .all())
+            return {"notifications": [
+                {
+                    "id": row.id,
+                    "task_name": row.task_name,
+                    "task_id": row.task_id,
+                    "status": row.status,
+                    "body": row.body,
+                    "timestamp": row.timestamp.isoformat() + "Z" if row.timestamp else None,
+                }
+                for row in rows
+            ]}
+        finally:
+            db.close()
+
+    @router.post("/notification-logs")
+    async def create_notification_log(request: Request):
+        """Persist an in-app toast so Settings can show notification history."""
+        user = _owner(request)
+        if not user:
+            raise HTTPException(401, "Authentication required")
+        body = await request.json()
+        message = str(body.get("body") or "").strip()[:2000]
+        if not message:
+            raise HTTPException(400, "Notification body required")
+        row = NotificationLog(
+            id=str(uuid.uuid4()), owner=user,
+            task_name=str(body.get("title") or "Odysseus")[:200],
+            status="error" if body.get("status") == "error" else "success",
+            body=message,
+        )
+        db = SessionLocal()
+        try:
+            db.add(row); db.commit()
+            return {"success": True}
+        finally:
+            db.close()
+
     @router.post("/{task_id}/clear-cache")
     async def clear_task_cache(request: Request, task_id: str):
         """Clear derived cache for one built-in task."""

routes/upload_routes.py

@@ -190,7 +190,8 @@ def setup_upload_routes(upload_handler):
             return None
         return session_id
 
-    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None,
+                                       gallery_id: str | None = None) -> str | None:
         """Make chat-uploaded images visible in Gallery without changing chat storage."""
         is_image_file = getattr(upload_handler, "is_image_file", None)
         if not callable(is_image_file):
@@ -205,6 +206,21 @@ def setup_upload_routes(upload_handler):
         db = SessionLocal()
         try:
             file_hash = meta.get("hash")
+            if gallery_id:
+                existing = db.query(GalleryImage).filter(
+                    GalleryImage.id == gallery_id,
+                    GalleryImage.is_active == True,  # noqa: E712
+                ).first()
+                if existing and (not owner or existing.owner == owner):
+                    image_dir = Path(GENERATED_IMAGES_DIR)
+                    image_dir.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(source_path, image_dir / existing.filename)
+                    existing.file_hash = file_hash
+                    existing.file_size = meta.get("size")
+                    existing.width = meta.get("width")
+                    existing.height = meta.get("height")
+                    db.commit()
+                    return existing.id
             if file_hash:
                 q = db.query(GalleryImage).filter(
                     GalleryImage.file_hash == file_hash,
@@ -259,6 +275,7 @@ def setup_upload_routes(upload_handler):
         request: Request,
         files: List[UploadFile] = File(...),
         session_id: Optional[str] = Form(None),
+        gallery_id: Optional[str] = Form(None),
     ):
         """Upload files with enhanced security and organization."""
         if not isinstance(session_id, str):
@@ -289,7 +306,7 @@ def setup_upload_routes(upload_handler):
             try:
                 owner = effective_user(request)
                 meta = upload_handler.save_upload(u, client_ip, owner=owner)
-                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
+                promoted_gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id, gallery_id)
                 item = {
                     "id": meta["id"],
                     "name": meta["name"],
@@ -303,8 +320,8 @@ def setup_upload_routes(upload_handler):
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
                 }
-                if gallery_id:
-                    item["gallery_id"] = gallery_id
+                if promoted_gallery_id:
+                    item["gallery_id"] = promoted_gallery_id
                 out.append(item)
             except HTTPException:
                 raise

routes/workspace_routes.py

@@ -82,4 +82,26 @@ def setup_workspace_routes():
         resolved = vet_workspace(path)
         return {"ok": resolved is not None, "path": resolved}
 
+    @router.get("/default")
+    def default_workspace(request: Request):
+        """Return the explicitly configured backend workspace, if usable.
+
+        WebUI has no local launch directory: it runs against this backend's
+        filesystem. An explicit default gives it the same zero-setup behavior
+        as TUI while keeping workspace access opt-in and server-vetted.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace default is admin-only")
+
+        configured = os.environ.get("ODYSSEUS_WORKSPACE_DEFAULT", "").strip()
+        if not configured:
+            return {"ok": False, "path": None}
+
+        from src.tool_execution import vet_workspace
+        from src.workspace_paths import backend_workspace_path
+
+        resolved = vet_workspace(backend_workspace_path(configured))
+        return {"ok": resolved is not None, "path": resolved}
+
     return router

scripts/add_hwfit_models.py

@@ -31,7 +31,43 @@ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
 DATA_PATH = os.path.join(os.path.dirname(__file__), "..", "services", "hwfit", "data", "hf_models.json")
 DATA_PATH = os.path.abspath(DATA_PATH)
 
-AUTHORS = ["cyankiwi"]
+# Official / major model-provider orgs to refresh into the Cookbook catalog.
+# Keep this broad enough that new first-party releases appear after running the
+# updater, while avoiding a global HF scan that would pull in every community fork.
+AUTHORS = [
+    # Community quant provider we already use for AWQ/FP8 serving recipes.
+    "cyankiwi",
+    # Major first-party model providers.
+    "Qwen",
+    "deepseek-ai",
+    "zai-org",
+    "MiniMaxAI",
+    "moonshotai",
+    "mistralai",
+    "meta-llama",
+    "google",
+    "google-deepmind",
+    "microsoft",
+    "nvidia",
+    "CohereLabs",
+    "ai21labs",
+    "Tencent-Hunyuan",
+    "ibm-granite",
+    "tiiuae",
+    "01-ai",
+    "allenai",
+    "HuggingFaceTB",
+    "openai",
+]
+BROAD_AUTHORS_SKIP_FALLBACK_PROBES = {
+    # These orgs have hundreds/thousands of mixed-purpose repos. For them,
+    # catalog only entries that can be sized from cheap list metadata / repo
+    # names; do not block refreshes on per-repo config/safetensors downloads.
+    "google",
+    "microsoft",
+    "nvidia",
+    "allenai",
+}
 # Specific repos to add (in addition to the authors above). Optional explicit
 # overrides {repo: {field: value}} for things the name/metadata can't convey.
 EXTRA_REPOS = {
@@ -50,6 +86,21 @@ _GENERIC_TAGS = {
     "quantized", "chat",
 }
 
+_GEN_MODEL_PIPELINES = {
+    "text-generation",
+    "text2text-generation",
+    "image-text-to-text",
+    "text-generation-inference",
+    "conversational",
+}
+
+_GEN_MODEL_KEYWORDS = (
+    "llama", "gemma", "qwen", "deepseek", "glm", "chatglm", "minimax",
+    "kimi", "moonshot", "mistral", "mixtral", "codestral", "ministral",
+    "phi", "mai", "nemotron", "granite", "command", "aya", "jamba",
+    "hunyuan", "yi-", "yi_", "falcon", "olmo", "openai",
+)
+
 api = HfApi()
 
 
@@ -207,6 +258,8 @@ def _quant_from_name(name):
     n = name.lower()
     if "nvfp4" in n:
         return "NVFP4"
+    if re.search(r"(^|[-_/])bf16($|[-_/])", n):
+        return "BF16"
     if "mxfp4" in n:
         return "MXFP4"
     if re.search(r"(^|[-_/])nf4($|[-_/])", n):
@@ -248,7 +301,7 @@ def _arch_from_tags(tags):
     return ""
 
 
-def _entry_from_modelinfo(mi, overrides):
+def _entry_from_modelinfo(mi, overrides, *, probe_config=True, probe_safetensors=True):
     name = mi.id
     provider = name.split("/")[0]
     total, active = _parse_params(name)
@@ -272,7 +325,7 @@ def _entry_from_modelinfo(mi, overrides):
     # before safetensors so non-standard names still resolve without a
     # per-repo manual override in EXTRA_REPOS. Source repo first (works for
     # unquantized models) then the quantized parent via base_model:.
-    if total is None:
+    if total is None and probe_config:
         config_targets = [name]
         bm = _base_model_tag(getattr(mi, "tags", None))
         if bm and bm != name:
@@ -293,7 +346,7 @@ def _entry_from_modelinfo(mi, overrides):
     # therefore undercounts real parameter count by the same factor, which
     # then feeds a wrong `min_vram_gb` downstream. Sum per-dtype and unpack
     # the packed I32 tensors so the catalog stores the true param count.
-    if total is None:
+    if total is None and probe_safetensors:
         try:
             full = api.model_info(name, files_metadata=False)
             st = getattr(full, "safetensors", None)
@@ -322,7 +375,8 @@ def _entry_from_modelinfo(mi, overrides):
     created = getattr(mi, "created_at", None)
     rel = created.strftime("%Y-%m-%d") if created else datetime.utcnow().strftime("%Y-%m-%d")
     # Rough RAM/VRAM hints (fit.py recomputes the real requirement from params+quant).
-    _BPP = {"AWQ-4bit": 0.58, "GPTQ-Int4": 0.58, "mlx-4bit": 0.55, "mlx-6bit": 0.85,
+    _BPP = {"F16": 2.0, "BF16": 2.0,
+            "AWQ-4bit": 0.58, "GPTQ-Int4": 0.58, "mlx-4bit": 0.55, "mlx-6bit": 0.85,
             "AWQ-8bit": 1.1, "GPTQ-Int8": 1.1, "mlx-8bit": 1.1, "FP8": 1.1,
             "FP4": 0.58, "NVFP4": 0.58, "MXFP4": 0.58, "NF4": 0.58,
             "INT4": 0.58, "INT8": 1.1, "W4A16": 0.58, "W8A8": 1.1, "W8A16": 1.1,
@@ -360,6 +414,28 @@ def _entry_from_modelinfo(mi, overrides):
     return entry
 
 
+def _is_likely_catalog_model(mi):
+    """Cheap prefilter before config/safetensors probes.
+
+    Major HF orgs include thousands of encoder, CV, audio, adapter, and demo
+    repos. Cookbook's serve catalog is for generative models, so only do the
+    expensive config/model_info fallback for repos that already look relevant
+    from list_models(full=True) metadata.
+    """
+    name = str(getattr(mi, "id", "") or "")
+    if not name:
+        return False
+    # Size-bearing model names are usually exactly what we want (7B, 70B, A3B).
+    if _parse_params(name)[0]:
+        return True
+    pipeline = str(getattr(mi, "pipeline_tag", "") or "").lower()
+    if pipeline in _GEN_MODEL_PIPELINES:
+        return True
+    tags = " ".join(str(t).lower() for t in (getattr(mi, "tags", None) or []))
+    haystack = f"{name.lower()} {pipeline} {tags}"
+    return any(k in haystack for k in _GEN_MODEL_KEYWORDS)
+
+
 def main():
     with open(DATA_PATH, encoding="utf-8") as f:
         catalog = json.load(f)
@@ -377,8 +453,16 @@ def main():
         for mi in models:
             if mi.id in existing and not overwrite:
                 continue
+            if not _is_likely_catalog_model(mi):
+                continue
             ov = EXTRA_REPOS.get(mi.id)
-            entry = _entry_from_modelinfo(mi, ov)
+            skip_fallbacks = author in BROAD_AUTHORS_SKIP_FALLBACK_PROBES
+            entry = _entry_from_modelinfo(
+                mi,
+                ov,
+                probe_config=not skip_fallbacks,
+                probe_safetensors=not skip_fallbacks,
+            )
             if entry:
                 to_add[mi.id] = entry
 

scripts/analyze_odysseus_eval_targets.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Rank next Odysseus tool-router improvement targets from eval artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def _load(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def _metric(record: dict[str, Any], key: str, default: Any = None) -> Any:
+    metrics = record.get("metrics") or {}
+    return metrics.get(key, default)
+
+
+def _tool_rounds(record: dict[str, Any]) -> int:
+    metrics = record.get("metrics") or {}
+    usage = metrics.get("usage_buckets") or []
+    round_models = metrics.get("round_models") or []
+    if usage:
+        return len(usage)
+    if round_models:
+        return len(round_models)
+    snapshots = record.get("model_request_snapshots") or []
+    if snapshots:
+        return len(snapshots)
+    return 0
+
+
+def _is_infra_failure_error(error: dict[str, Any]) -> bool:
+    if not isinstance(error, dict):
+        return False
+    status = error.get("status")
+    text = " ".join(
+        str(error.get(key) or "")
+        for key in ("error", "message", "detail", "type")
+    ).lower()
+    if status in {502, 503, 504, 520, 521, 522, 523, 524}:
+        return True
+    return bool(
+        "cannot reach" in text
+        or "connection refused" in text
+        or "connection reset" in text
+        or "connect timeout" in text
+        or "read timeout" in text
+        or "unreachable" in text
+        or "cooldown active" in text
+        or "upstream protocol error" in text
+        or ("upstream" in text and "failed" in text)
+    )
+
+
+def _record_has_infra_error(record: dict[str, Any]) -> bool:
+    if record.get("infra_failure") is True:
+        return True
+    errors = list(record.get("stream_errors") or [])
+    stream_exception = record.get("stream_exception")
+    if isinstance(stream_exception, dict):
+        errors.append(stream_exception)
+    return any(_is_infra_failure_error(error) for error in errors)
+
+
+def _record_status(record: dict[str, Any]) -> str:
+    if _record_has_infra_error(record):
+        return "infra"
+    if not record.get("native_call_ok"):
+        return "routing"
+    if not record.get("command_contract_ok"):
+        return "contract"
+    if not record.get("tool_invocation_ok"):
+        return "invocation"
+    if not record.get("command_outcome_ok"):
+        return "outcome"
+    if not record.get("response_quality_ok"):
+        return "response"
+    if record.get("duplicate_textual_call"):
+        return "duplicate_text"
+    if record.get("repetitive_tool_call"):
+        return "repeat"
+    return "pass"
+
+
+def _first_output(record: dict[str, Any]) -> dict[str, Any]:
+    outputs = record.get("tool_outputs") or []
+    return outputs[0] if outputs else {}
+
+
+def _print_row(record: dict[str, Any]) -> None:
+    case = record.get("case")
+    status = _record_status(record)
+    first_tool = record.get("first_tool")
+    expected = record.get("expected_tool")
+    output = _first_output(record)
+    input_tokens = _metric(record, "input_tokens")
+    response_time = _metric(record, "response_time")
+    elapsed = record.get("elapsed_seconds")
+    rounds = _tool_rounds(record)
+    exit_code = output.get("exit_code")
+    print(
+        f"- {case}: status={status}, expected={expected}, first={first_tool}, "
+        f"rounds={rounds}, input={input_tokens}, response={response_time}s, "
+        f"elapsed={elapsed}s, exit={exit_code}"
+    )
+
+
+def _top(records: list[dict[str, Any]], key, limit: int) -> list[dict[str, Any]]:
+    return sorted(records, key=key, reverse=True)[:limit]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("artifact", type=Path)
+    parser.add_argument("--limit", type=int, default=12)
+    args = parser.parse_args()
+
+    artifact = _load(args.artifact)
+    records = list(artifact.get("records") or [])
+    infra = [record for record in records if _record_has_infra_error(record)]
+    evaluable = [record for record in records if not _record_has_infra_error(record)]
+    failed = [record for record in evaluable if _record_status(record) != "pass"]
+    slow = _top(
+        [record for record in evaluable if _metric(record, "response_time") is not None],
+        lambda record: float(_metric(record, "response_time", 0) or 0),
+        args.limit,
+    )
+    token_heavy = _top(
+        [record for record in evaluable if _metric(record, "input_tokens") is not None],
+        lambda record: int(_metric(record, "input_tokens", 0) or 0),
+        args.limit,
+    )
+    multi_round = _top(
+        [record for record in evaluable if _tool_rounds(record) > 1],
+        lambda record: (_tool_rounds(record), float(_metric(record, "response_time", 0) or 0)),
+        args.limit,
+    )
+
+    print(f"artifact: {args.artifact}")
+    print(f"model: {artifact.get('model')}")
+    print(f"cases: {artifact.get('cases', len(records))}")
+    print(f"infra: {len(infra)}")
+    print(f"evaluable: {len(evaluable)}")
+    print(f"failures: {len(failed)}")
+    print()
+
+    print("failures:")
+    if failed:
+        for record in failed:
+            _print_row(record)
+    else:
+        print("- none")
+    print()
+
+    print(f"slowest_{len(slow)}:")
+    for record in slow:
+        _print_row(record)
+    print()
+
+    print(f"token_heaviest_{len(token_heavy)}:")
+    for record in token_heavy:
+        _print_row(record)
+    print()
+
+    print(f"multi_round_{len(multi_round)}:")
+    if multi_round:
+        for record in multi_round:
+            _print_row(record)
+    else:
+        print("- none")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/assemble_sft_clean_corpus.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Assemble kept and validated repaired sessions into a clean SFT corpus."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--trace", type=Path, required=True)
+    parser.add_argument("--verdicts", type=Path, required=True)
+    parser.add_argument("--repairs", type=Path, action="append", default=[])
+    parser.add_argument("--out-trace", type=Path, required=True)
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+
+    source: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in load_jsonl(args.trace):
+        source[str(row.get("session_id") or "")].append(row)
+    verdicts = {str(row.get("session_id") or ""): row for row in load_jsonl(args.verdicts)}
+    repaired: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for path in args.repairs:
+        for row in load_jsonl(path):
+            repaired[str(row.get("session_id") or "")].append(row)
+
+    output: list[dict[str, Any]] = []
+    excluded: list[dict[str, Any]] = []
+    counts: Counter[str] = Counter()
+    for session_id in sorted(source):
+        verdict = verdicts.get(session_id)
+        decision = str((verdict or {}).get("verdict") or "missing")
+        if decision == "keep":
+            output.extend(source[session_id])
+            counts["kept"] += 1
+        elif decision == "repair" and repaired.get(session_id):
+            output.extend(repaired[session_id])
+            counts["repaired"] += 1
+        else:
+            counts["excluded"] += 1
+            excluded.append({
+                "session_id": session_id,
+                "verdict": decision,
+                "issues": (verdict or {}).get("issues") or [],
+                "repair_missing": decision == "repair" and session_id not in repaired,
+            })
+
+    args.out_trace.parent.mkdir(parents=True, exist_ok=True)
+    args.out_trace.write_text(
+        "\n".join(json.dumps(row, ensure_ascii=False) for row in output) + ("\n" if output else ""),
+        encoding="utf-8",
+    )
+    report = {
+        "source_sessions": len(source),
+        "output_sessions": counts["kept"] + counts["repaired"],
+        "output_turns": len(output),
+        "decisions": dict(counts),
+        "excluded": excluded,
+    }
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({key: value for key, value in report.items() if key != "excluded"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_email_sft_with_deepseek.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+"""Audit recent Odysseus email SFT conversations with a DeepSeek judge."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sqlite3
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DB = ROOT / "data" / "app.db"
+OUT_DIR = ROOT / "data" / "audits"
+
+
+EMAIL_RE = re.compile(
+    r"\b(email|emails|inbox|mailbox|attachment|attachments|draft|reply|archive|"
+    r"delete|spam|blocked|unblock|read|unread|favorite|done|contact)\b",
+    re.I,
+)
+
+
+def decrypt_secret(value: str) -> str:
+    if not value or not value.startswith("enc:"):
+        return value or ""
+    from cryptography.fernet import Fernet
+
+    key = (ROOT / "data" / ".app_key").read_bytes()
+    return Fernet(key).decrypt(value[len("enc:") :].encode("ascii")).decode("utf-8")
+
+
+def db() -> sqlite3.Connection:
+    con = sqlite3.connect(DB)
+    con.row_factory = sqlite3.Row
+    return con
+
+
+def deepseek_endpoint(con: sqlite3.Connection, endpoint_id: str | None = None, model: str | None = None) -> dict[str, str]:
+    if endpoint_id:
+        row = con.execute(
+            """
+            SELECT id, name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE id = ?
+              AND COALESCE(api_key, '') != ''
+            """,
+            (endpoint_id,),
+        ).fetchone()
+    else:
+        row = con.execute(
+        """
+        SELECT id, name, base_url, api_key, cached_models
+        FROM model_endpoints
+        WHERE is_enabled = 1
+          AND COALESCE(api_key, '') != ''
+          AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+        ORDER BY CASE WHEN lower(name) = 'deepseek' THEN 0 ELSE 1 END
+        LIMIT 1
+        """
+    ).fetchone()
+    if row is None:
+        raise RuntimeError("No enabled DeepSeek endpoint with an API key found in model_endpoints")
+    models = json.loads(row["cached_models"] or "[]")
+    selected = model or (models[0] if models else "deepseek-chat")
+    return {
+        "id": row["id"],
+        "name": row["name"],
+        "base_url": row["base_url"],
+        "api_key": decrypt_secret(row["api_key"] or ""),
+        "model": selected,
+    }
+
+
+def compact_tool_event(ev: dict[str, Any]) -> dict[str, Any]:
+    out = str(ev.get("output") or "")
+    return {
+        "tool": ev.get("tool"),
+        "command": ev.get("command"),
+        "output": out[:1200] + ("..." if len(out) > 1200 else ""),
+        "exit_code": ev.get("exit_code"),
+    }
+
+
+def session_payload(con: sqlite3.Connection, sid: str) -> dict[str, Any]:
+    s = con.execute(
+        "SELECT id, name, created_at, updated_at, message_count FROM sessions WHERE id = ?",
+        (sid,),
+    ).fetchone()
+    messages = []
+    for m in con.execute(
+        "SELECT role, content, metadata, timestamp FROM chat_messages WHERE session_id = ? ORDER BY timestamp, id",
+        (sid,),
+    ):
+        meta: dict[str, Any] = {}
+        if m["metadata"]:
+            try:
+                meta = json.loads(m["metadata"])
+            except json.JSONDecodeError:
+                meta = {}
+        content = m["content"] or ""
+        thinking = meta.get("thinking")
+        if isinstance(thinking, str) and len(thinking) > 1000:
+            thinking = thinking[:1000] + "..."
+        messages.append(
+            {
+                "role": m["role"],
+                "timestamp": m["timestamp"],
+                "content": content[:2500] + ("..." if len(content) > 2500 else ""),
+                "thinking": thinking,
+                "tool_events": [compact_tool_event(ev) for ev in meta.get("tool_events") or []],
+            }
+        )
+    docs = []
+    for d in con.execute(
+        """
+        SELECT id, title, language, current_content, source_email_uid, updated_at
+        FROM documents
+        WHERE session_id = ?
+        ORDER BY updated_at DESC
+        LIMIT 3
+        """,
+        (sid,),
+    ):
+        content = d["current_content"] or ""
+        docs.append(
+            {
+                "id": d["id"],
+                "title": d["title"],
+                "language": d["language"],
+                "source_email_uid": d["source_email_uid"],
+                "content": content[:1800] + ("..." if len(content) > 1800 else ""),
+            }
+        )
+    return {
+        "session": dict(s),
+        "messages": messages,
+        "open_documents": docs,
+    }
+
+
+def recent_email_sessions(con: sqlite3.Connection, owner: str, limit: int) -> list[str]:
+    rows = con.execute(
+        """
+        SELECT id
+        FROM sessions
+        WHERE owner = ?
+        ORDER BY updated_at DESC
+        LIMIT ?
+        """,
+        (owner, limit),
+    ).fetchall()
+    keep = []
+    for row in rows:
+        text = "\n".join(
+            r["content"] or ""
+            for r in con.execute("SELECT content FROM chat_messages WHERE session_id = ?", (row["id"],))
+        )
+        tools = "\n".join(
+            r["metadata"] or ""
+            for r in con.execute("SELECT metadata FROM chat_messages WHERE session_id = ?", (row["id"],))
+        )
+        if EMAIL_RE.search(text) or "mcp__email" in tools or "list_email" in tools:
+            keep.append(row["id"])
+    return keep
+
+
+def session_ids_from_results(path: Path) -> list[str]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    rows = payload.get("results") if isinstance(payload, dict) else payload
+    if not isinstance(rows, list):
+        raise RuntimeError(f"Expected results list in {path}")
+    out: list[str] = []
+    for row in rows:
+        sid = str(row.get("session_id") or "").strip()
+        if sid and sid not in out:
+            out.append(sid)
+    return out
+
+
+def judge_prompt(batch: list[dict[str, Any]]) -> list[dict[str, str]]:
+    system = """You are auditing Odysseus email-agent conversations for SFT training quality.
+Return strict JSON only: {"results":[...]}.
+For every session, decide and copy back `session_id` and `session_name` from `session`.
+- verdict: keep, repair, or delete.
+- trainable_score: 0-100.
+- issues: short strings.
+- repairs: concrete edits needed, or [].
+- date_risk: none, low, medium, high.
+- thinking_trace_risk: none, low, medium, high.
+- rationale: one concise sentence.
+
+Important audit rules:
+- Keep only traces where user intent, tool calls, tool outputs, and final answer align.
+- Repair/delete if assistant claimed an email action without a corresponding tool event.
+- Repair/delete if it says tools are unavailable when email tools were actually needed/available.
+- Repair/delete repeated resend/stale-loop traces unless the bad branch is removed.
+- Repair/delete visible raw harness dumps, unpolished tool output, or synthetic/fake/SFT leaks in assistant/user message `content`.
+- Do not penalize raw text inside `tool_events.output` by itself. Tool outputs are allowed to be raw; only flag them when the assistant-facing final content also exposed the dump or when the tool result is semantically wrong.
+- Date-relative tasks are safe only if the trace includes a clear current date/timezone context or a tool query using explicit date bounds. Otherwise flag date_risk.
+- Thinking traces are usable only if they reflect correct tool choice and do not mention fake fixtures, harness bugs, stale injected data, or false tool unavailability.
+- Multi-intent user requests must satisfy all parts or be repair/delete.
+- Be strict: these are for training a model, not UI QA."""
+    user = json.dumps({"current_date": "2026-08-24", "timezone": "UTC", "sessions": batch}, ensure_ascii=False)
+    return [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+
+def call_judge(endpoint: dict[str, str], batch: list[dict[str, Any]]) -> dict[str, Any]:
+    payload = {
+        "model": endpoint["model"],
+        "messages": judge_prompt(batch),
+        "temperature": 0,
+        "max_tokens": 3500,
+        "response_format": {"type": "json_object"},
+    }
+    req = urllib.request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {endpoint['api_key']}",
+        },
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=75) as resp:
+        data = json.loads(resp.read().decode("utf-8"))
+    content = data["choices"][0]["message"]["content"]
+    if not isinstance(content, str) or not content.strip():
+        raise ValueError("Judge returned empty message content")
+    return json.loads(content)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--owner", default="sft_alex_creator")
+    ap.add_argument("--limit", type=int, default=140)
+    ap.add_argument("--batch-size", type=int, default=5)
+    ap.add_argument("--sleep", type=float, default=0.4)
+    ap.add_argument("--endpoint-id")
+    ap.add_argument("--model")
+    ap.add_argument("--results-file", type=Path, default=None, help="Audit exact session_ids from an overseer/eval actual_results.json")
+    args = ap.parse_args()
+
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    con = db()
+    endpoint = deepseek_endpoint(con, endpoint_id=args.endpoint_id, model=args.model)
+    if args.results_file:
+        sids = session_ids_from_results(args.results_file)
+    else:
+        sids = recent_email_sessions(con, args.owner, args.limit)
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    out_jsonl = OUT_DIR / f"email_sft_deepseek_audit_{args.owner}_{stamp}.jsonl"
+    out_md = OUT_DIR / f"email_sft_deepseek_audit_{args.owner}_{stamp}.md"
+
+    all_results: list[dict[str, Any]] = []
+    for i in range(0, len(sids), args.batch_size):
+        batch_sids = sids[i : i + args.batch_size]
+        batch = [session_payload(con, sid) for sid in batch_sids]
+        for attempt in range(3):
+            try:
+                judged = call_judge(endpoint, batch)
+                break
+            except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError, TypeError, ValueError) as exc:
+                if attempt == 2:
+                    raise
+                time.sleep(2 + attempt * 3)
+        results = judged.get("results", [])
+        for j, result in enumerate(results):
+            if j < len(batch):
+                result.setdefault("session_id", batch[j]["session"]["id"])
+                result.setdefault("session_name", batch[j]["session"]["name"])
+        with out_jsonl.open("a", encoding="utf-8") as f:
+            for result in results:
+                f.write(json.dumps(result, ensure_ascii=False) + "\n")
+        all_results.extend(results)
+        print(f"judged {min(i + args.batch_size, len(sids))}/{len(sids)}")
+        time.sleep(args.sleep)
+
+    counts: dict[str, int] = {}
+    for r in all_results:
+        counts[r.get("verdict", "unknown")] = counts.get(r.get("verdict", "unknown"), 0) + 1
+
+    lines = [
+        f"# Email SFT DeepSeek Audit: {args.owner}",
+        "",
+        f"- Sessions judged: {len(all_results)}",
+        f"- Source recent limit: {args.limit}",
+        f"- Endpoint: {endpoint.get('name')} ({endpoint.get('id')})",
+        f"- Model: {endpoint['model']}",
+        f"- Verdict counts: {json.dumps(counts, sort_keys=True)}",
+        "",
+        "## Repair/Delete Queue",
+        "",
+    ]
+    for r in all_results:
+        if r.get("verdict") == "keep":
+            continue
+        sid = r.get("session_id") or r.get("id") or r.get("session", {}).get("id")
+        name = r.get("session_name") or r.get("name") or ""
+        issues = ", ".join(r.get("issues") or [])
+        repairs = "; ".join(
+            item if isinstance(item, str) else json.dumps(item, ensure_ascii=False, sort_keys=True)
+            for item in (r.get("repairs") or [])
+        )
+        lines.append(f"- `{sid}` {name} -- **{r.get('verdict')}** score={r.get('trainable_score')} issues={issues} repairs={repairs}")
+    lines.extend(["", "## Keep Candidates", ""])
+    for r in all_results:
+        if r.get("verdict") != "keep":
+            continue
+        sid = r.get("session_id") or r.get("id") or r.get("session", {}).get("id")
+        name = r.get("session_name") or r.get("name") or ""
+        lines.append(f"- `{sid}` {name} -- score={r.get('trainable_score')} date={r.get('date_risk')} thinking={r.get('thinking_trace_risk')}")
+    out_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"jsonl={out_jsonl}")
+    print(f"markdown={out_md}")
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_historical_tool_routing.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Audit current capability routing against recorded historical tool turns.
+
+This is intentionally read-only: it never creates sessions or executes tools.
+Recorded assistant tool events provide the expected families; the current turn
+contract is evaluated with the original preceding conversation as history.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+from collections import Counter
+from pathlib import Path
+
+from src.turn_contract import FAMILY_TOOLS, canonical_tool, requested_capabilities
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_DB = Path("/home/pewds/odysseus-cookbook-fresh/data/app.db")
+DEFAULT_ANCHOR = "a37dcb3b-6864-4266-a115-f9e87aafd0eb"
+
+
+def tool_family(tool: str, command: object) -> set[str]:
+    name = canonical_tool(tool)
+    families = {family for family, tools in FAMILY_TOOLS.items() if name in tools}
+    # ui_control is a rendering/action bridge. Its command identifies the
+    # product family; do not label every such turn as the generic UI family.
+    if name == "ui_control":
+        text = str(command or "").lower()
+        if "email" in text:
+            return {"email"}
+        if "calendar" in text or "event" in text:
+            return {"calendar"}
+        if "note" in text:
+            return {"notes"}
+        if "document" in text or "editor" in text:
+            return {"documents"}
+    return families
+
+
+def metadata_tools(raw: str | None) -> set[str]:
+    try:
+        metadata = json.loads(raw or "{}")
+    except (TypeError, json.JSONDecodeError):
+        return set()
+    expected: set[str] = set()
+    for event in metadata.get("tool_events") or []:
+        expected.update(tool_family(event.get("tool", ""), event.get("command")))
+    return expected
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--db", type=Path, default=DEFAULT_DB)
+    parser.add_argument("--anchor", default=DEFAULT_ANCHOR)
+    parser.add_argument("--owner", default="sft_alex_creator")
+    parser.add_argument("--out", type=Path, default=ROOT / "reports/historical-routing-audit.json")
+    args = parser.parse_args()
+
+    con = sqlite3.connect(args.db)
+    con.row_factory = sqlite3.Row
+    anchor = con.execute("SELECT created_at FROM sessions WHERE id = ?", (args.anchor,)).fetchone()
+    if anchor is None:
+        raise SystemExit(f"Anchor session not found: {args.anchor}")
+    sessions = con.execute(
+        "SELECT id, name, created_at FROM sessions WHERE owner = ? AND created_at >= ? "
+        "ORDER BY created_at, id", (args.owner, anchor[0])
+    ).fetchall()
+
+    rows: list[dict] = []
+    seen: set[tuple] = set()
+    for session in sessions:
+        messages = con.execute(
+            "SELECT id, role, content, metadata, timestamp FROM chat_messages "
+            "WHERE session_id = ? ORDER BY timestamp, id", (session["id"],)
+        ).fetchall()
+        history: list[dict[str, str]] = []
+        for index, message in enumerate(messages):
+            role, content = message["role"], message["content"]
+            if role != "user":
+                history.append({"role": role, "content": content})
+                continue
+            following = next((m for m in messages[index + 1:] if m["role"] == "assistant"), None)
+            expected = metadata_tools(following["metadata"] if following else None)
+            if not expected:
+                history.append({"role": role, "content": content})
+                continue
+            key = (tuple((h["role"], h["content"].strip().lower()) for h in history), content.strip().lower(), tuple(sorted(expected)))
+            if key in seen:
+                history.append({"role": role, "content": content})
+                continue
+            seen.add(key)
+            actual = set(requested_capabilities(content, history))
+            missing = expected - actual
+            rows.append({
+                "session_id": session["id"], "session_name": session["name"],
+                "message_id": message["id"], "prompt": content,
+                "expected": sorted(expected), "actual": sorted(actual),
+                "missing": sorted(missing), "passed": not missing,
+            })
+            history.append({"role": role, "content": content})
+
+    failures = [row for row in rows if not row["passed"]]
+    report = {
+        "source_db": str(args.db), "anchor": args.anchor, "owner": args.owner,
+        "sessions_scanned": len(sessions), "labeled_unique_turns": len(rows),
+        "passed": len(rows) - len(failures), "failed": len(failures),
+        "accuracy": round((len(rows) - len(failures)) / len(rows), 6) if rows else None,
+        "missing_family_counts": dict(sorted(Counter(f for row in failures for f in row["missing"]).items())),
+        "failures": failures,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    print(json.dumps({key: report[key] for key in ("sessions_scanned", "labeled_unique_turns", "passed", "failed", "accuracy", "missing_family_counts")}, indent=2))
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/audit_search_pipeline.py

@@ -0,0 +1,53 @@
+"""Read-only, reproducible provider probe. Prints JSON; never changes settings.
+
+Run with the application's Python from the repository root. Queries are public
+regressions plus unrelated controls. Coverage is diagnostic, not an accuracy score.
+"""
+import concurrent.futures
+import json
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import httpx
+from services.search.providers import _get_search_instance, _safesearch_for
+
+QUERIES = [
+    "What country has best meat",
+    "Sweden 78 year old British woman deportation Brexit residence application",
+    "Latest news in AI",
+    "Any latest info on quantum physics",
+    "What year did Ethiopia become independent",
+    "PostgreSQL transaction isolation documentation",
+    "Kyoto weather tomorrow",
+]
+ENGINES = ["bing", "mojeek", "presearch", "duckduckgo", "google", "bing news", "yep"]
+
+
+def probe(pair):
+    query, engine = pair
+    start = time.monotonic()
+    try:
+        response = httpx.get(
+            _get_search_instance() + "/search",
+            params={"q": query, "engines": engine, "format": "json",
+                    "language": "en", "safesearch": _safesearch_for("searxng")},
+            timeout=20,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return {"query": query, "engine": engine,
+                "seconds": round(time.monotonic() - start, 2),
+                "unresponsive": data.get("unresponsive_engines", []),
+                "results": [{k: row.get(k) for k in (
+                    "title", "url", "content", "engines", "publishedDate"
+                )} for row in data.get("results", [])[:5]]}
+    except Exception as exc:
+        return {"query": query, "engine": engine, "error": type(exc).__name__}
+
+
+if __name__ == "__main__":
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+        rows = list(pool.map(probe, [(q, e) for q in QUERIES for e in ENGINES]))
+    print(json.dumps(rows, ensure_ascii=False, indent=2))

scripts/audit_sft_corpus_with_deepseek.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""Audit an Odysseus SFT JSONL corpus and use DeepSeek for semantic review."""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import concurrent.futures
+import hashlib
+import json
+import random
+import re
+import sqlite3
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_TRACE = ROOT / "data" / "sft_traces" / "sft_alex_creator.jsonl"
+OUT_DIR = ROOT / "data" / "audits"
+
+LEAK_RE = re.compile(
+    r"fake-(?:sender|odysseus)|synthetic (?:sft|fixture)|safe for training|"
+    r"training traces?|you are a fish|prompt injection|harness (?:bug|issue|dump)",
+    re.I,
+)
+UNAVAILABLE_RE = re.compile(
+    r"(?:i (?:do not|don.t|cannot|can.t)|there(?: is|'s) no) .{0,55}"
+    r"(?:tool|access|email|calendar|memory|document|browser|shell)",
+    re.I,
+)
+RAW_DUMP_RE = re.compile(r"Here are your (?:emails|events) \(\d+\):", re.I)
+FAILURE_RE = re.compile(
+    r"(?:permission denied|requires? .{0,30}(?:dependency|package)|not configured|"
+    r"tool calls? failed|internal server error|traceback|timed out)",
+    re.I,
+)
+
+
+def decrypt_secret(value: str) -> str:
+    if not value or not value.startswith("enc:"):
+        return value or ""
+    from cryptography.fernet import Fernet
+
+    key = (ROOT / "data" / ".app_key").read_bytes()
+    return Fernet(key).decrypt(value[4:].encode("ascii")).decode("utf-8")
+
+
+def deepseek_endpoint(endpoint_id: str | None, model: str | None) -> dict[str, str]:
+    con = sqlite3.connect(ROOT / "data" / "app.db")
+    con.row_factory = sqlite3.Row
+    if endpoint_id:
+        row = con.execute(
+            "SELECT * FROM model_endpoints WHERE id=? AND COALESCE(api_key,'') != ''",
+            (endpoint_id,),
+        ).fetchone()
+    else:
+        row = con.execute(
+            """SELECT * FROM model_endpoints
+               WHERE is_enabled=1 AND COALESCE(api_key,'') != ''
+                 AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+               ORDER BY CASE WHEN lower(name)='deepseek' THEN 0 ELSE 1 END LIMIT 1"""
+        ).fetchone()
+    if row is None:
+        raise RuntimeError("No enabled DeepSeek endpoint with an API key")
+    models = json.loads(row["cached_models"] or "[]")
+    return {
+        "id": row["id"],
+        "name": row["name"],
+        "base_url": row["base_url"],
+        "api_key": decrypt_secret(row["api_key"]),
+        "model": model or (models[0] if models else "deepseek-chat"),
+    }
+
+
+def load_rows(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            rows.append({"_invalid_line": line_no, "_error": str(exc), "_raw": line[:500]})
+            continue
+        row["_line"] = line_no
+        rows.append(row)
+    return rows
+
+
+def live_session_ids(owner: str) -> set[str]:
+    con = sqlite3.connect(ROOT / "data" / "app.db")
+    try:
+        return {str(row[0]) for row in con.execute("SELECT id FROM sessions WHERE owner = ?", (owner,))}
+    finally:
+        con.close()
+
+
+def row_flags(row: dict[str, Any]) -> list[str]:
+    if "_invalid_line" in row:
+        return ["invalid_json"]
+    flags = []
+    user = str(row.get("user") or "")
+    assistant = str(row.get("assistant") or "")
+    thinking = str(row.get("thinking") or "")
+    visible = "\n".join((user, assistant, thinking))
+    events = row.get("tool_events") or []
+    if not user.strip() or not assistant.strip():
+        flags.append("missing_user_or_assistant")
+    if LEAK_RE.search(visible):
+        flags.append("fixture_or_harness_leak")
+    if UNAVAILABLE_RE.search(assistant):
+        flags.append("possible_false_tool_unavailability")
+    if RAW_DUMP_RE.search(assistant):
+        flags.append("raw_harness_style_answer")
+    if any(FAILURE_RE.search(str(ev.get("output") or "")) for ev in events):
+        flags.append("tool_failure_present")
+    if events and not assistant.strip():
+        flags.append("tool_call_without_final_answer")
+    if len(row.get("round_texts") or []) > 2:
+        nonempty = [str(x).strip() for x in row.get("round_texts") or [] if str(x).strip()]
+        if len(nonempty) > 1 and len(set(nonempty)) < len(nonempty):
+            flags.append("repeated_round_text")
+    return flags
+
+
+def compact_row(row: dict[str, Any]) -> dict[str, Any]:
+    def clip(value: Any, size: int) -> str:
+        text = str(value or "")
+        return text[:size] + ("..." if len(text) > size else "")
+
+    return {
+        "line": row.get("_line"),
+        "message_id": row.get("message_id"),
+        "user": clip(row.get("user"), 1200),
+        "assistant": clip(row.get("assistant"), 2200),
+        "thinking": clip(row.get("thinking"), 1600),
+        "flags": row_flags(row),
+        "tools": [
+            {
+                "tool": ev.get("tool"),
+                "command": clip(ev.get("command"), 700),
+                "output": clip(ev.get("output"), 1100),
+                "exit_code": ev.get("exit_code"),
+            }
+            for ev in (row.get("tool_events") or [])
+        ],
+    }
+
+
+def _parse_json_message(message: dict[str, Any]) -> dict[str, Any]:
+    content = str(message.get("content") or message.get("reasoning_content") or "").strip()
+    content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content, flags=re.I | re.S).strip()
+    if not content.startswith("{"):
+        match = re.search(r"\{.*\}", content, flags=re.S)
+        if match:
+            content = match.group(0)
+    if not content:
+        raise ValueError("DeepSeek returned empty content and reasoning_content")
+    return json.loads(content)
+
+
+def judge(endpoint: dict[str, str], sessions: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    system = """You are a strict SFT corpus auditor for a general tool-using agent.
+Return JSON only as {"results":[...]}. Return exactly one result per session.
+Each result: session_id, verdict (keep|repair|delete), score (0-100), issues (strings), repairs (specific strings), and coverage_notes.
+
+Judge the complete behavior and whether the response is a good speaking-style target. Keep only when intent, reasoning, tool selection, arguments, tool outputs, state changes, follow-ups, and final answers agree, and the visible answer is concise, natural, and synthesized for the user. Repair means a coherent trace can be fixed by removing/replacing specific turns or text. Delete means the trajectory teaches a materially wrong strategy or is too corrupted.
+
+Flag false tool-unavailability claims, repeated answers/turns, stale resend branches, missing requested actions, success claims without successful tool evidence, malformed tool arguments, raw harness dumps presented as the answer, fixture/SFT/harness/prompt-injection discussion, incorrect relative dates/timezones, unsafe destructive actions, needless tools, tool loops, and thinking that contradicts the final action. Also mark repair when the final answer mechanically echoes tool output, repeats metadata the user did not request, narrates internal routing, asks needless follow-up questions, or is substantially more verbose than needed. A failed tool call is acceptable only when the assistant handles it correctly and does not teach a bad workaround. Do not penalize raw formatting that exists only inside tool output. For multi-intent prompts, every requested part must be handled. Be conservative because these traces train both tool strategy and response style."""
+    payload = {
+        "model": endpoint["model"],
+        "messages": [
+            {"role": "system", "content": system},
+            {"role": "user", "content": json.dumps({"audit_date": "2026-08-30", "timezone": "UTC", "sessions": sessions}, ensure_ascii=False)},
+        ],
+        "temperature": 0,
+        "max_tokens": 12000,
+        "response_format": {"type": "json_object"},
+    }
+    req = urllib.request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=120) as response:
+        result = json.loads(response.read().decode())
+    results = _parse_json_message(result["choices"][0]["message"])["results"]
+    expected_ids = [str(session.get("session_id") or "") for session in sessions]
+    actual_ids = [str(item.get("session_id") or "") for item in results]
+    if len(results) != len(sessions) or sorted(actual_ids) != sorted(expected_ids):
+        raise ValueError(
+            f"DeepSeek verdict IDs do not match batch: expected={expected_ids!r} actual={actual_ids!r}"
+        )
+    by_id = {str(item["session_id"]): item for item in results}
+    return [by_id[session_id] for session_id in expected_ids]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--trace", type=Path, default=DEFAULT_TRACE)
+    parser.add_argument("--live-owner", help="Only audit traced sessions still present in app.db for this owner")
+    parser.add_argument("--endpoint-id")
+    parser.add_argument("--model", default="deepseek-v4-flash")
+    parser.add_argument("--sample-per-tool", type=int, default=2)
+    parser.add_argument("--max-sessions", type=int, default=260)
+    parser.add_argument("--all-sessions", action="store_true", help="Semantically review every session in scope")
+    parser.add_argument("--exclude-verdicts", type=Path, help="Skip session IDs already present in this verdict JSONL")
+    parser.add_argument("--batch-size", type=int, default=4)
+    parser.add_argument("--workers", type=int, default=6)
+    parser.add_argument("--seed", type=int, default=17)
+    parser.add_argument("--skip-deepseek", action="store_true")
+    args = parser.parse_args()
+
+    rows = load_rows(args.trace)
+    if args.live_owner:
+        live_ids = live_session_ids(args.live_owner)
+        rows = [row for row in rows if str(row.get("session_id") or "") in live_ids]
+    sessions: dict[str, list[dict[str, Any]]] = collections.defaultdict(list)
+    tools: collections.Counter[str] = collections.Counter()
+    models: collections.Counter[str] = collections.Counter()
+    flag_counts: collections.Counter[str] = collections.Counter()
+    duplicate_ids: collections.Counter[str] = collections.Counter()
+    content_hashes: collections.defaultdict[str, list[dict[str, Any]]] = collections.defaultdict(list)
+    for row in rows:
+        sid = str(row.get("session_id") or f"invalid-line-{row.get('_invalid_line')}")
+        sessions[sid].append(row)
+        models[str((row.get("metadata") or {}).get("model") or "unknown")] += 1
+        duplicate_ids[str(row.get("message_id") or "missing")] += 1
+        digest = hashlib.sha256(json.dumps([row.get("user"), row.get("assistant"), row.get("tool_events")], sort_keys=True, default=str).encode()).hexdigest()
+        content_hashes[digest].append(row)
+        for flag in row_flags(row):
+            flag_counts[flag] += 1
+        for event in row.get("tool_events") or []:
+            tools[str(event.get("tool") or "unknown")] += 1
+
+    suspicious = {sid for sid, turns in sessions.items() if any(row_flags(row) for row in turns)}
+    by_tool: dict[str, list[str]] = collections.defaultdict(list)
+    for sid, turns in sessions.items():
+        for tool in {str(e.get("tool")) for row in turns for e in row.get("tool_events") or [] if e.get("tool")}:
+            by_tool[tool].append(sid)
+    rng = random.Random(args.seed)
+    if args.all_sessions:
+        selected = set(sessions)
+    else:
+        selected = set(suspicious)
+        for tool, candidates in sorted(by_tool.items()):
+            pool = sorted(set(candidates) - selected)
+            selected.update(rng.sample(pool, min(args.sample_per_tool, len(pool))))
+        selected = set(sorted(selected)[: args.max_sessions])
+    if args.exclude_verdicts:
+        reviewed = {
+            str(json.loads(line).get("session_id") or "")
+            for line in args.exclude_verdicts.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        }
+        selected.difference_update(reviewed)
+
+    stamp = time.strftime("%Y%m%d_%H%M%S")
+    out = OUT_DIR / f"sft_corpus_deepseek_audit_{stamp}"
+    out.mkdir(parents=True, exist_ok=True)
+    deterministic = {
+        "trace": str(args.trace),
+        "live_owner": args.live_owner,
+        "turns": len(rows),
+        "sessions": len(sessions),
+        "tool_counts": dict(tools.most_common()),
+        "model_counts": dict(models.most_common()),
+        "flag_counts": dict(flag_counts.most_common()),
+        "suspicious_sessions": len(suspicious),
+        "duplicate_message_ids": {k: v for k, v in duplicate_ids.items() if v > 1},
+        "exact_duplicate_rows": sum(len(v) - 1 for v in content_hashes.values() if len(v) > 1),
+        "deepseek_selected_sessions": len(selected),
+        "all_sessions": args.all_sessions,
+        "excluded_verdicts": str(args.exclude_verdicts) if args.exclude_verdicts else None,
+    }
+    (out / "coverage.json").write_text(json.dumps(deterministic, indent=2), encoding="utf-8")
+    with (out / "deterministic_repair_queue.jsonl").open("w", encoding="utf-8") as handle:
+        for sid in sorted(suspicious):
+            handle.write(json.dumps({"session_id": sid, "flags": sorted({f for r in sessions[sid] for f in row_flags(r)}), "lines": [r.get("_line") for r in sessions[sid]]}) + "\n")
+
+    judged: list[dict[str, Any]] = []
+    if not args.skip_deepseek:
+        endpoint = deepseek_endpoint(args.endpoint_id, args.model)
+        chosen = sorted(selected)
+        batches = []
+        for start in range(0, len(chosen), args.batch_size):
+            ids = chosen[start : start + args.batch_size]
+            batch = [{"session_id": sid, "name": sessions[sid][0].get("session_name"), "turns": [compact_row(r) for r in sessions[sid]]} for sid in ids]
+            batches.append((start, batch))
+
+        def run_batch(item: tuple[int, list[dict[str, Any]]]) -> tuple[int, list[dict[str, Any]]]:
+            start, batch = item
+            for attempt in range(3):
+                try:
+                    results = judge(endpoint, batch)
+                    return start, results
+                except (urllib.error.URLError, TimeoutError, KeyError, ValueError, json.JSONDecodeError) as exc:
+                    if attempt == 2:
+                        raise RuntimeError(f"DeepSeek batch failed at {start}: {exc}") from exc
+                    time.sleep(3 + attempt * 4)
+            raise AssertionError("unreachable")
+
+        completed = 0
+        ordered: dict[int, list[dict[str, Any]]] = {}
+        with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
+            futures = [pool.submit(run_batch, item) for item in batches]
+            for future in concurrent.futures.as_completed(futures):
+                start, results = future.result()
+                ordered[start] = results
+                completed += len(results)
+                print(f"deepseek {completed}/{len(chosen)}", flush=True)
+        for start in sorted(ordered):
+            judged.extend(ordered[start])
+        with (out / "deepseek_verdicts.jsonl").open("w", encoding="utf-8") as handle:
+            for result in judged:
+                handle.write(json.dumps(result, ensure_ascii=False) + "\n")
+
+    verdicts = collections.Counter(str(row.get("verdict") or "unknown") for row in judged)
+    report = [
+        "# SFT Corpus Audit", "",
+        f"- Trace: `{args.trace}`", f"- Turns: {len(rows)}", f"- Sessions: {len(sessions)}",
+        f"- Tools represented: {len(tools)}", f"- Suspicious sessions (deterministic): {len(suspicious)}",
+        f"- Exact duplicate rows: {deterministic['exact_duplicate_rows']}",
+        f"- DeepSeek sessions reviewed: {len(judged)}", f"- DeepSeek verdicts: `{dict(verdicts)}`", "",
+        "## Deterministic Flags", "",
+    ]
+    report.extend(f"- {name}: {count}" for name, count in flag_counts.most_common())
+    report.extend(["", "## Lowest-Coverage Tools", ""])
+    report.extend(f"- `{tool}`: {count}" for tool, count in sorted(tools.items(), key=lambda x: (x[1], x[0]))[:20])
+    report.extend(["", "## DeepSeek Repair/Delete Queue", ""])
+    for row in judged:
+        if row.get("verdict") == "keep":
+            continue
+        report.append(f"- `{row.get('session_id')}` **{row.get('verdict')}** score={row.get('score')}: {'; '.join(row.get('issues') or [])}")
+    (out / "report.md").write_text("\n".join(report) + "\n", encoding="utf-8")
+    print(f"output={out}")
+
+
+if __name__ == "__main__":
+    main()

scripts/audit_typo_tool_routing.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Build and score deterministic typo variants of real labeled tool prompts."""
+
+from __future__ import annotations
+
+import argparse, hashlib, json, re, sqlite3
+from collections import Counter
+from pathlib import Path
+
+from src.turn_contract import requested_capabilities
+
+DB = Path("/home/pewds/odysseus-cookbook-fresh/data/app.db")
+ANCHOR = "a37dcb3b-6864-4266-a115-f9e87aafd0eb"
+TRIGGERS = {
+    "calendar": ("calendar", "event", "meeting", "appointment", "agenda"),
+    "notes": ("note", "notes", "checklist", "groceries"),
+    "tasks": ("task", "tasks", "todo", "reminder"),
+    "skills": ("skill", "skills"),
+    "memory": ("memory", "memories", "remember", "forget"),
+    "documents": ("document", "documents", "doc", "editor"),
+    "email": ("email", "emails", "inbox", "mail", "spam"),
+    "search_browser": ("search", "web", "browse", "browser", "website", "youtube"),
+    "shell_files": ("file", "files", "folder", "directory", "shell", "terminal", "workspace", "bash", "python"),
+    "cookbook_admin": ("cookbook", "endpoint", "model", "server", "download", "settings"),
+}
+TOOL_FAMILY = {
+    "manage_calendar": "calendar", "manage_notes": "notes", "manage_tasks": "tasks",
+    "manage_skills": "skills", "manage_memory": "memory", "search_chats": "memory",
+    "manage_documents": "documents", "create_document": "documents", "edit_document": "documents",
+    "update_document": "documents", "suggest_document": "documents",
+    "list_email_accounts": "email", "list_emails": "email", "search_emails": "email",
+    "read_email": "email", "send_email": "email", "reply_to_email": "email", "draft_email": "email",
+    "web_search": "search_browser", "web_fetch": "search_browser", "private_browser": "search_browser",
+    "youtube_tool": "search_browser", "search_hf_models": "search_browser",
+    "bash": "shell_files", "python": "shell_files", "read_file": "shell_files", "write_file": "shell_files",
+    "list_models": "cookbook_admin", "list_served_models": "cookbook_admin", "serve_model": "cookbook_admin",
+    "stop_served_model": "cookbook_admin", "list_cookbook_servers": "cookbook_admin", "manage_endpoints": "cookbook_admin",
+}
+NEIGHBOR = {"a":"s","e":"r","i":"o","o":"p","s":"d","t":"y","r":"t","l":"k","n":"m","m":"n","d":"f","c":"v","b":"n","w":"e","f":"g","g":"h","h":"j","p":"o","k":"l","v":"b","u":"i"}
+
+def variants(word: str) -> list[tuple[str,str]]:
+    i = max(1, min(len(word)-2, len(word)//2))
+    out = [("delete", word[:i]+word[i+1:]), ("duplicate", word[:i]+word[i]+word[i:])]
+    if i+1 < len(word): out.append(("transpose", word[:i]+word[i+1]+word[i]+word[i+2:]))
+    repl = NEIGHBOR.get(word[i].lower(), "x")
+    out.append(("neighbor", word[:i]+repl+word[i+1:]))
+    if len(word) >= 6: out.append(("split", word[:i]+" "+word[i:]))
+    return out
+
+def expected_family(metadata: str | None) -> str | None:
+    try: events = json.loads(metadata or "{}").get("tool_events") or []
+    except json.JSONDecodeError: return None
+    families = []
+    for event in events:
+        tool = str(event.get("tool") or "").rsplit("__",1)[-1]
+        if TOOL_FAMILY.get(tool): families.append(TOOL_FAMILY[tool])
+    return families[0] if families and len(set(families)) == 1 else None
+
+def main() -> int:
+    ap=argparse.ArgumentParser(); ap.add_argument("--db",type=Path,default=DB); ap.add_argument("--out",type=Path,required=True); ap.add_argument("--per-family",type=int,default=20); a=ap.parse_args()
+    con=sqlite3.connect(a.db); con.row_factory=sqlite3.Row
+    t0=con.execute("select created_at from sessions where id=?",(ANCHOR,)).fetchone()[0]
+    sessions=con.execute("select id from sessions where owner='sft_alex_creator' and created_at>=? order by created_at,id",(t0,)).fetchall()
+    seeds={f:[] for f in TRIGGERS}
+    for s in sessions:
+        ms=con.execute("select role,content,metadata from chat_messages where session_id=? order by timestamp,id",(s[0],)).fetchall(); history=[]
+        for i,m in enumerate(ms):
+            if m['role']!='user': history.append({'role':m['role'],'content':m['content']}); continue
+            nxt=next((x for x in ms[i+1:] if x['role']=='assistant'),None); fam=expected_family(nxt['metadata'] if nxt else None)
+            if fam and len(seeds[fam])<a.per_family and fam in requested_capabilities(m['content'],history):
+                hit=next((w for w in TRIGGERS[fam] if re.search(r'\b'+re.escape(w)+r'\b',m['content'],re.I)),None)
+                if hit: seeds[fam].append((s[0],m['content'],tuple(history),hit))
+            history.append({'role':'user','content':m['content']})
+    rows=[]
+    for fam,items in seeds.items():
+        for sid,prompt,history,word in items:
+            for kind,bad in variants(word):
+                changed=re.sub(r'\b'+re.escape(word)+r'\b',bad,prompt,count=1,flags=re.I)
+                actual=sorted(requested_capabilities(changed,history)); digest=hashlib.sha256((sid+changed).encode()).hexdigest()
+                rows.append({'split':'blind' if int(digest[:2],16)<64 else 'dev','family':fam,'source_session':sid,'mutation':kind,'prompt':changed,'actual':actual,'passed':fam in actual})
+    summary = {}
+    for split in ('dev', 'blind'):
+        selected = [row for row in rows if row['split'] == split]
+        passed = sum(row['passed'] for row in selected)
+        exact = sum(row['actual'] == [row['family']] for row in selected)
+        wrong = sum(bool(row['actual']) and row['family'] not in row['actual']
+                    and row['actual'] != ['unknown'] for row in selected)
+        abstained = sum(not row['actual'] or row['actual'] == ['unknown'] for row in selected)
+        summary[split] = {
+            'total': len(selected), 'passed': passed,
+            'accuracy': round(passed / len(selected), 6) if selected else None,
+            'exact': exact, 'exact_accuracy': round(exact / len(selected), 6) if selected else None,
+            'wrong_family': wrong,
+            'wrong_family_rate': round(wrong / len(selected), 6) if selected else None,
+            'abstained': abstained,
+        }
+    report={'summary':summary,'family_failures':dict(Counter(r['family'] for r in rows if not r['passed'])),'rows':rows}
+    a.out.write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n'); print(json.dumps({'summary':summary,'family_failures':report['family_failures']},indent=2)); return 0
+if __name__=='__main__': raise SystemExit(main())

scripts/build_odysseus_everyday_deepseek_heldout_cases.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import ModelEndpoint, SessionLocal
+
+DEFAULT_OUT = REPO_ROOT / "data/evals/ody_everyday_deepseek_heldout_v1_20260821/cases.json"
+
+
+FAMILIES: dict[str, dict[str, Any]] = {
+    "notes_create": {
+        "count": 3,
+        "instruction": "Personal note creation requests. The prompt must ask to add/create/save a note with the exact marker as the note title and a short body.",
+        "case": {
+            "kind": "note",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_notes",
+            "must_mutate": "note_created",
+        },
+        "default_user": "Add a note titled __MARKER__ saying buy oats after school pickup",
+    },
+    "tasks_recurring": {
+        "count": 3,
+        "instruction": "Recurring reminder/automation requests involving email/search words. The correct behavior is to create a scheduled task, not run the inner action now. Include exact marker as the task name.",
+        "case": {
+            "kind": "task",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_tasks",
+            "must_mutate": "task_created",
+        },
+        "default_user": "Every morning at 7:30, remind me to review the latest inbox email. Name it __MARKER__",
+    },
+    "calendar_create": {
+        "count": 2,
+        "instruction": "Calendar create requests for tomorrow at 7pm, with exact marker as title. Keep tomorrow/7pm so the existing state check applies.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_created_2026_08_22_19",
+        },
+        "default_user": "Add dinner tomorrow at 7pm titled __MARKER__",
+    },
+    "calendar_move": {
+        "count": 2,
+        "instruction": "Calendar move requests. Ask to move the event with exact marker to 8pm tomorrow.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T19:00:00",
+                "dtend": "2026-08-22T20:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_moved_2026_08_22_20",
+        },
+        "default_user": "Move my calendar event __MARKER__ to 8pm tomorrow",
+    },
+    "calendar_delete": {
+        "count": 2,
+        "instruction": "Calendar delete requests. Ask to delete/remove/cancel the existing event with exact marker as the name.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T13:00:00",
+                "dtend": "2026-08-22T14:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_deleted",
+        },
+        "default_user": "Delete the calendar event named __MARKER__",
+    },
+    "email_latest": {
+        "count": 3,
+        "instruction": "Personal inbox/latest email requests. They must clearly refer to the user's own email, not public web search.",
+        "case": {
+            "kind": "email",
+            "expect_first_tool_any": ["mcp__email__list_emails", "list_emails"],
+            "forbidden_tools": ["web_search", "web_fetch"],
+            "must_answer_any": ["From:", "UID", "Booking.com", "latest email"],
+        },
+        "default_user": "What's my latest emails",
+    },
+    "web_synthesis": {
+        "count": 3,
+        "instruction": "Public web lookup requests about why snails bubble/foam. The prompt should require lookup and explanation, not just links.",
+        "case": {
+            "kind": "web",
+            "expect_first_tool": "web_search",
+            "forbidden_repeat_tools": ["web_search"],
+            "must_answer_any": ["mucus", "foam", "bubble"],
+            "must_answer_any_2": ["stress", "irritant", "predator", "moisture", "defense"],
+            "forbidden_final": ["Here are links for that topic", "WEB SEARCH RESULTS", "```sources"],
+        },
+        "default_user": "Look up why snails bubble up sometimes",
+    },
+    "draft_active_email": {
+        "count": 3,
+        "instruction": "Active email compose draft edit requests. The prompt must ask to write/update the open draft and include the phrase '8am works'.",
+        "case": {
+            "kind": "draft",
+            "active_document": {
+                "title": "Everyday email draft probe",
+                "language": "email",
+                "content": (
+                    "To: test@example.com\n"
+                    "Subject: Re: Test manual draft\n"
+                    "In-Reply-To: <manual@example.com>\n"
+                    "References: <manual@example.com>\n"
+                    "X-Source-UID: 999999\n"
+                    "---\n\n"
+                    "---------- Previous message ----------\n"
+                    "Can you confirm the meeting time?\n"
+                ),
+            },
+            "expect_first_tool_any": ["update_document", "edit_document"],
+            "forbidden_tools": ["manage_calendar", "web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+            "must_mutate": "document_contains_8am",
+        },
+        "default_user": "Write a response to it saying 8am works for me",
+    },
+}
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    db = SessionLocal()
+    try:
+        row = (
+            db.query(ModelEndpoint)
+            .filter(ModelEndpoint.name.ilike("%deepseek%"), ModelEndpoint.is_enabled == True)  # noqa: E712
+            .order_by(ModelEndpoint.updated_at.desc())
+            .first()
+        )
+        if row is None or not row.api_key:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row.name,
+            "base_url": row.base_url,
+            "api_key": row.api_key,
+            "cached_models": row.cached_models or "",
+        }
+    finally:
+        db.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: str) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint["cached_models"] or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        pass
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown."},
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.7,
+        "max_tokens": 3000,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=90) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content.strip(), flags=re.I | re.S)
+    parsed = json.loads(content)
+    return {"model": model, "content": parsed}
+
+
+def valid_user(family: str, text: Any) -> bool:
+    if not isinstance(text, str):
+        return False
+    lowered = text.lower()
+    if family in {"notes_create", "tasks_recurring", "calendar_create", "calendar_move", "calendar_delete"} and "__MARKER__" not in text:
+        return False
+    if family == "calendar_create" and ("tomorrow" not in lowered or "7" not in lowered):
+        return False
+    if family == "calendar_move" and ("tomorrow" not in lowered or "8" not in lowered):
+        return False
+    if family == "draft_active_email" and "8am works" not in lowered:
+        return False
+    return 6 <= len(text.split()) <= 32
+
+
+def build_cases(generated: dict[str, Any]) -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for family, spec in FAMILIES.items():
+        prompts = generated.get(family, [])
+        if not isinstance(prompts, list):
+            prompts = []
+        prompts = [item for item in prompts if valid_user(family, item)]
+        prompts.append(spec["default_user"])
+        chosen: list[str] = []
+        for prompt in prompts:
+            key = prompt.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            chosen.append(prompt)
+            if len(chosen) >= spec["count"]:
+                break
+        while len(chosen) < spec["count"]:
+            chosen.append(spec["default_user"])
+        for idx, user in enumerate(chosen):
+            case = dict(spec["case"])
+            case.update({"id": f"deepseek_{family}_{idx:02d}", "user": user, "deepseek_family": family})
+            cases.append(case)
+    return cases
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+
+    prompt = {
+        "task": "Generate held-out everyday Odysseus tool-use eval prompts.",
+        "date_context": "Current date is 2026-08-21 Asia/Tokyo; tomorrow is 2026-08-22.",
+        "requirements": [
+            "Return JSON object only.",
+            "Keys must be exactly the family names provided.",
+            "Each value is a list of natural user prompts.",
+            "For marker families, include the literal placeholder __MARKER__ exactly once.",
+            "Do not copy the default prompt; produce paraphrases.",
+            "Keep prompts short and realistic.",
+        ],
+        "families": {name: {"count": spec["count"], "instruction": spec["instruction"], "default": spec["default_user"]} for name, spec in FAMILIES.items()},
+    }
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    response = call_deepseek(endpoint, json.dumps(prompt, ensure_ascii=False))
+    cases = build_cases(response["content"])
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": "build_odysseus_everyday_deepseek_heldout_cases.py",
+        "provider": "DeepSeek",
+        "model": response["model"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "families": {name: spec["count"] for name, spec in FAMILIES.items()},
+        "raw_generated": response["content"],
+        "cases": cases,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"out": str(args.out), "cases": len(cases), "model": response["model"], "elapsed_seconds": payload["elapsed_seconds"]}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_everyday_deepseek_heldout_v3_cases.py

@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from core.database import ModelEndpoint, SessionLocal
+
+
+DEFAULT_OUT = REPO_ROOT / "data/evals/ody_everyday_deepseek_heldout_v3_20260821/cases.json"
+
+
+FAMILIES: dict[str, dict[str, Any]] = {
+    "negative_email_concept": {
+        "count": 3,
+        "instruction": "Text-only questions about what email/inbox/reply concepts mean. Do not ask to access the user's mailbox.",
+        "case": {
+            "kind": "negative_email",
+            "expect_no_tool": True,
+            "must_answer_any": ["email", "message", "reply", "inbox"],
+            "forbidden_tools": ["web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+        },
+        "default_user": "What does replying to an email mean? Don't open my inbox.",
+    },
+    "negative_calendar_concept": {
+        "count": 3,
+        "instruction": "Text-only calendar questions that explicitly do not ask to create/update/delete events.",
+        "case": {
+            "kind": "negative_calendar",
+            "expect_no_tool": True,
+            "must_answer_any": ["calendar", "event", "invite", "schedule"],
+            "forbidden_tools": ["manage_calendar"],
+        },
+        "default_user": "What is a calendar invite? Don't add anything.",
+    },
+    "negative_web_no_lookup": {
+        "count": 3,
+        "instruction": "Text-only web/search concept prompts that explicitly say not to search or look anything up.",
+        "case": {
+            "kind": "negative_web",
+            "expect_no_tool": True,
+            "must_answer_any": ["search", "web", "pages", "results"],
+            "forbidden_tools": ["web_search"],
+        },
+        "default_user": "Explain what search results are without searching.",
+    },
+    "notes_create": {
+        "count": 4,
+        "instruction": "Personal note creation requests. Include literal __MARKER__ exactly once as the note title and a short body.",
+        "case": {
+            "kind": "note",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_notes",
+            "must_mutate": "note_created",
+        },
+        "default_user": "Save a note titled __MARKER__ with body pick up dry cleaning",
+    },
+    "tasks_recurring": {
+        "count": 4,
+        "instruction": "Recurring reminder/automation requests that mention email/search/web/inbox words. Correct behavior is scheduled task creation, not doing the inner action immediately. Include __MARKER__ exactly once as task name.",
+        "case": {
+            "kind": "task",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_tasks",
+            "forbidden_tools": ["web_search", "mcp__email__list_emails"],
+            "must_mutate": "task_created",
+        },
+        "default_user": "Create a recurring task named __MARKER__ to check my inbox every morning at 7:30",
+    },
+    "calendar_create": {
+        "count": 4,
+        "instruction": "Calendar create requests for tomorrow at 7pm. Include __MARKER__ exactly once as title/name.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_created_2026_08_22_19",
+        },
+        "default_user": "Put __MARKER__ on my calendar tomorrow at 7pm",
+    },
+    "calendar_move": {
+        "count": 4,
+        "instruction": "Calendar move/reschedule requests for an existing event. Include __MARKER__ exactly once and move it to 8pm tomorrow.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T19:00:00",
+                "dtend": "2026-08-22T20:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "forbidden_tools": ["manage_tasks"],
+            "must_mutate": "calendar_moved_2026_08_22_20",
+        },
+        "default_user": "Reschedule __MARKER__ to tomorrow at 8pm",
+    },
+    "calendar_delete": {
+        "count": 4,
+        "instruction": "Calendar delete/remove/cancel requests for an existing event by title/name. Include __MARKER__ exactly once.",
+        "case": {
+            "kind": "calendar",
+            "marker": "__MARKER__",
+            "precreate_calendar_event": {
+                "summary": "__MARKER__",
+                "dtstart": "2026-08-22T13:00:00",
+                "dtend": "2026-08-22T14:00:00",
+            },
+            "expect_first_tool": "manage_calendar",
+            "must_mutate": "calendar_deleted",
+        },
+        "default_user": "Cancel the calendar event titled __MARKER__",
+    },
+    "email_latest": {
+        "count": 4,
+        "instruction": "Personal latest/recent inbox requests. They must refer to the user's own email and must not sound like public web search.",
+        "case": {
+            "kind": "email",
+            "expect_first_tool_any": ["mcp__email__list_emails", "list_emails"],
+            "forbidden_tools": ["web_search", "web_fetch"],
+            "must_answer_any": ["From:", "UID", "latest email", "email"],
+        },
+        "default_user": "Show me the latest thing in my inbox.",
+    },
+    "web_synthesis": {
+        "count": 4,
+        "instruction": "Public web lookup requests about why snails bubble/foam. Must require lookup plus a concise explanation, not just links.",
+        "case": {
+            "kind": "web",
+            "expect_first_tool": "web_search",
+            "forbidden_repeat_tools": ["web_search"],
+            "must_answer_any": ["mucus", "foam", "bubble"],
+            "must_answer_any_2": ["stress", "irritant", "predator", "moisture", "defense"],
+            "forbidden_final": ["Here are links for that topic", "WEB SEARCH RESULTS", "```sources"],
+        },
+        "default_user": "Find out why snails foam up and explain the reason.",
+    },
+    "draft_active_email": {
+        "count": 4,
+        "instruction": "Active email compose draft edit requests. Ask to write/update the open/current/active draft, and include phrase '8am works'. Do not ask to send.",
+        "case": {
+            "kind": "draft",
+            "active_document": {
+                "title": "Everyday email draft probe",
+                "language": "email",
+                "content": (
+                    "To: test@example.com\n"
+                    "Subject: Re: Test manual draft\n"
+                    "In-Reply-To: <manual@example.com>\n"
+                    "References: <manual@example.com>\n"
+                    "X-Source-UID: 999999\n"
+                    "---\n\n"
+                    "---------- Previous message ----------\n"
+                    "Can you confirm the meeting time?\n"
+                ),
+            },
+            "expect_first_tool_any": ["update_document", "edit_document"],
+            "forbidden_tools": ["manage_calendar", "web_search", "mcp__email__list_emails", "mcp__email__read_email"],
+            "must_mutate": "document_contains_8am",
+        },
+        "default_user": "In the active email draft, write that 8am works for me.",
+        "fallback_users": [
+            "In the active email draft, write that 8am works for me.",
+            "Update the open email draft to say 8am works.",
+            "Add to the current draft that 8am works for me.",
+            "Write back in the active draft that 8am works.",
+        ],
+    },
+}
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    db = SessionLocal()
+    try:
+        row = (
+            db.query(ModelEndpoint)
+            .filter(
+                ModelEndpoint.name.ilike("%deepseek%"),
+                ModelEndpoint.is_enabled == True,  # noqa: E712
+                ModelEndpoint.api_key.isnot(None),
+                ModelEndpoint.api_key != "",
+            )
+            .order_by(ModelEndpoint.updated_at.desc())
+            .first()
+        )
+        if row is None or not row.api_key:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row.name,
+            "base_url": row.base_url,
+            "api_key": row.api_key,
+            "cached_models": row.cached_models or "",
+        }
+    finally:
+        db.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: str) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint["cached_models"] or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        pass
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown or commentary."},
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.85,
+        "max_tokens": 5000,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=90) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", (content or "").strip(), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    try:
+        parsed = json.loads(cleaned)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(f"DeepSeek response was not JSON: {cleaned[:1000]!r}") from exc
+    return {"model": model, "content": parsed}
+
+
+def valid_user(family: str, text: Any) -> bool:
+    if not isinstance(text, str):
+        return False
+    lowered = text.lower()
+    marker_family = family in {
+        "notes_create",
+        "tasks_recurring",
+        "calendar_create",
+        "calendar_move",
+        "calendar_delete",
+    }
+    if marker_family and text.count("__MARKER__") != 1:
+        return False
+    if family == "calendar_create" and ("tomorrow" not in lowered or "7" not in lowered):
+        return False
+    if family == "calendar_move" and ("tomorrow" not in lowered or "8" not in lowered):
+        return False
+    if family == "draft_active_email" and "8am works" not in lowered:
+        return False
+    if family.startswith("negative_") and any(word in lowered for word in ("open my", "show me my", "latest", "create", "delete", "remove", "schedule it")):
+        return False
+    return 5 <= len(text.split()) <= 34
+
+
+def build_cases(generated: dict[str, Any]) -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for family, spec in FAMILIES.items():
+        prompts = generated.get(family, [])
+        if not isinstance(prompts, list):
+            prompts = []
+        prompts = [item for item in prompts if valid_user(family, item)]
+        prompts.append(spec["default_user"])
+        chosen: list[str] = []
+        for prompt in prompts:
+            key = prompt.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            chosen.append(prompt)
+            if len(chosen) >= spec["count"]:
+                break
+        fallback_users = spec.get("fallback_users") or [spec["default_user"]]
+        fallback_idx = 0
+        while len(chosen) < spec["count"]:
+            fallback = fallback_users[fallback_idx % len(fallback_users)]
+            fallback_idx += 1
+            key = fallback.lower()
+            if key in seen and len(fallback_users) > 1:
+                continue
+            seen.add(key)
+            chosen.append(fallback)
+        for idx, user in enumerate(chosen):
+            case = dict(spec["case"])
+            case.update({"id": f"deepseek_v3_{family}_{idx:02d}", "user": user, "deepseek_family": family})
+            cases.append(case)
+    return cases
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+
+    prompt = {
+        "task": "Generate broader held-out everyday Odysseus tool-use eval prompts.",
+        "date_context": "Current date is 2026-08-21 Asia/Tokyo; tomorrow is 2026-08-22.",
+        "requirements": [
+            "Return JSON object only.",
+            "Keys must be exactly the family names provided.",
+            "Each value is a list of natural user prompts.",
+            "Generate at least count+3 prompts per family so validation can discard weak ones.",
+            "For marker families, include literal placeholder __MARKER__ exactly once.",
+            "Do not copy the default prompt; produce realistic paraphrases with varied syntax.",
+            "Avoid multi-intent prompts; each prompt should test one requested action.",
+        ],
+        "families": {
+            name: {
+                "count": spec["count"],
+                "instruction": spec["instruction"],
+                "default": spec["default_user"],
+            }
+            for name, spec in FAMILIES.items()
+        },
+    }
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    response = call_deepseek(endpoint, json.dumps(prompt, ensure_ascii=False))
+    cases = build_cases(response["content"])
+    payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": "build_odysseus_everyday_deepseek_heldout_v3_cases.py",
+        "provider": "DeepSeek",
+        "model": response["model"],
+        "elapsed_seconds": round(time.time() - started, 3),
+        "families": {name: spec["count"] for name, spec in FAMILIES.items()},
+        "raw_generated": response["content"],
+        "cases": cases,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"out": str(args.out), "cases": len(cases), "model": response["model"], "elapsed_seconds": payload["elapsed_seconds"]}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_realistic_web_synthesis_rows.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ACTUALS = [
+    REPO_ROOT / "data/evals/ody_search_teacher_pipeline_20260821/deepseek_actual/actual_results.json",
+    REPO_ROOT / "data/evals/ody_v57_quick_live_search_cases_20260821/v59_run_20260821_2042/actual_results.json",
+]
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v60_realistic_verbose_web_synthesis_20260821")
+
+WEB_TOOLS = {"web_search", "web_fetch"}
+FORBIDDEN_FINAL_RE = re.compile(
+    r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b|from the search results|results indicate|returned snippets|top results|i searched",
+    re.IGNORECASE,
+)
+
+TOOL_SCHEMAS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "web_search",
+            "description": "Search the public web for source-backed information.",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "web_fetch",
+            "description": "Fetch a specific URL when search snippets do not contain enough evidence.",
+            "parameters": {
+                "type": "object",
+                "properties": {"url": {"type": "string"}},
+                "required": ["url"],
+            },
+        },
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def load_teacher_endpoint(db_path: Path, model: str | None) -> dict[str, str]:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        row = conn.execute(
+            """
+            SELECT base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE is_enabled = 1
+              AND api_key IS NOT NULL
+              AND api_key != ''
+              AND (lower(name) LIKE '%deepseek%' OR lower(id) LIKE '%deepseek%')
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+    finally:
+        conn.close()
+    if row is None:
+        raise RuntimeError("no enabled DeepSeek endpoint with API key found in app DB")
+    selected_model = model
+    if not selected_model:
+        cached = json.loads(row["cached_models"] or "[]")
+        selected_model = cached[0] if cached else "deepseek-v4-flash"
+    return {"base_url": row["base_url"], "api_key": row["api_key"], "model": selected_model}
+
+
+def call_json(endpoint: dict[str, str], payload: dict[str, Any]) -> dict[str, Any]:
+    body = {
+        "model": endpoint["model"],
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "Return strict JSON only. You are creating SFT final answers for web tool traces. "
+                    "Do not include chain-of-thought or prose outside JSON."
+                ),
+            },
+            {"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
+        ],
+        "temperature": 0.2,
+        "max_tokens": 900,
+        "response_format": {"type": "json_object"},
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=180) as resp:
+        parsed = json.loads(resp.read().decode("utf-8"))
+    text = str(parsed["choices"][0]["message"].get("content") or "").strip()
+    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
+    return json.loads(text)
+
+
+def normalize_args(tool: str, args: Any) -> dict[str, Any]:
+    if isinstance(args, dict):
+        return args
+    if isinstance(args, str):
+        stripped = args.strip()
+        if stripped.startswith("{"):
+            try:
+                parsed = json.loads(stripped)
+                if isinstance(parsed, dict):
+                    return parsed
+            except json.JSONDecodeError:
+                pass
+        return {"query": stripped} if tool == "web_search" else {"url": stripped}
+    return {}
+
+
+def compact_tool_output(text: str, max_chars: int = 3000) -> str:
+    text = re.sub(r"\r\n?", "\n", text or "").strip()
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    if len(text) <= max_chars:
+        return text
+    sources = ""
+    if text.startswith("```sources"):
+        end = text.find("```", 3)
+        if end != -1:
+            sources = text[: end + 3].strip()
+    summary_match = re.search(r"SEARCH RESULTS SUMMARY:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)", text, re.DOTALL)
+    summary = summary_match.group("body").strip() if summary_match else ""
+    fetched_match = re.search(r"FETCHED PAGE CONTENT:\n[-]+\n(?P<body>.*?)(?:\n={10,}|\Z)", text, re.DOTALL)
+    fetched = fetched_match.group("body").strip() if fetched_match else ""
+    chunks = [chunk for chunk in [sources, summary[:1600], fetched[:900]] if chunk]
+    compact = "\n\n".join(chunks).strip()
+    if not compact:
+        compact = text[:max_chars].rstrip()
+    return compact[:max_chars].rstrip()
+
+
+def load_results(paths: list[Path]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for path in paths:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+        for result in payload.get("results") or []:
+            key = f"{path}:{result.get('id')}"
+            if key in seen:
+                continue
+            seen.add(key)
+            result = dict(result)
+            result["_source_path"] = str(path)
+            out.append(result)
+    return out
+
+
+def load_teacher_finals(path: Path | None) -> dict[str, str]:
+    if path is None:
+        return {}
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    finals: dict[str, str] = {}
+    for item in payload.get("edits") or []:
+        if not item.get("accepted"):
+            continue
+        edited = item.get("edited") or {}
+        final = str(edited.get("final") or "").strip()
+        if final and not FORBIDDEN_FINAL_RE.search(final):
+            finals[str(item.get("id"))] = final
+    return finals
+
+
+def usable_web_steps(result: dict[str, Any], max_tools: int) -> list[dict[str, Any]]:
+    calls = result.get("tool_calls") or []
+    outputs = result.get("tool_outputs") or []
+    steps: list[dict[str, Any]] = []
+    for idx, call in enumerate(calls):
+        tool = call.get("tool") or call.get("name")
+        if tool not in WEB_TOOLS:
+            continue
+        if idx >= len(outputs):
+            continue
+        output = outputs[idx]
+        if output.get("tool") and output.get("tool") not in WEB_TOOLS:
+            continue
+        args = normalize_args(tool, call.get("args"))
+        if tool == "web_search" and not args.get("query"):
+            continue
+        if tool == "web_fetch" and not args.get("url"):
+            continue
+        content = compact_tool_output(str(output.get("output") or ""))
+        if not content:
+            continue
+        steps.append({"tool": tool, "args": args, "output": content})
+        if len(steps) >= max_tools:
+            break
+    return steps
+
+
+def teacher_final(endpoint: dict[str, str], result: dict[str, Any], steps: list[dict[str, Any]]) -> dict[str, Any]:
+    prompt = {
+        "task": "Write the assistant's final answer after these web tool calls.",
+        "current_date": "2026-08-21",
+        "user": result.get("user") or "",
+        "prior_turns": result.get("prior_turns") or [],
+        "tool_steps": steps,
+        "bad_actual_final": result.get("final_answer") or "",
+        "requirements": [
+            "Return JSON with should_train boolean, final string, and reason string.",
+            "Use the tool evidence to answer the user's actual question directly.",
+            "If snippets are insufficient for a precise value, say the best supported answer and the uncertainty briefly.",
+            "Do not say 'from the search results', 'results indicate', 'snippets', 'I searched', or list sources.",
+            "Do not copy raw snippets. Synthesize.",
+            "Keep the final to 1-4 short sentences.",
+            "If this request should not have searched, set should_train=false.",
+        ],
+    }
+    return call_json(endpoint, prompt)
+
+
+def build_row(result: dict[str, Any], steps: list[dict[str, Any]], final: str) -> dict[str, Any] | None:
+    final = re.sub(r"\s+", " ", final).strip()
+    if not final or len(final) > 900 or FORBIDDEN_FINAL_RE.search(final):
+        return None
+    messages: list[dict[str, Any]] = []
+    for turn in result.get("prior_turns") or []:
+        if isinstance(turn, dict) and turn.get("user"):
+            messages.append({"role": "user", "content": str(turn["user"])})
+            if turn.get("assistant"):
+                messages.append({"role": "assistant", "content": str(turn["assistant"])})
+    messages.append({"role": "user", "content": result.get("user") or ""})
+    for idx, step in enumerate(steps):
+        call_id = f"call_{result.get('id', 'web')}_{idx}"
+        messages.append({
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "id": call_id,
+                "type": "function",
+                "function": {
+                    "name": step["tool"],
+                    "arguments": json.dumps(step["args"], separators=(",", ":"), ensure_ascii=True),
+                },
+            }],
+        })
+        messages.append({"role": "tool", "tool_call_id": call_id, "content": step["output"]})
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "tools": TOOL_SCHEMAS,
+        "generator": "odysseus_realistic_verbose_web_synthesis_teacher",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "source_path": result.get("_source_path"),
+            "source_pass": result.get("pass"),
+            "actual_final": result.get("final_answer") or "",
+        },
+    }
+    row["uuid"] = stable_id("ody_v60_realistic_web_synthesis", row)
+    return row
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--actual", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--db", type=Path, default=REPO_ROOT / "data/app.db")
+    parser.add_argument("--teacher-model", default="")
+    parser.add_argument("--teacher-edits", type=Path)
+    parser.add_argument("--max-cases", type=int, default=180)
+    parser.add_argument("--max-tools", type=int, default=3)
+    args = parser.parse_args()
+
+    paths = args.actual or DEFAULT_ACTUALS
+    final_by_id = load_teacher_finals(args.teacher_edits)
+    endpoint = None if final_by_id else load_teacher_endpoint(args.db, args.teacher_model or None)
+    results = load_results(paths)
+    candidates = []
+    for result in results:
+        if result.get("kind") != "web":
+            continue
+        steps = usable_web_steps(result, args.max_tools)
+        if steps and steps[0]["tool"] == "web_search":
+            candidates.append((result, steps))
+    candidates = candidates[: args.max_cases]
+
+    rows: list[dict[str, Any]] = []
+    audits: list[dict[str, Any]] = []
+    for result, steps in candidates:
+        try:
+            if result.get("id") in final_by_id:
+                edited = {
+                    "should_train": True,
+                    "final": final_by_id[str(result.get("id"))],
+                    "reason": "reused existing teacher-edited final",
+                }
+            else:
+                assert endpoint is not None
+                edited = teacher_final(endpoint, result, steps)
+            row = None
+            if edited.get("should_train") is True:
+                row = build_row(result, steps, str(edited.get("final") or ""))
+            accepted = row is not None
+            if accepted:
+                rows.append(row)
+            audits.append({
+                "id": result.get("id"),
+                "source_path": result.get("_source_path"),
+                "accepted": accepted,
+                "tool_count": len(steps),
+                "actual_final": result.get("final_answer") or "",
+                "teacher": edited,
+            })
+        except Exception as exc:
+            audits.append({"id": result.get("id"), "source_path": result.get("_source_path"), "accepted": False, "error": repr(exc)})
+        print(json.dumps({"processed": len(audits), "accepted": len(rows), "id": result.get("id")}), flush=True)
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 10 == 9 else train).append(row)
+    for name, subset in [("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)]:
+        (args.out_dir / name).write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset), encoding="utf-8")
+    (args.out_dir / "audit.json").write_text(json.dumps({"audit": audits}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_actuals": [str(path) for path in paths],
+        "candidate_cases": len(candidates),
+        "accepted_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "max_tools": args.max_tools,
+        "goal": "train direct synthesis after realistic verbose web_search/web_fetch outputs",
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "audit": str(args.out_dir / "audit.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_search_teacher_edited_rows.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ACTUAL = REPO_ROOT / "data/evals/ody_search_teacher_pipeline_20260821/deepseek_actual/actual_results.json"
+DEFAULT_OUT_DIR = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v58_teacher_edited_search_traces_20260821")
+
+WEB_TOOLS = {"web_search", "web_fetch"}
+SOURCE_DUMP_RE = re.compile(r"WEB SEARCH RESULTS|```sources|\b\d+\s+Web sources\b", re.IGNORECASE)
+META_FINAL_RE = re.compile(r"\b(the user asked|the user is asking|tool evidence|i should answer)\b", re.IGNORECASE)
+
+TOOL_SCHEMAS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "web_search",
+            "description": "Search the public web for source-backed information.",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "web_fetch",
+            "description": "Fetch a specific URL when search snippets do not contain enough evidence.",
+            "parameters": {
+                "type": "object",
+                "properties": {"url": {"type": "string"}},
+                "required": ["url"],
+            },
+        },
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def call_json(base_url: str, api_key: str, model: str, payload: dict[str, Any]) -> dict[str, Any]:
+    body = {
+        "model": model,
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "Return strict JSON only. You are editing tool-use traces for SFT. "
+                    "Do not include chain-of-thought or prose outside JSON."
+                ),
+            },
+            {"role": "user", "content": json.dumps(payload, ensure_ascii=False)},
+        ],
+        "temperature": 0.25,
+        "max_tokens": 2200,
+        "response_format": {"type": "json_object"},
+    }
+    req = request.Request(
+        base_url.rstrip("/") + "/chat/completions",
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=180) as resp:
+        parsed = json.loads(resp.read().decode("utf-8"))
+    text = str(parsed["choices"][0]["message"].get("content") or "").strip()
+    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE | re.DOTALL).strip()
+    return json.loads(text)
+
+
+def summarize_outputs(result: dict[str, Any]) -> list[dict[str, Any]]:
+    outputs = []
+    for idx, output in enumerate(result.get("tool_outputs") or []):
+        text = str(output.get("output") or "")
+        outputs.append({
+            "tool": output.get("tool"),
+            "output_head": text[:1800],
+            "output_tail": text[-800:] if len(text) > 1800 else "",
+            "exit_code": output.get("exit_code"),
+            "call_args": (result.get("tool_calls") or [{}])[idx].get("args") if idx < len(result.get("tool_calls") or []) else None,
+        })
+    return outputs
+
+
+def needs_teacher_edit(result: dict[str, Any]) -> bool:
+    final = str(result.get("final_answer") or "")
+    tools = result.get("tool_names") or []
+    failures = result.get("failures") or []
+    if result.get("kind") != "web":
+        return False
+    if not tools or tools[0] != "web_search":
+        return True
+    if any(tool not in WEB_TOOLS for tool in tools):
+        return True
+    if len(tools) > 3:
+        return True
+    if SOURCE_DUMP_RE.search(final) or META_FINAL_RE.search(final):
+        return True
+    if len(final.split()) < 8:
+        return True
+    if failures:
+        return True
+    return False
+
+
+def teacher_edit(endpoint: dict[str, str], result: dict[str, Any]) -> dict[str, Any]:
+    prompt = {
+        "task": "Edit this failed/weak Odysseus web tool trace into one minimal correct SFT trace.",
+        "current_date": "2026-08-21",
+        "user": result.get("user"),
+        "prior_turns": result.get("prior_turns") or [],
+        "actual_tool_calls": result.get("tool_calls") or [],
+        "actual_tool_outputs": summarize_outputs(result),
+        "actual_final": result.get("final_answer") or "",
+        "failures": result.get("failures") or [],
+        "requirements": [
+            "Return JSON with should_train boolean, reason string, trace array, and final string.",
+            "If the user request is evergreen/simple and should not search, set should_train=false.",
+            "For search-worthy requests, trace must contain 1 to 3 tool steps.",
+            "Each trace step must have tool, args, and output.",
+            "Allowed tools are only web_search and web_fetch.",
+            "web_search args must be an object like {\"query\":\"...\"}. The query must preserve the important nouns, requested property, location, time, and follow-up context.",
+            "Use web_fetch only after a search when snippets are insufficient and include a plausible URL from the search evidence.",
+            "The output field should be concise synthetic tool evidence, not a huge raw dump. It must contain enough evidence to justify the final.",
+            "The final must answer directly in 1-4 sentences. No source dumps. No 'the user asked'.",
+            "Do not hardcode this exact test; infer the general correct behavior from the request.",
+        ],
+    }
+    return call_json(endpoint["base_url"], endpoint["api_key"], endpoint["model"], prompt)
+
+
+def build_row(result: dict[str, Any], edited: dict[str, Any]) -> dict[str, Any] | None:
+    if edited.get("should_train") is not True:
+        return None
+    trace = edited.get("trace")
+    final = re.sub(r"\s+", " ", str(edited.get("final") or "")).strip()
+    if not isinstance(trace, list) or not trace or len(trace) > 3:
+        return None
+    if not final or SOURCE_DUMP_RE.search(final) or META_FINAL_RE.search(final) or len(final) > 1200:
+        return None
+    messages: list[dict[str, Any]] = [{"role": "user", "content": result.get("user") or ""}]
+    for idx, step in enumerate(trace):
+        if not isinstance(step, dict):
+            return None
+        tool = str(step.get("tool") or "")
+        if tool not in WEB_TOOLS:
+            return None
+        args = step.get("args") or {}
+        if isinstance(args, str):
+            try:
+                args = json.loads(args)
+            except json.JSONDecodeError:
+                args = {"query": args} if tool == "web_search" else {"url": args}
+        if tool == "web_search" and not str(args.get("query") or "").strip():
+            return None
+        if tool == "web_fetch" and not str(args.get("url") or "").strip():
+            return None
+        output = str(step.get("output") or "").strip()
+        if not output or len(output) > 1800:
+            output = output[:1800].rstrip()
+        call_id = f"call_{result.get('id', 'trace')}_{idx}"
+        messages.append({
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "id": call_id,
+                "type": "function",
+                "function": {"name": tool, "arguments": json.dumps(args, separators=(",", ":"), ensure_ascii=True)},
+            }],
+        })
+        messages.append({"role": "tool", "tool_call_id": call_id, "content": output})
+    messages.append({"role": "assistant", "content": final})
+    row = {
+        "messages": messages,
+        "tools": TOOL_SCHEMAS,
+        "generator": "odysseus_deepseek_teacher_edited_search_trace",
+        "metadata": {
+            "source_result_id": result.get("id"),
+            "source_pass": result.get("pass"),
+            "actual_tool_names": result.get("tool_names") or [],
+            "teacher_reason": edited.get("reason") or "",
+        },
+    }
+    row["uuid"] = stable_id("ody_v58_teacher_edited_search", row)
+    return row
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--actual", type=Path, default=DEFAULT_ACTUAL)
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR)
+    parser.add_argument("--base-url", default=os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"))
+    parser.add_argument("--model", default=os.environ.get("DEEPSEEK_TEACHER_MODEL", "deepseek-chat"))
+    parser.add_argument("--api-key", default=os.environ.get("DEEPSEEK_API_KEY", ""))
+    parser.add_argument("--max-cases", type=int, default=120)
+    args = parser.parse_args()
+    if not args.api_key:
+        raise RuntimeError("DEEPSEEK_API_KEY is required")
+    payload = json.loads(args.actual.read_text(encoding="utf-8"))
+    endpoint = {"base_url": args.base_url, "api_key": args.api_key, "model": args.model}
+    candidates = [result for result in payload.get("results") or [] if needs_teacher_edit(result)]
+    candidates = candidates[: args.max_cases]
+    rows: list[dict[str, Any]] = []
+    edits: list[dict[str, Any]] = []
+    for result in candidates:
+        try:
+            edited = teacher_edit(endpoint, result)
+            row = build_row(result, edited)
+            accepted = row is not None
+            if accepted:
+                rows.append(row)
+            edits.append({
+                "id": result.get("id"),
+                "user": result.get("user"),
+                "accepted": accepted,
+                "actual_tool_names": result.get("tool_names") or [],
+                "actual_final": result.get("final_answer") or "",
+                "edited": edited,
+            })
+        except Exception as exc:
+            edits.append({"id": result.get("id"), "user": result.get("user"), "accepted": False, "error": repr(exc)})
+        print(json.dumps({"processed": len(edits), "accepted": len(rows), "id": result.get("id")}), flush=True)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % 8 == 7 else train).append(row)
+    for name, subset in [("all.jsonl", rows), ("train.jsonl", train), ("val.jsonl", val)]:
+        (args.out_dir / name).write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in subset), encoding="utf-8")
+    (args.out_dir / "edits.json").write_text(json.dumps({"edits": edits}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    manifest = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "source_actual_results": str(args.actual),
+        "candidate_cases": len(candidates),
+        "accepted_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "allowed_tools": sorted(WEB_TOOLS),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "edits": str(args.out_dir / "edits.json"),
+        },
+    }
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_odysseus_tool_efficiency_sft_slice.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+"""Build a small targeted Odysseus tool-router SFT slice.
+
+This slice targets current measured gaps rather than broad tool coverage:
+
+- one-call manage_memory add;
+- one-call manage_memory add inside CRUD follow-through;
+- clean manage_tasks create schema;
+- contextual web_search follow-up after a normal answer;
+- no-tool chat boundaries.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+
+MANAGE_MEMORY_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_memory",
+        "description": "Manage saved memories: list, add, edit, delete, or search.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string", "enum": ["list", "add", "edit", "delete", "search"]},
+                "text": {"type": "string"},
+                "memory_id": {"type": "string"},
+                "category": {"type": "string", "enum": ["fact", "event", "contact", "preference"]},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+MANAGE_TASKS_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_tasks",
+        "description": "Manage scheduled or recurring background tasks.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string", "enum": ["list", "create", "edit", "delete", "pause", "resume"]},
+                "task_id": {"type": "string"},
+                "name": {"type": "string"},
+                "prompt": {"type": "string"},
+                "task_type": {"type": "string", "enum": ["llm", "research", "action"]},
+                "schedule": {"type": "string"},
+                "scheduled_time": {"type": "string"},
+                "output_target": {"type": "string"},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {"type": "string"},
+                "time_filter": {"type": "string", "enum": ["day", "week", "month", "year"]},
+            },
+            "required": ["query"],
+        },
+    },
+}
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def memory_rows() -> list[dict[str, Any]]:
+    markers = [
+        ("Remember this temporary eval fact: {text}.", "fact"),
+        ("Save this about me: {text}.", "fact"),
+        ("Store this preference: {text}.", "preference"),
+        ("Add this to memory: {text}.", "fact"),
+        ("Add to memory that {text}.", "fact"),
+        ("Please remember: {text}.", "fact"),
+        ("Save this as a memory: {text}.", "fact"),
+        ("Keep this in saved memory: {text}.", "fact"),
+        ("Can you remember this for later: {text}.", "fact"),
+        ("Put this in memory: {text}.", "fact"),
+        ("Make a memory that says {text}.", "fact"),
+        ("I want you to remember that {text}.", "fact"),
+        ("Save this preference for me: {text}.", "preference"),
+        ("Add a saved fact: {text}.", "fact"),
+    ]
+    facts = [
+        "I prefer concise travel checklists",
+        "My current project is organizing public domain art references",
+        "I like calendar summaries grouped by day",
+        "My preferred invoice label is Tsuki admin",
+        "I want model eval notes kept short",
+        "I use Runpod for temporary H100 training jobs",
+        "I prefer source links when asking for websites",
+        "My document drafts should stay in markdown",
+        "short eval probes should use temporary fixture markers",
+        "tool add calls should include the memory text immediately",
+        "memory cleanup should be checked after CRUD evals",
+        "adapter comparisons should record both correctness and efficiency",
+        "I prefer benchmark summaries to include artifact paths",
+        "I want Odysseus tool tests to report input tokens",
+        "I prefer LAN testing before blaming model latency",
+        "I like public domain art links from official sources",
+        "I want temporary eval memories deleted after tests",
+        "I prefer compact prompts for Qwen tool-router evals",
+        "I track LoRA quality by correctness and tool efficiency",
+        "I want web-link followups to use search when URLs are requested",
+        "I prefer no-tool answers for general knowledge reminders",
+        "I want memory add calls to avoid validation retries",
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, text in enumerate(facts):
+        template, category = markers[i % len(markers)]
+        user = template.format(text=text)
+        args = {"action": "add", "text": text, "category": category}
+        call = tool_call("manage_memory", args, f"memory_add_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": f"Memory added: [{category}] {text}"},
+                {"role": "assistant", "content": "Done."},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "memory_one_call_add",
+                "target_issue": "avoid_incomplete_manage_memory_add_first_call",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_memory", row)
+        rows.append(row)
+    return rows
+
+
+def memory_crud_rows() -> list[dict[str, Any]]:
+    specs = [
+        (
+            "ODY-EVAL-CRUD-MEMORY-FLOW alpha checkpoint",
+            "ODY-EVAL-CRUD-MEMORY-FLOW beta checkpoint",
+            "fact",
+        ),
+        (
+            "I prefer one paragraph status updates for model evals",
+            "I prefer concise bullet status updates for model evals",
+            "preference",
+        ),
+        (
+            "My current benchmark focus is Odysseus tool-call efficiency",
+            "My current benchmark focus is memory add one-call efficiency",
+            "fact",
+        ),
+        (
+            "I use temporary memory fixtures during harness tests",
+            "I delete temporary memory fixtures after harness tests",
+            "fact",
+        ),
+        (
+            "I want saved memory changes to avoid retry tool calls",
+            "I want saved memory add calls to include text immediately",
+            "preference",
+        ),
+        (
+            "Runpod H100 jobs should be tracked in short notes",
+            "Runpod H100 jobs should be tracked with adapter and eval paths",
+            "fact",
+        ),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (alpha, beta, category) in enumerate(specs):
+        memory_id = f"mem_eff_{i:02d}"
+        add_call = tool_call(
+            "manage_memory",
+            {"action": "add", "text": alpha, "category": category},
+            f"memory_crud_add_{i}",
+        )
+        edit_call = tool_call(
+            "manage_memory",
+            {"action": "edit", "memory_id": memory_id, "text": beta},
+            f"memory_crud_edit_{i}",
+        )
+        delete_call = tool_call(
+            "manage_memory",
+            {"action": "delete", "memory_id": memory_id},
+            f"memory_crud_delete_{i}",
+        )
+        row = {
+            "messages": [
+                {"role": "user", "content": f"Remember this temporary eval fact: {alpha}."},
+                {"role": "assistant", "content": "", "tool_calls": [add_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": add_call["id"],
+                    "content": f"Memory added: [{category}] {alpha}\nMemory id: {memory_id}",
+                },
+                {"role": "assistant", "content": "Done."},
+                {"role": "user", "content": f"Update that memory to say {beta}."},
+                {"role": "assistant", "content": "", "tool_calls": [edit_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": edit_call["id"],
+                    "content": f"Memory updated: {beta}\nMemory id: {memory_id}",
+                },
+                {"role": "assistant", "content": "Updated."},
+                {"role": "user", "content": "Delete that memory."},
+                {"role": "assistant", "content": "", "tool_calls": [delete_call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": delete_call["id"],
+                    "content": f"Memory '{memory_id}' deleted",
+                },
+                {"role": "assistant", "content": "Deleted."},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL],
+            "generator": "targeted_efficiency_static_v2",
+            "metadata": {
+                "category": "memory_crud_one_call_followthrough",
+                "target_issue": "avoid_incomplete_manage_memory_add_first_call_in_crud_context",
+                "expected_tool_calls_per_turn": [1, 1, 1],
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_memory_crud", row)
+        rows.append(row)
+    return rows
+
+
+def task_rows() -> list[dict[str, Any]]:
+    specs = [
+        ("Daily email triage checkpoint", "Summarize unread important email each morning.", "daily", "09:00"),
+        ("Weekly invoice reminder", "Remind me to review open invoices every Monday.", "weekly", "08:30"),
+        ("Runpod spend check", "Check the Runpod budget note and remind me if follow-up is needed.", "daily", "18:00"),
+        ("Calendar prep", "Prepare a short next-day calendar summary.", "daily", "20:00"),
+        ("Research queue sweep", "Review saved research tasks and list blockers.", "weekly", "10:00"),
+        ("Document cleanup reminder", "Remind me to tidy stale editor documents.", "weekly", "16:00"),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (name, prompt, schedule, scheduled_time) in enumerate(specs):
+        user = f"Create a scheduled task named {name} that runs {schedule} at {scheduled_time} UTC and has prompt: {prompt}"
+        args = {
+            "action": "create",
+            "name": name,
+            "prompt": prompt,
+            "task_type": "llm",
+            "schedule": schedule,
+            "scheduled_time": scheduled_time,
+            "output_target": "chat",
+        }
+        call = tool_call("manage_tasks", args, f"task_create_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {"role": "tool", "tool_call_id": call["id"], "content": f"Task created: {name}"},
+                {"role": "assistant", "content": "Task created."},
+            ],
+            "tools": [MANAGE_TASKS_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "task_create_clean_schema",
+                "target_issue": "avoid_loose_task_create_fields",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_task", row)
+        rows.append(row)
+    return rows
+
+
+def web_followup_rows() -> list[dict[str, Any]]:
+    first_answers = [
+        (
+            "What are some good sites for public domain art?",
+            "Good public domain art sources include Wikimedia Commons, The Met Open Access, Rijksmuseum Rijksstudio, Smithsonian Open Access, and the Library of Congress.",
+            "send links",
+            "public domain art Wikimedia Commons Met Open Access Rijksmuseum Smithsonian Library of Congress official links",
+        ),
+        (
+            "What are good places to find old maps online?",
+            "Good places include the Library of Congress, David Rumsey Map Collection, Wikimedia Commons, and Old Maps Online.",
+            "sned links for those",
+            "old maps Library of Congress David Rumsey Wikimedia Commons Old Maps Online official links",
+        ),
+        (
+            "Where can I find free classical music recordings?",
+            "Try Musopen, Wikimedia Commons audio, Internet Archive, and IMSLP for public domain scores and recordings.",
+            "for the websites",
+            "free classical music recordings Musopen Wikimedia Commons Internet Archive IMSLP official links",
+        ),
+        (
+            "What are reliable sources for public domain books?",
+            "Project Gutenberg, Internet Archive, HathiTrust, Standard Ebooks, and Open Library are useful starting points.",
+            "send the links",
+            "public domain books Project Gutenberg Internet Archive HathiTrust Standard Ebooks Open Library official links",
+        ),
+    ]
+    rows: list[dict[str, Any]] = []
+    for i, (user1, answer1, user2, query) in enumerate(first_answers):
+        call = tool_call("web_search", {"query": query}, f"web_followup_{i}")
+        row = {
+            "messages": [
+                {"role": "user", "content": user1},
+                {"role": "assistant", "content": answer1},
+                {"role": "user", "content": user2},
+                {"role": "assistant", "content": "", "tool_calls": [call]},
+                {
+                    "role": "tool",
+                    "tool_call_id": call["id"],
+                    "content": "Search results include official source pages with URLs.",
+                },
+                {"role": "assistant", "content": "Here are the official links."},
+            ],
+            "tools": [WEB_SEARCH_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "contextual_web_followup",
+                "target_issue": "use_web_search_for_links_followup_after_general_answer",
+                "expected_tool_calls": 1,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_web", row)
+        rows.append(row)
+    return rows
+
+
+def no_tool_rows() -> list[dict[str, Any]]:
+    pairs = [
+        ("hi", "Hi! How can I help?"),
+        ("who are you?", "I am Odysseus, your assistant."),
+        ("what does VAT stand for?", "VAT stands for value-added tax."),
+        ("where is Sweden on a map?", "Sweden is in Northern Europe, on the Scandinavian Peninsula between Norway and Finland."),
+        ("thanks", "You're welcome."),
+        ("what is public domain art?", "Public domain art is artwork whose copyright has expired, was waived, or never applied, so it can generally be reused freely."),
+        ("do you remember what VAT stands for?", "VAT stands for value-added tax."),
+        ("what should I remember before buying public domain art?", "Check the source, copyright status, license notes, and whether the site marks the work as public domain."),
+        ("remind me what Sweden is bordered by", "Sweden borders Norway to the west and Finland to the east."),
+        ("what does it mean to remember something in a computer?", "It means storing data or information so it can be retrieved later."),
+        ("do you remember what HTML stands for?", "HTML stands for HyperText Markup Language."),
+        ("remind me what CPU means", "CPU means central processing unit."),
+        ("can you remind me where Italy is?", "Italy is in Southern Europe, extending into the Mediterranean Sea."),
+        ("what should I remember when choosing a password?", "Use a long unique password, avoid reuse, and store it in a password manager."),
+        ("remember in Python, what does len do?", "In Python, len returns the number of items in an object such as a string, list, or dictionary."),
+        ("what did I ask you to remember in this sentence: remember to check licenses?", "You asked about the phrase 'remember to check licenses'; it means to keep license checks in mind."),
+        ("does remember always mean save to memory?", "No. It can mean recall, explain, or keep something in mind; only explicit save/store requests need memory tools."),
+        ("remind me what public domain means", "Public domain means a work is not restricted by copyright and can generally be used freely."),
+        ("what should I remember before using AI-generated images?", "Check licensing, provenance, consent issues, and whether the output matches your intended use."),
+        ("do you remember the capital of Sweden?", "The capital of Sweden is Stockholm."),
+        ("remind me how to spell calendar", "Calendar is spelled c-a-l-e-n-d-a-r."),
+        ("what does a computer memory store?", "Computer memory stores data and instructions for use by programs."),
+    ]
+    rows: list[dict[str, Any]] = []
+    for user, assistant in pairs:
+        row = {
+            "messages": [
+                {"role": "user", "content": user},
+                {"role": "assistant", "content": assistant},
+            ],
+            "tools": [MANAGE_MEMORY_TOOL, MANAGE_TASKS_TOOL, WEB_SEARCH_TOOL],
+            "generator": "targeted_efficiency_static_v1",
+            "metadata": {
+                "category": "no_tool_boundary",
+                "target_issue": "avoid_overcalling_tools_on_general_chat",
+                "expected_tool_calls": 0,
+            },
+        }
+        row["uuid"] = stable_id("ody_eff_boundary", row)
+        rows.append(row)
+    return rows
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(row)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--out-dir",
+        default="/home/pewds/odysseus-finetune/data/targeted_efficiency/odysseus_tool_efficiency_v1_20260820",
+    )
+    parser.add_argument("--val-every", type=int, default=5)
+    args = parser.parse_args()
+
+    rows = memory_rows() + memory_crud_rows() + task_rows() + web_followup_rows() + no_tool_rows()
+    train, val = split_rows(rows, args.val_every)
+    out_dir = Path(args.out_dir)
+    write_jsonl(out_dir / "train.jsonl", train)
+    write_jsonl(out_dir / "val.jsonl", val)
+    write_jsonl(out_dir / "all.jsonl", rows)
+    manifest = {
+        "name": out_dir.name,
+        "total_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "source_eval": "/home/pewds/odysseus-cookbook-fresh/data/evals/qwen35_9b_v44_memory_onecall_efficiency_20260820_202333.json",
+        "categories": {
+            category: sum(1 for row in rows if row["metadata"]["category"] == category)
+            for category in sorted({row["metadata"]["category"] for row in rows})
+        },
+        "acceptance_target": (
+            "memory_add_one_call_efficiency should reach 2/2 efficiency; "
+            "memory_crud_followthrough should reach 3/3 correctness and 3/3 efficiency; "
+            "memory_add_wording_variants_efficiency should reach 6/6 correctness and 6/6 efficiency; "
+            "memory_no_tool_boundary should reach 4/4 no-tool correctness; "
+            "full contextual correctness should remain 42/42 or better."
+        ),
+    }
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    print(json.dumps(manifest, indent=2, ensure_ascii=True))
+
+
+if __name__ == "__main__":
+    main()

scripts/build_odysseus_v54_live_gap_teacher_sft.py

@@ -0,0 +1,557 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib import request
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_OUT = Path("/home/pewds/odysseus-finetune/data/teacher_live_gaps/odysseus_v54_live_gap_teacher_20260821")
+DEFAULT_EVAL_OUT = REPO_ROOT / "data/evals/ody_v54_live_gap_teacher_heldout_20260821/cases.json"
+
+
+WEB_SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "web_search",
+        "description": "Search the web for current or source-backed information.",
+        "parameters": {
+            "type": "object",
+            "properties": {"query": {"type": "string"}},
+            "required": ["query"],
+        },
+    },
+}
+
+
+CALENDAR_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "manage_calendar",
+        "description": "Create, update, list, and delete calendar events.",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "action": {"type": "string"},
+                "summary": {"type": "string"},
+                "dtstart": {"type": "string"},
+                "dtend": {"type": "string"},
+            },
+            "required": ["action"],
+        },
+    },
+}
+
+
+FAMILIES: list[dict[str, Any]] = [
+    {
+        "name": "web_synthesis_animal_foam",
+        "train_count": 48,
+        "heldout_count": 12,
+        "instruction": (
+            "Public web lookup questions about animals producing foam, bubbles, froth, or mucus. "
+            "The assistant must search once with specific biological terms and then synthesize a concise cause/explanation. "
+            "Rows should include snails often, but also a few other small animal examples. Final answers must mention the relevant mechanism, "
+            "not dump links or say evidence is insufficient when the simulated evidence is enough."
+        ),
+    },
+    {
+        "name": "web_retry_after_weak_results",
+        "train_count": 24,
+        "heldout_count": 8,
+        "instruction": (
+            "The first web_search result is weak, dictionary-like, or off-topic. The assistant should make one improved web_search "
+            "with better scientific/current terms, then synthesize the answer. Focus on failures where a generic query found dictionary/noise."
+        ),
+    },
+    {
+        "name": "calendar_ambiguous_time_boundary",
+        "train_count": 16,
+        "heldout_count": 6,
+        "instruction": (
+            "Calendar requests with relative dates and ambiguous times. If the user says 8pm/8 PM/evening at 8, create or update 20:00. "
+            "If the user only says 'at 8' without AM/PM or context, ask a short clarification instead of guessing 8pm."
+        ),
+    },
+]
+
+
+def stable_id(prefix: str, obj: dict[str, Any]) -> str:
+    payload = json.dumps(obj, sort_keys=True, ensure_ascii=True)
+    return prefix + "_" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+
+
+def clean_text(value: Any) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()
+
+
+def clean_terms(value: Any) -> list[str]:
+    if isinstance(value, str):
+        text = clean_text(value)
+        return [text] if text else []
+    if isinstance(value, list):
+        return [clean_text(item) for item in value if clean_text(item)]
+    return []
+
+
+def tool_call(name: str, arguments: dict[str, Any], suffix: str) -> dict[str, Any]:
+    return {
+        "id": f"call_{suffix}",
+        "type": "function",
+        "function": {
+            "name": name,
+            "arguments": json.dumps(arguments, separators=(",", ":"), ensure_ascii=True),
+        },
+    }
+
+
+def deepseek_endpoint() -> dict[str, str]:
+    api_key = os.environ.get("DEEPSEEK_API_KEY", "").strip()
+    if api_key:
+        return {
+            "name": "env-deepseek",
+            "base_url": os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
+            "api_key": api_key,
+            "cached_models": os.environ.get("DEEPSEEK_MODEL", "deepseek-chat"),
+        }
+    db_path = REPO_ROOT / "data/app.db"
+    conn = sqlite3.connect(str(db_path))
+    try:
+        conn.row_factory = sqlite3.Row
+        row = conn.execute(
+            """
+            SELECT name, base_url, api_key, cached_models
+            FROM model_endpoints
+            WHERE lower(name) LIKE '%deepseek%'
+              AND COALESCE(is_enabled, 0) = 1
+              AND COALESCE(api_key, '') != ''
+            ORDER BY updated_at DESC
+            LIMIT 1
+            """
+        ).fetchone()
+        if not row:
+            raise RuntimeError("no enabled DeepSeek endpoint with API key")
+        return {
+            "name": row["name"],
+            "base_url": row["base_url"],
+            "api_key": row["api_key"],
+            "cached_models": row["cached_models"] or "",
+        }
+    finally:
+        conn.close()
+
+
+def call_deepseek(endpoint: dict[str, str], prompt: dict[str, Any], max_tokens: int = 8000) -> dict[str, Any]:
+    model = "deepseek-chat"
+    try:
+        cached = json.loads(endpoint.get("cached_models") or "[]")
+        if cached:
+            model = cached[0]
+    except json.JSONDecodeError:
+        if endpoint.get("cached_models"):
+            model = endpoint["cached_models"]
+    payload = {
+        "model": model,
+        "messages": [
+            {"role": "system", "content": "Return strict JSON only. No markdown or commentary."},
+            {"role": "user", "content": json.dumps(prompt, ensure_ascii=False)},
+        ],
+        "temperature": 0.65,
+        "max_tokens": max_tokens,
+    }
+    req = request.Request(
+        endpoint["base_url"].rstrip("/") + "/chat/completions",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {endpoint['api_key']}"},
+        method="POST",
+    )
+    with request.urlopen(req, timeout=120) as resp:
+        body = json.loads(resp.read().decode("utf-8"))
+    content = body["choices"][0]["message"]["content"]
+    cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", (content or "").strip(), flags=re.I | re.S)
+    if not cleaned.startswith("{"):
+        match = re.search(r"\{.*\}", cleaned, flags=re.S)
+        if match:
+            cleaned = match.group(0)
+    return {"model": model, "content": json.loads(cleaned)}
+
+
+def teacher_prompt(family: dict[str, Any], count: int, batch: int) -> dict[str, Any]:
+    return {
+        "task": "Generate Odysseus SFT specs for live tool-use gaps.",
+        "current_state": {
+            "model": "qwen35-9b-tool-router-v53-web-repair",
+            "live_gap_eval": "DeepSeek-heldout v3 rescored 38/41",
+            "real_failures": [
+                "Web search often searches but returns a weak snippet dump instead of a concise explanation.",
+                "If search evidence is weak/noisy, the route should search again with better terms instead of giving up or dumping links.",
+                "Calendar generated heldout contained an ambiguous 'tomorrow at 8' case; do not teach that bare 8 means 8pm.",
+            ],
+        },
+        "family": family["name"],
+        "count": count,
+        "batch": batch,
+        "family_instruction": family["instruction"],
+        "requirements": [
+            "Return JSON object with key rows: list.",
+            "Return exactly count rows.",
+            "Every row must have user and final.",
+            "Web rows need ideal_query, evidence, query_must_include, answer_must_include.",
+            "Retry rows also need bad_query and bad_evidence.",
+            "Calendar rows need calendar_args for tool rows or no_tool=true for clarification rows.",
+            "Use varied casual wording and typos, but do not include private names, email addresses, or secrets.",
+            "Final answers must be concise and user-facing.",
+            "Never include raw source blocks, WEB SEARCH RESULTS, or link dumps in final.",
+        ],
+        "target_examples_not_to_copy": [
+            "Look up why snails produce foam and give me a short explanation.",
+            "Why do snails make foam? Check online and explain briefly.",
+            "Search the web for the reason snails bubble up, then summarize it concisely.",
+            "Move EVENT to tomorrow at 8 PM.",
+            "Move EVENT to tomorrow at 8.",
+        ],
+    }
+
+
+def valid_spec(family: str, item: Any) -> bool:
+    if not isinstance(item, dict):
+        return False
+    user = clean_text(item.get("user"))
+    final = clean_text(item.get("final"))
+    if len(user.split()) < 4 or len(user) > 240 or not final:
+        return False
+    if any(bad in final for bad in ("WEB SEARCH RESULTS", "```sources", "Here are links")):
+        return False
+    if family.startswith("web_"):
+        if not clean_text(item.get("ideal_query")):
+            return False
+        if family == "web_retry_after_weak_results" and not clean_text(item.get("bad_query")):
+            return False
+    if family == "calendar_ambiguous_time_boundary":
+        if item.get("no_tool"):
+            return bool(re.search(r"\b(?:am|pm|morning|evening|clarify|which)\b", final, re.I))
+        args = item.get("calendar_args")
+        if not isinstance(args, dict):
+            return False
+        action = str(args.get("action") or "").lower()
+        if action not in {"create_event", "update_event"}:
+            return False
+        return bool(args.get("summary") and args.get("dtstart") and args.get("dtend"))
+    return True
+
+
+def deterministic_calendar_specs() -> list[dict[str, Any]]:
+    tool_specs = [
+        ("move the meeting to tomorrow at 8 PM", "update_event", "meeting", "2026-08-23T20:00:00", "Done. The meeting is moved to tomorrow at 8:00 PM."),
+        ("reschedule dinner to tomorrow at 8 in the evening", "update_event", "dinner", "2026-08-23T20:00:00", "Done. Dinner is rescheduled to tomorrow at 8:00 PM."),
+        ("shift the appointment to tomorrow at 8 PM", "update_event", "appointment", "2026-08-23T20:00:00", "Done. The appointment is moved to tomorrow at 8:00 PM."),
+        ("schedule a call for Friday at 8 PM", "create_event", "Call", "2026-08-28T20:00:00", "Scheduled the call for Friday at 8:00 PM."),
+        ("add lunch with Sam next Monday at 8pm", "create_event", "Lunch with Sam", "2026-08-24T20:00:00", "Scheduled lunch with Sam for next Monday at 8:00 PM."),
+        ("book dinner Friday evening at 8", "create_event", "Dinner", "2026-08-28T20:00:00", "Scheduled dinner for Friday at 8:00 PM."),
+        ("move the party to tomorrow evening at 8", "update_event", "party", "2026-08-23T20:00:00", "Done. The party is moved to tomorrow at 8:00 PM."),
+        ("change my workout event to tomorrow at 8pm", "update_event", "workout", "2026-08-23T20:00:00", "Done. The workout is moved to tomorrow at 8:00 PM."),
+    ]
+    specs: list[dict[str, Any]] = []
+    for user, action, summary, start, final in tool_specs:
+        hour = int(start[11:13]) + 1
+        specs.append({
+            "user": user,
+            "calendar_args": {
+                "action": action,
+                "summary": summary,
+                "dtstart": start,
+                "dtend": start[:11] + f"{hour:02d}" + start[13:],
+            },
+            "tool_result": "AI: Calendar updated.",
+            "final": final,
+        })
+    for user in [
+        "move meeting to tomorrow at 8",
+        "can u move my workout to tmrw at 8?",
+        "book dinner for Friday at 8?",
+        "shift the appointment to tomorrow at 8",
+        "move the event to tomorrow at 8",
+        "reschedule lunch next Monday at 8",
+        "change the appointment to the day after tomorrow at 8",
+        "push the call to Friday at 8",
+        "put the dentist appointment tomorrow at 8",
+        "move my calendar event to 8 tomorrow",
+        "schedule dinner at 8",
+        "set the meeting for 8 tomorrow",
+        "can we do the appointment at 8",
+        "change it to 8",
+    ]:
+        specs.append({
+            "user": user,
+            "no_tool": True,
+            "final": "Do you mean 8 AM or 8 PM?",
+        })
+    return specs
+
+
+def build_sft_row(family: str, idx: int, spec: dict[str, Any], split: str) -> dict[str, Any]:
+    user = clean_text(spec["user"])
+    final = clean_text(spec["final"])
+    messages: list[dict[str, Any]] = [{"role": "user", "content": user}]
+    tools: list[dict[str, Any]] = []
+    expected_calls = 0
+    if family == "web_retry_after_weak_results":
+        bad = tool_call("web_search", {"query": clean_text(spec["bad_query"])}, f"{family}_{idx}_bad")
+        good = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}_good")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [bad]},
+            {"role": "tool", "tool_call_id": bad["id"], "content": clean_text(spec.get("bad_evidence"))},
+            {"role": "assistant", "content": "", "tool_calls": [good]},
+            {"role": "tool", "tool_call_id": good["id"], "content": clean_text(spec.get("evidence"))},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [WEB_SEARCH_TOOL]
+        expected_calls = 2
+    elif family.startswith("web_"):
+        call = tool_call("web_search", {"query": clean_text(spec["ideal_query"])}, f"{family}_{idx}")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": clean_text(spec.get("evidence"))},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [WEB_SEARCH_TOOL]
+        expected_calls = 1
+    elif family == "calendar_ambiguous_time_boundary" and spec.get("no_tool"):
+        messages.append({"role": "assistant", "content": final})
+    else:
+        args = dict(spec["calendar_args"])
+        call = tool_call("manage_calendar", args, f"{family}_{idx}")
+        messages.extend([
+            {"role": "assistant", "content": "", "tool_calls": [call]},
+            {"role": "tool", "tool_call_id": call["id"], "content": clean_text(spec.get("tool_result")) or "AI: Calendar updated."},
+            {"role": "assistant", "content": final},
+        ])
+        tools = [CALENDAR_TOOL]
+        expected_calls = 1
+
+    row = {
+        "messages": messages,
+        "tools": tools,
+        "generator": "deepseek_teacher_v54_live_gap",
+        "metadata": {
+            "category": family,
+            "split": split,
+            "expected_tool_calls": expected_calls,
+            "query_must_include": clean_terms(spec.get("query_must_include")),
+            "answer_must_include": clean_terms(spec.get("answer_must_include")),
+            "source_failures": [
+                "deepseek_v3_web_synthesis_00",
+                "deepseek_v3_web_synthesis_03",
+                "deepseek_v3_calendar_move_02",
+            ],
+        },
+    }
+    row["uuid"] = stable_id("ody_v54_live_gap", row)
+    return row
+
+
+def build_eval_case(family: str, idx: int, spec: dict[str, Any]) -> dict[str, Any]:
+    case: dict[str, Any] = {
+        "id": f"v54_live_gap_{family}_{idx:02d}",
+        "kind": "calendar" if family == "calendar_ambiguous_time_boundary" else "web",
+        "user": clean_text(spec["user"]),
+        "deepseek_family": family,
+        "forbidden_final": ["WEB SEARCH RESULTS", "```sources", "Here are links for that topic"],
+    }
+    if family.startswith("web_"):
+        user_lower = case["user"].lower()
+        if re.search(r"\b(?:search|look\s+up|check\s+online|web|find\s+out|google)\b", user_lower):
+            case["expect_first_tool"] = "web_search"
+        if family != "web_retry_after_weak_results":
+            case["max_web_searches"] = 1
+        if family == "web_synthesis_animal_foam":
+            case["must_answer_any"] = ["mucus", "foam", "bubble", "froth", "slime"]
+            case["must_answer_any_2"] = [
+                "stress",
+                "defense",
+                "irritat",
+                "moisture",
+                "predator",
+                "protect",
+                "osmosis",
+                "salt",
+            ]
+        else:
+            terms = clean_terms(spec.get("answer_must_include"))
+            expanded: list[str] = []
+            for term in terms:
+                expanded.extend(part.strip() for part in re.split(r"[,/]| or ", term) if part.strip())
+            if expanded:
+                case["must_answer_any"] = expanded[:8]
+    elif spec.get("no_tool"):
+        case["expect_no_tool"] = True
+        case["forbidden_tools"] = ["manage_calendar"]
+        case["must_answer_any"] = ["AM", "PM", "morning", "evening", "clarify", "which"]
+    else:
+        case["expect_first_tool"] = "manage_calendar"
+    return case
+
+
+def split_rows(rows: list[dict[str, Any]], val_every: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    train: list[dict[str, Any]] = []
+    val: list[dict[str, Any]] = []
+    for idx, row in enumerate(rows):
+        (val if idx % val_every == val_every - 1 else train).append(row)
+    return train, val
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("".join(json.dumps(row, ensure_ascii=True) + "\n" for row in rows), encoding="utf-8")
+
+
+def file_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT)
+    parser.add_argument("--eval-out", type=Path, default=DEFAULT_EVAL_OUT)
+    parser.add_argument("--val-every", type=int, default=6)
+    args = parser.parse_args()
+
+    endpoint = deepseek_endpoint()
+    started = time.time()
+    previous_manifest = args.out_dir / "manifest.json"
+    model = ""
+    if previous_manifest.exists():
+        with contextlib.suppress(Exception):
+            model = str(json.loads(previous_manifest.read_text(encoding="utf-8")).get("model") or "")
+    if not model:
+        model = "deepseek-chat"
+    raw: dict[str, Any] = {}
+    rows: list[dict[str, Any]] = []
+    heldout: list[dict[str, Any]] = []
+    seen_users: set[str] = set()
+
+    for family in FAMILIES:
+        needed = family["train_count"] + family["heldout_count"]
+        generated: list[dict[str, Any]] = []
+        valid: list[dict[str, Any]] = []
+        cache_path = args.out_dir / f"raw_{family['name']}.json"
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        if family["name"] == "calendar_ambiguous_time_boundary":
+            generated = deterministic_calendar_specs()
+            valid = [item for item in generated if valid_spec(family["name"], item)]
+            cache_path.write_text(
+                json.dumps({"family": family["name"], "rows": generated, "source": "deterministic_schema_valid"}, ensure_ascii=False, indent=2) + "\n",
+                encoding="utf-8",
+            )
+        elif cache_path.exists():
+            cached = json.loads(cache_path.read_text(encoding="utf-8"))
+            generated = cached.get("rows", []) if isinstance(cached, dict) else []
+            valid = [item for item in generated if valid_spec(family["name"], item)]
+        for batch in range(1, 16):
+            if len(valid) >= needed:
+                break
+            response = call_deepseek(endpoint, teacher_prompt(family, min(18, needed + 4), batch))
+            model = response["model"]
+            batch_rows = response["content"].get("rows", [])
+            if isinstance(batch_rows, list):
+                generated.extend(batch_rows)
+                valid = [item for item in generated if valid_spec(family["name"], item)]
+                cache_path.write_text(
+                    json.dumps({"family": family["name"], "rows": generated}, ensure_ascii=False, indent=2) + "\n",
+                    encoding="utf-8",
+                )
+        raw[family["name"]] = generated
+        train_count = 0
+        heldout_count = 0
+        for item in valid:
+            key = clean_text(item["user"]).lower()
+            if key in seen_users:
+                continue
+            seen_users.add(key)
+            if train_count < family["train_count"]:
+                rows.append(build_sft_row(family["name"], train_count, item, "train_or_val"))
+                train_count += 1
+            elif heldout_count < family["heldout_count"]:
+                heldout.append(build_eval_case(family["name"], heldout_count, item))
+                heldout_count += 1
+            if train_count >= family["train_count"] and heldout_count >= family["heldout_count"]:
+                break
+        if train_count < family["train_count"] or heldout_count < family["heldout_count"]:
+            raise RuntimeError(
+                f"{family['name']} valid rows short: train {train_count}/{family['train_count']}, "
+                f"heldout {heldout_count}/{family['heldout_count']}"
+            )
+
+    train, val = split_rows(rows, args.val_every)
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    write_jsonl(args.out_dir / "train.jsonl", train)
+    write_jsonl(args.out_dir / "val.jsonl", val)
+    write_jsonl(args.out_dir / "all.jsonl", rows)
+    (args.out_dir / "raw_teacher.json").write_text(json.dumps(raw, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    args.eval_out.parent.mkdir(parents=True, exist_ok=True)
+    eval_payload = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "generator": Path(__file__).name,
+        "provider": endpoint["name"],
+        "model": model,
+        "source": "V53 DeepSeek-heldout live-gap failures",
+        "cases": heldout,
+    }
+    args.eval_out.write_text(json.dumps(eval_payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    manifest = {
+        "name": args.out_dir.name,
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "provider": endpoint["name"],
+        "model": model,
+        "elapsed_seconds": round(time.time() - started, 3),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(heldout),
+        "categories": {family["name"]: sum(1 for row in rows if row["metadata"]["category"] == family["name"]) for family in FAMILIES},
+        "heldout_categories": {family["name"]: sum(1 for case in heldout if case["deepseek_family"] == family["name"]) for family in FAMILIES},
+        "source_eval": "data/evals/ody_everyday_deepseek_heldout_v53_current_20260821_1508_dynamic_calendar_rescored/actual_results.json",
+        "acceptance_target": (
+            "Train as a narrow V54 top-up only after reviewing rows. Promote only if V54 passes live-hard, "
+            "DeepSeek-heldout rescored cases, V54 live-gap heldout, and old CRUD regression."
+        ),
+        "files": {
+            "train": str(args.out_dir / "train.jsonl"),
+            "val": str(args.out_dir / "val.jsonl"),
+            "all": str(args.out_dir / "all.jsonl"),
+            "raw_teacher": str(args.out_dir / "raw_teacher.json"),
+            "heldout_eval": str(args.eval_out),
+        },
+    }
+    for key, value in list(manifest["files"].items()):
+        manifest[f"{key}_sha256"] = file_sha256(Path(value))
+    (args.out_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+
+    print(json.dumps({
+        "out_dir": str(args.out_dir),
+        "eval_out": str(args.eval_out),
+        "total_sft_rows": len(rows),
+        "train_rows": len(train),
+        "val_rows": len(val),
+        "heldout_cases": len(heldout),
+        "categories": manifest["categories"],
+        "heldout_categories": manifest["heldout_categories"],
+        "model": model,
+    }, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())