Squash Odysseus development history

.dockerignore

@@ -30,6 +30,8 @@ secrets.env~
 .idea/
 dev-docs/
 docs/
+website/
+assets/branding/
 *.md
 *.db
 *.sqlite

.env.example

@@ -67,6 +67,11 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Auth & Security
 # ============================================================
 
+# Optional backend workspace used automatically by the WebUI when no workspace
+# is saved in the browser. This must be a directory visible to the backend;
+# with host-workspace mapping, a host path is translated before vetting.
+# ODYSSEUS_WORKSPACE_DEFAULT=/workspace/project
+
 # Enable authentication (default: true)
 # AUTH_ENABLED=true
 
@@ -76,12 +81,32 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Change this if another local service already uses 7000 (macOS AirPlay often does).
 # APP_PORT=7000
 
+# Optional HTTP address advertised in companion/mobile pairing codes. Set this
+# when Docker would otherwise advertise a container address or loopback. Use a
+# LAN or Tailscale IPv4 address, a single-label hostname, or an mDNS *.local
+# name that the phone can reach. HTTPS and public hostnames are not supported
+# by the current companion client. Do not include credentials, a path, query,
+# or fragment.
+# COMPANION_BASE_URL=http://192.168.1.50:7000
+
 # Development-only auth bypass for loopback requests.
 # Keep false for Docker, LAN, reverse proxy, and any shared deployment.
 # LOCALHOST_BYPASS=false
 
-# Mark session cookies Secure. Set true when Odysseus is served through HTTPS
-# by a trusted reverse proxy or private access gateway.
+# Skip the external-context exact-approval pause for unattended local agents.
+# Keep false for shared or internet-exposed deployments.
+# ODYSSEUS_UNATTENDED_MODE=false
+
+# Optional post-external-context tool approval gate. Off by default because it
+# can block normal agent work; enable only for deployments that want this fence.
+# ODYSSEUS_TOOL_APPROVAL_GATE=0
+
+# Mark session cookies Secure. Left unset, this follows the request scheme:
+# an HTTPS login gets a Secure cookie, a plain-HTTP one does not. Set true to
+# force it on, or false to force it off while you still serve plain HTTP.
+# Upgrading: this used to default to false. Drop a leftover SECURE_COOKIES=false
+# from your .env unless you still need that escape hatch — it keeps HTTPS logins
+# on a non-Secure cookie.
 # SECURE_COOKIES=true
 
 # Optional: pre-seed the first admin password during setup.
@@ -151,6 +176,21 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Local HTTP setups may use the callback URL inferred by the application.
 # GOOGLE_OAUTH_REDIRECT_URI=https://your-domain.com/api/email/oauth/google/callback
 
+# Origin the MCP OAuth callback is sent back to, for remote (Streamable HTTP)
+# MCP servers that register it dynamically. Defaults to http://localhost:$APP_PORT,
+# which is right only when you reach Odysseus directly on that port. Set it for
+# HTTPS, reverse-proxy, hosted, and Docker installs — inside the container the
+# app always listens on 7000 and cannot see the host port map, so the default is
+# wrong there whenever APP_PORT is not 7000.
+#
+# Not for Google MCP servers. Those use Desktop App credentials, and Google only
+# accepts loopback redirect URIs for that client type, so a public origin here is
+# rejected with redirect_uri_mismatch. Leave it unset for a Google-only install:
+# the loopback default is what Google wants, and remote users finish through the
+# paste-back page, which never has to load the redirect.
+# https://developers.google.com/identity/protocols/oauth2/native-app
+# OAUTH_REDIRECT_BASE_URL=https://your-domain.com
+
 # ============================================================
 # Misc
 # ============================================================
@@ -189,6 +229,7 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400       # email compose attachment (25 MB)
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
+# ODYSSEUS_TTS_CACHE_MAX_BYTES=524288000                 # TTS cache (500 MB)
 
 # ============================================================
 # Host Docker access (explicit opt-in)
@@ -210,6 +251,37 @@ SEARXNG_INSTANCE=http://localhost:8080
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
 # COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
 
+# ============================================================
+# Host workspace access (explicit opt-in)
+# ============================================================
+# Docker installs normally see only the container filesystem and /app/data.
+# Enable this when the agent should edit a real host workspace like Codex.
+# This is high-trust: the mounted tree is writable by the Odysseus container.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+# ODYSSEUS_HOST_WORKSPACE_DIR=/home/you
+# ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+#
+# Host workspace access can be combined with host Docker access and GPU overlays:
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-docker.yml
+
+# ============================================================
+# Host network access (explicit opt-in, Linux Docker)
+# ============================================================
+# Docker bridge networking hides some host/LAN/VPN behavior from the agent:
+# mDNS, some LAN discovery, local VPN/Tailscale state, and host namespace
+# assumptions may differ from native Codex. Enable this only for high-trust
+# local installs where the Odysseus container should share the host network.
+#
+# With host networking, Docker port publishing is disabled and the app listens
+# directly on APP_PORT. The bundled SearXNG/Chroma services stay in Docker and
+# are reached through their host-published loopback ports.
+# COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+# APP_BIND=127.0.0.1
+# APP_PORT=7000
+# ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE=http://127.0.0.1:8080
+# ODYSSEUS_HOST_NETWORK_CHROMADB_HOST=127.0.0.1
+# ODYSSEUS_HOST_NETWORK_CHROMADB_PORT=8100
+
 # ============================================================
 # GPU support (Docker Compose)
 # ============================================================
@@ -238,3 +310,5 @@ SEARXNG_INSTANCE=http://localhost:8080
 
 # APP_DATA_DIR=./data
 # APP_LOGS_DIR=./logs
+# Maximum serialized layered photo-editor draft size (default: 256 MiB).
+ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=268435456

.gitattributes

@@ -15,6 +15,13 @@ docker/entrypoint.sh text eol=lf
 *.cmd text eol=crlf
 *.bat text eol=crlf
 
+# Vendored third-party bundles in static/lib/ are published minified artifacts
+# and must stay byte-identical to what npm ships — stripping trailing whitespace
+# to satisfy `git diff --check` would desync them from the upstream release. Turn
+# the whitespace check off for that tree instead, and keep the bundles out of
+# GitHub's language statistics.
+static/lib/** -whitespace linguist-vendored
+
 # Binary assets — never normalize.
 *.png  binary
 *.jpg  binary

.github/CODEOWNERS

@@ -6,4 +6,4 @@
 # A per-area ownership map (security/auth, CI, frontend, agent internals, with
 # multiple named owners per line) is being worked out in issue #593; once
 # agreed it replaces this file. Until then, required reviews and the security
-# CI gate (docs/security-ci.md) remain in force via branch protection.
+# CI gate (website/security-ci.md) remain in force via branch protection.

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -26,6 +26,18 @@ body:
         - label: I am running the latest code from the `dev` branch (the default branch you get on clone, where fixes land first) and the bug still reproduces there. Please `git pull` the latest `dev` before filing.
           required: true
 
+  - type: input
+    id: revision
+    attributes:
+      label: Odysseus Revision
+      description: |
+        From the repository root (on the host when using Docker), run
+        `git show -s --abbrev=12 --format='%h (%cs)' HEAD`
+        and paste the output exactly.
+      placeholder: "1fef4929cf1d (2026-08-11)"
+    validations:
+      required: true
+
   - type: dropdown
     id: install-method
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -8,8 +8,8 @@ body:
       value: |
         **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues)
         and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first.
-        The [roadmap](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md) is directional rather than a complete backlog.
-        Feature requests that duplicate an existing issue or accepted proposal may be closed as duplicates.
+        Feature requests that duplicate [ROADMAP.md](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md)
+        or an existing open issue will be closed as duplicates.
 
         If your idea needs community input before it becomes a concrete proposal,
         start a [discussion](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas) instead.

.github/pull_request_template.md

@@ -28,6 +28,7 @@ Fixes #
 - [ ] This PR targets `dev`
 - [ ] My changes are limited to the scope described above — no unrelated refactors or whitespace changes mixed in.
 - [ ] I actually ran the app (`docker compose up` or `uvicorn app:app`) and verified the change works end-to-end. Type-checks and unit tests are not enough.
+- [ ] I did not run the app/runtime validation and stated that gap in **How to Test**. Leave this unchecked when the app-run box above is checked.
 
 ## How to Test
 

.github/scripts/check-issue-description.js

@@ -41,6 +41,14 @@ module.exports = async ({ github, context, core }) => {
       break;
 
     case 'bug': {
+      const revisionText = section('Odysseus Revision');
+      if (!/^[0-9a-f]{12} \(\d{4}-\d{2}-\d{2}\)$/i.test(revisionText)) {
+        failures.push(
+          '**Odysseus Revision** — paste the 12-character commit SHA and date, ' +
+          'for example `1fef4929cf1d (2026-08-11)`',
+        );
+      }
+
       if (!section('Install Method')) {
         failures.push('**Install Method** — select how you installed Odysseus');
       }
@@ -153,6 +161,16 @@ module.exports = async ({ github, context, core }) => {
     }
   }
 
+  const LABEL_BAD  = 'needs more info';
+  const LABEL_GOOD = 'ready for review';
+
+  // Closed issues are no longer awaiting review.
+  // This also prevents later edits to closed issues from restoring the label.
+  if (issue.state === 'closed') {
+    await dropLabel(LABEL_GOOD);
+    return;
+  }
+
   // ── Find existing bot comment to update in-place ──────────────────────────
   const MARKER = '<!-- issue-description-check -->';
   const { data: comments } = await github.rest.issues.listComments({
@@ -160,9 +178,6 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER));
 
-  const LABEL_BAD  = 'needs more info';
-  const LABEL_GOOD = 'ready for review';
-
   if (failures.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });

.github/scripts/check-pr-description.js

@@ -21,11 +21,11 @@ module.exports = async ({ github, context, core }) => {
     return strip(m?.[0].replace(new RegExp(`#+\\s+${heading}`, 'i'), '') ?? '');
   }
 
-  const problems = [];
+  const descriptionProblems = [];
 
   // 1. Summary must be filled in.
   if (section('Summary').length < 20) {
-    problems.push('**Summary** is empty or too short — describe what changed and why.');
+    descriptionProblems.push('**Summary** is empty or too short — describe what changed and why.');
   }
 
   // 2. Linked Issue must reference a real issue. Accept a bare #NNN, a closing
@@ -34,18 +34,18 @@ module.exports = async ({ github, context, core }) => {
   const linkedSection = section('Linked Issue');
   const hasIssueRef = /#\d+\b/.test(linkedSection) || /\/issues\/\d+/.test(linkedSection);
   if (!linkedSection || !hasIssueRef) {
-    problems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
+    descriptionProblems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
   }
 
   // 3. At least one Type of Change box must be checked.
   const typeBlock = body.match(/##\s+Type of Change[\s\S]*?(?=\n##\s|$)/i)?.[0] ?? '';
   if (!/- \[x\]/i.test(typeBlock)) {
-    problems.push('**Type of Change** — check at least one box.');
+    descriptionProblems.push('**Type of Change** — check at least one box.');
   }
 
   // 4. Duplicate-search checklist item must be checked.
   if (!/- \[x\] I searched/i.test(body)) {
-    problems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
+    descriptionProblems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
   }
 
   // 5. How to Test must contain enough real detail for a reviewer to act on.
@@ -53,7 +53,83 @@ module.exports = async ({ github, context, core }) => {
   //    code block — so we only require non-trivial content, not a specific shape.
   const howTo = section('How to Test');
   if (howTo.length < 30) {
-    problems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
+    descriptionProblems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
+  }
+
+  // Classify paths from GitHub's API. This workflow runs in the privileged base
+  // context, so it must never check out or execute code from the PR branch.
+  const changedFiles = await github.paginate(github.rest.pulls.listFiles, {
+    owner, repo, pull_number: prNum, per_page: 100,
+  });
+  const changedPaths = changedFiles.map(file => file.filename);
+
+  function isUiSensitivePath(filename) {
+    const path = filename.toLowerCase();
+    return path.startsWith('static/')
+      || path.startsWith('templates/')
+      || /\.(?:html?|css|svg)$/.test(path);
+  }
+
+  function isDocsOnlyPath(filename) {
+    const path = filename.toLowerCase();
+    return /\.(?:md|mdx|rst|adoc|txt)$/.test(path)
+      || (path.startsWith('docs/') && !isUiSensitivePath(path));
+  }
+
+  function isRuntimeSensitivePath(filename) {
+    const path = filename.toLowerCase();
+    if (isUiSensitivePath(path)) return false;
+    if (path.startsWith('tests/') || path.startsWith('.github/')) return false;
+    return /^(?:app\.py|routes\/|services\/|src\/|core\/|mcp_servers\/|scripts\/|docker\/)/.test(path)
+      || /^(?:dockerfile|docker-compose.*\.ya?ml|requirements(?:-optional)?\.txt|pyproject\.toml|setup\.py)$/.test(path)
+      || /\.(?:py|sh|ps1|bat)$/.test(path);
+  }
+
+  let classification = 'tooling';
+  if (changedPaths.some(isUiSensitivePath)) {
+    classification = 'UI-sensitive';
+  } else if (changedPaths.some(isRuntimeSensitivePath)) {
+    classification = 'backend/runtime';
+  } else if (changedPaths.length > 0 && changedPaths.every(isDocsOnlyPath)) {
+    classification = 'docs-only';
+  }
+
+  const appRan = /- \[x\]\s+I actually ran the app\b/i.test(body);
+  const appNotRun = /- \[x\]\s+I did not run the app\/runtime validation\b/i.test(body);
+  // Anchor on the wording, not the template's emphasis: a ticked box the author
+  // retyped without the surrounding ** renders identically on the PR page, so
+  // treating it as unchecked is invisible from their side. Matches the two
+  // attestations above, which already ignore formatting.
+  const screenshotChecked = /- \[x\]\s+[*_]{0,2}Screenshot or short clip[*_]{0,2}/i.test(body);
+  const screenshotSection = section('Screenshots / clips');
+  const hasVisualEvidence = /!\[[^\]]*\]\([^)]+\)|<(?:img|video|source)\b[^>]*(?:src|href)=|https?:\/\/[^\s)]+/i.test(screenshotSection);
+  const evidenceGaps = [];
+  let needsRuntimeValidation = false;
+  let needsVisualEvidence = false;
+
+  if (classification === 'backend/runtime' || classification === 'UI-sensitive') {
+    if (appRan && appNotRun) {
+      needsRuntimeValidation = true;
+      evidenceGaps.push('The app-run and explicit not-run boxes are both checked. Select the one state that is true.');
+    } else if (!appRan) {
+      needsRuntimeValidation = true;
+      if (appNotRun) {
+        evidenceGaps.push('The author explicitly reports that app/runtime validation was not performed.');
+      } else {
+        evidenceGaps.push('App/runtime validation is not author-attested. Check the run box only after running it, or check the explicit not-run box and describe the gap.');
+      }
+    }
+  }
+
+  if (classification === 'UI-sensitive') {
+    if (!screenshotChecked) {
+      needsVisualEvidence = true;
+      evidenceGaps.push('The screenshot/clip checkbox is not checked for this UI-sensitive change.');
+    }
+    if (!hasVisualEvidence) {
+      needsVisualEvidence = true;
+      evidenceGaps.push('The Screenshots / clips section does not contain an actual attachment or link.');
+    }
   }
 
   // ── Comment ──────────────────────────────────────────────────────────────
@@ -62,22 +138,43 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => (c.body ?? '').includes(MARKER));
 
-  if (problems.length === 0) {
+  if (descriptionProblems.length === 0 && evidenceGaps.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });
     }
   } else {
-    const commentBody = [
-      MARKER,
-      '⚠️ **PR description — action needed**',
-      '',
-      'The following required sections are missing or incomplete. Please update the PR description to address them:',
-      '',
-      problems.map(p => `- ${p}`).join('\n'),
+    const commentLines = [MARKER];
+    if (descriptionProblems.length > 0) {
+      commentLines.push(
+        '⚠️ **PR description — action needed**',
+        '',
+        'The following required sections are missing or incomplete. Please update the PR description to address them:',
+        '',
+        descriptionProblems.map(problem => `- ${problem}`).join('\n'),
+      );
+    } else {
+      commentLines.push(
+        '⚠️ **PR description is complete; validation evidence is still outstanding**',
+        '',
+        `Changed-file classification: **${classification}**.`,
+      );
+    }
+    if (evidenceGaps.length > 0) {
+      commentLines.push(
+        '',
+        '**Author-reported runtime / visual state**',
+        '',
+        evidenceGaps.map(gap => `- ${gap}`).join('\n'),
+        '',
+        'Checkboxes are author attestations. GitHub Actions results remain the execution evidence for CI; this check does not prove that a local command ran.',
+      );
+    }
+    commentLines.push(
       '',
       '---',
-      '_This comment is deleted automatically once all sections are complete._',
-    ].join('\n');
+      '_This comment updates automatically when the description or changed files change._',
+    );
+    const commentBody = commentLines.join('\n');
 
     if (existing) {
       await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody });
@@ -97,34 +194,47 @@ module.exports = async ({ github, context, core }) => {
       return true;
     } catch (e) {
       if (e.status === 404) return false;
+      if (e.status === 403) {
+        core.warning(`Could not inspect label "${name}" — token lacks label read access; skipping.`);
+        return false;
+      }
       throw e;
     }
   }
 
-  async function swapLabel(num, add, remove) {
-    if (await labelExists(add)) {
+  async function setLabel(name, wanted) {
+    if (wanted && await labelExists(name)) {
       try {
-        await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] });
+        await github.rest.issues.addLabels({ owner, repo, issue_number: prNum, labels: [name] });
       } catch (e) {
         // Fail soft on a token that can't write labels so a label permission
         // problem never masks the actual description verdict.
-        if (e.status !== 403) throw e;
-        core.warning(`Could not add "${add}" — token lacks label write here; skipping.`);
+        if (e.status !== 403 && e.status !== 404) throw e;
+        core.warning(`Could not add "${name}" — label is unavailable or the token lacks label write access; skipping.`);
       }
+    } else if (wanted) {
+      core.warning(`Label "${name}" does not exist in the repo — skipping. Create it once to enable labelling.`);
     } else {
-      core.warning(`Label "${add}" does not exist in the repo — skipping. Create it once to enable labelling.`);
-    }
-    try {
-      await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name: remove });
-    } catch (e) {
-      if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
+      try {
+        await github.rest.issues.removeLabel({ owner, repo, issue_number: prNum, name });
+      } catch (e) {
+        if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
+      }
     }
   }
 
-  if (problems.length === 0) {
-    await swapLabel(prNum, 'ready for review', 'needs work');
-  } else {
-    await swapLabel(prNum, 'needs work', 'ready for review');
-    core.setFailed(`PR description has ${problems.length} issue(s) — see bot comment for details.`);
+  const descriptionComplete = descriptionProblems.length === 0;
+  const evidenceComplete = evidenceGaps.length === 0;
+  const isDraft = Boolean(context.payload.pull_request.draft);
+  await setLabel(
+    'ready for review',
+    descriptionComplete && evidenceComplete && !isDraft,
+  );
+  await setLabel('needs work', !descriptionComplete);
+  await setLabel('needs runtime validation', needsRuntimeValidation);
+  await setLabel('needs visual evidence', needsVisualEvidence);
+
+  if (!descriptionComplete) {
+    core.setFailed(`PR description has ${descriptionProblems.length} issue(s) — see bot comment for details.`);
   }
 };

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
   pull_request:
 
 # Least privilege: none of the jobs write to the repo.
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
@@ -73,10 +73,10 @@ jobs:
     name: Python syntax (compileall)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: "3.11"
       # Byte-compile sources — catches syntax errors without installing deps.
@@ -86,10 +86,10 @@ jobs:
     name: JS syntax (node --check)
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6.4.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
           node-version: "20"
       # Syntax-check our own JS (skip vendored libs in static/lib).
@@ -103,17 +103,14 @@ jobs:
   python-tests:
     name: Python tests (pytest)
     runs-on: ubuntu-latest
-    # Informational for now: the suite has known flaky / environment-dependent
-    # failures (test isolation + embedding-model assertions). Tracked under the
-    # ROADMAP "fresh install smoke tests" item; make this required once green.
-    continue-on-error: true
+    # Make Python test validation authoritative for the configured scope.
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
 
-      # Detect whether this PR only touches documentation files.
+      # Detect whether this PR only touches repository prose outside the Pages site.
       # If so, skip the expensive pytest run while still reporting a passing check.
       - name: Check for docs-only changes
         id: docs-check
@@ -125,9 +122,10 @@ jobs:
             BASE="${{ github.event.before }}"
             HEAD="${{ github.sha }}"
           fi
-          # List all changed files; if every file matches docs/markdown patterns, skip pytest.
+          # Keep website/ and assets/branding/ out of this bypass: pytest owns
+          # regression guards for their published-file and orphan-asset contracts.
           changed=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null || git diff --name-only HEAD~1 HEAD)
-          non_docs=$(echo "$changed" | grep -Ev '^(docs/|.*\.md$|\.github/[^/]+\.md$)' || true)
+          non_docs=$(echo "$changed" | grep -Ev '^(docs/|[^/]+\.md$|\.github/[^/]+\.md$)' || true)
           if [ -z "$non_docs" ]; then
             echo "docs_only=true" >> "$GITHUB_OUTPUT"
             echo "Docs-only change detected — skipping pytest."
@@ -135,7 +133,7 @@ jobs:
             echo "docs_only=false" >> "$GITHUB_OUTPUT"
           fi
 
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         if: steps.docs-check.outputs.docs_only != 'true'
         with:
           python-version: "3.11"

.github/workflows/codeql.yml

@@ -27,15 +27,15 @@ jobs:
         language: [actions, javascript-typescript, python]
     steps:
       - name: Checkout
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           languages: ${{ matrix.language }}
           build-mode: none
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/container-scan.yml

@@ -37,12 +37,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Lint Dockerfile
-        uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5  # v3.3.0
+        uses: hadolint/hadolint-action@2a66e89f53d0771bb131a7fa31f3136336094aa6  # v3.4.0
         with:
           dockerfile: Dockerfile
           # DL3008: pinning apt package versions is impractical on a -slim base

.github/workflows/container-trivy.yml

@@ -23,12 +23,16 @@ on:
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
   push:
     branches: [main]
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
   workflow_dispatch:
 
@@ -52,17 +56,17 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
 
       # Build without pushing so a broken Dockerfile is caught here, and the
       # exact image we ship is what gets scanned.
       - name: Build image
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           push: false
@@ -93,15 +97,15 @@ jobs:
       security-events: write  # upload SARIF to the Security tab
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
 
       - name: Build image
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           push: false
@@ -119,7 +123,7 @@ jobs:
           TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
 
       - name: Upload Trivy results
-        uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd  # v4.37.7
         with:
           sarif_file: trivy-results.sarif
           category: trivy-image

.github/workflows/dependency-review.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
@@ -55,12 +55,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: '3.12'
 

.github/workflows/deploy-pages.yml

@@ -0,0 +1,50 @@
+name: Deploy GitHub Pages
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'website/**'
+      - '.github/workflows/deploy-pages.yml'
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Package static site
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pages: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d  # v6.0.0
+      - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697  # v1.0.13
+        with:
+          source: website
+          destination: _site
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9  # v5.0.0
+        with:
+          path: _site
+
+  deploy:
+    name: Deploy static site
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128  # v5.0.0

.github/workflows/docker-publish.yml

@@ -14,6 +14,8 @@ on:
     paths-ignore:
       - '**.md'
       - 'docs/**'
+      - 'website/**'
+      - 'assets/branding/**'
       - '.github/ISSUE_TEMPLATE/**'
 
 concurrency:
@@ -45,20 +47,20 @@ jobs:
             arch: arm64
             runner: ubuntu-24.04-arm
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
       - name: Log in to GHCR
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee  # v4.2.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push by digest
         id: build
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf  # v7.2.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a  # v7.3.0
         with:
           context: .
           platforms: ${{ matrix.platform }}
@@ -86,7 +88,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       - name: Read APP_VERSION + short sha
@@ -103,16 +105,16 @@ jobs:
           pattern: digest-*
           merge-multiple: true
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5  # v4.1.0
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e  # v4.3.0
       - name: Log in to GHCR
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee  # v4.2.0
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Compute tags
         id: meta
-        uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9  # v6.1.0
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302  # v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |

.github/workflows/issue-description-check.yml

@@ -2,7 +2,7 @@ name: ci / issue description check
 
 on:
   issues:
-    types: [opened, edited, reopened]
+    types: [opened, edited, reopened, closed]
 
 permissions:
   issues: write
@@ -14,7 +14,7 @@ jobs:
     # Skip bots (Dependabot, release-drafter, etc.)
     if: ${{ github.event.issue.user.type != 'Bot' }}
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           sparse-checkout: .github/scripts
           persist-credentials: false

.github/workflows/pr-description-check.yml

@@ -5,7 +5,11 @@ on:
   # works on fork PRs. Safe here: the checkout pins to the base branch (no fork
   # code runs) and the scripts only read context.payload and call the GitHub API.
   pull_request_target:  # zizmor: ignore[dangerous-triggers]
-    types: [opened, edited, synchronize, reopened, ready_for_review]
+    types: [opened, edited, synchronize, reopened, ready_for_review, converted_to_draft]
+
+concurrency:
+  group: pr-description-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
 
 # Default-deny at the workflow level; each job opts into only the scopes it needs.
 # Note: modifying a PR's labels/comments needs pull-requests:write even though the
@@ -23,7 +27,7 @@ jobs:
     # Skip bots: they open PRs programmatically and have their own process.
     if: github.event.pull_request.user.type != 'Bot'
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           ref: ${{ github.base_ref }}
           sparse-checkout: .github/scripts
@@ -59,12 +63,14 @@ jobs:
 
   check-mergeable:
     name: Flag unmergeable PRs
+    needs: check-description
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       issues: write
-    # Skip bots: they open PRs programmatically and have their own process.
-    if: github.event.pull_request.user.type != 'Bot'
+    # Run after description validation failures, but never from an obsolete
+    # workflow run canceled by a newer PR event.
+    if: ${{ !cancelled() && github.event.pull_request.user.type != 'Bot' }}
     steps:
       - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
         with:

.github/workflows/secret-scan.yml

@@ -35,7 +35,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           # Full history so a secret committed in an earlier commit (and later
           # deleted) is still caught -- deletion does not remove it from Git.

.github/workflows/workflow-security.yml

@@ -36,7 +36,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
@@ -61,12 +61,12 @@ jobs:
       contents: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
           python-version: '3.12'
 

.gitignore

@@ -85,6 +85,24 @@ output.txt.txt
 !docs/**/*.gif
 !docs/**/*.webp
 
+# …except shipped website and branding media.
+!website/**/*.jpg
+!website/**/*.jpeg
+!website/**/*.png
+!website/**/*.gif
+!website/**/*.bmp
+!website/**/*.webp
+!website/**/*.tiff
+!website/**/*.pdf
+!assets/branding/**/*.jpg
+!assets/branding/**/*.jpeg
+!assets/branding/**/*.png
+!assets/branding/**/*.gif
+!assets/branding/**/*.bmp
+!assets/branding/**/*.webp
+!assets/branding/**/*.tiff
+!assets/branding/**/*.pdf
+
 # Reports and temp files
 reports/
 tasks/

ACKNOWLEDGMENTS.md

@@ -65,6 +65,16 @@ Vendored in `static/lib/` and served directly:
 | [jsPDF](https://github.com/parallax/jsPDF) (bundled in html2pdf) | PDF generation | MIT |
 | [html2canvas](https://github.com/niklasvh/html2canvas) (bundled in html2pdf) | DOM → canvas rasterization | MIT |
 | [node-qrcode](https://github.com/soldair/node-qrcode) (`qrcode.min.js`) | QR-code rendering (2FA setup) | MIT |
+| [KaTeX](https://github.com/KaTeX/KaTeX) v0.16.22 (`katex/katex.min.{js,css}` + `katex/fonts/*.woff2`) | Math typesetting | MIT ([`licenses/KaTeX-MIT-LICENSE.txt`](licenses/KaTeX-MIT-LICENSE.txt)) |
+| [Mermaid](https://github.com/mermaid-js/mermaid) v11.16.1 (`mermaid.min.js`) | Diagrams from text | MIT ([`licenses/Mermaid-MIT-LICENSE.txt`](licenses/Mermaid-MIT-LICENSE.txt)) |
+
+KaTeX and Mermaid are loaded on first use by `static/js/markdown.js` rather than
+from `index.html`, so a session that renders no math and no diagram never fetches
+either. Only the `.woff2` KaTeX fonts are shipped, matching `static/fonts/`; the
+`.woff` and `.ttf` variants its stylesheet also lists are never requested by a
+browser that supports `woff2`. The bundles are the published npm artifacts,
+unmodified — `.gitattributes` turns the whitespace check off for `static/lib/`
+so they can stay byte-identical to upstream.
 
 ## Front-end libraries loaded at runtime (CDN)
 
@@ -72,8 +82,6 @@ Referenced from `cdn.jsdelivr.net` / `cdnjs.cloudflare.com` at runtime — not v
 
 | Library | Purpose | License |
 |---|---|---|
-| [KaTeX](https://github.com/KaTeX/KaTeX) 0.16.22 | Math typesetting | MIT |
-| [Mermaid](https://github.com/mermaid-js/mermaid) 11 | Diagrams from text | MIT |
 | [Pyodide](https://github.com/pyodide/pyodide) 0.27.5 | In-browser Python runtime | MPL-2.0 |
 | [PDFObject](https://github.com/pipwerks/PDFObject) 2.1.1 | Inline PDF embedding | MIT |
 

Dockerfile

@@ -18,6 +18,10 @@ FROM python:3.14-slim
 # launch inside Docker.
 # nodejs/npm provide npx for the built-in Browser MCP server.
 # chromium provides the actual browser binary used by that MCP server.
+# fontconfig + Noto CJK provide real fallback glyphs for multilingual pages;
+# Chromium otherwise renders Chinese/Japanese/Korean labels as empty boxes.
+# iproute2/iputils-ping/net-tools/dnsutils/nmap give Docker-hosted agents the
+# basic network inspection toolkit expected by local LAN/debugging tasks.
 # gosu lets the entrypoint drop privileges cleanly so signals still reach
 # uvicorn directly (no extra shell layer like `su`/`sudo` would add).
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -28,8 +32,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     nodejs \
     npm \
     chromium \
+    fontconfig \
+    fonts-noto-cjk \
     tmux \
     openssh-client \
+    iproute2 \
+    iputils-ping \
+    net-tools \
+    dnsutils \
+    nmap \
     gosu \
     libgl1 \
     libglib2.0-0t64 \
@@ -37,6 +48,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libmagic1 \
     && rm -rf /var/lib/apt/lists/*
 
+# Private browser automation wrapper used by the native `private_browser` tool.
+# Chromium is installed above, so agent-browser can drive the existing browser
+# binary without paying `npx` startup/install overhead on each tool call.
+RUN npm install -g agent-browser@0.35.0 --omit=dev --loglevel=error
+
 # libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
 # libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
 # slim base omits them, so the Cookbook "install realesrgan" path imports cv2

HARNESS_VERSION

@@ -0,0 +1 @@
+0.20.5

README.md

@@ -1,7 +1,5 @@
-# Odysseus
-
 <p align="center">
-  <img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
+  <img src="assets/branding/odysseus-wordmark.png" alt="Odysseus" width="238">
 </p>
 
 <p align="center">
@@ -10,9 +8,7 @@
 
 <p align="center">
   <a href="#quick-start">Quick Start</a> ·
-  <a href="docs/setup.md">Setup Guide</a> ·
-  <a href="docs/ARCHITECTURE.md">Architecture</a> ·
-  <a href="SECURITY.md">Security</a> ·
+  <a href="website/setup.md">Setup Guide</a> ·
   <a href="CONTRIBUTING.md">Contributing</a> ·
   <a href="ROADMAP.md">Roadmap</a>
 </p>
@@ -22,7 +18,7 @@
 </p>
 
 <p align="center">
-  <img src="docs/odysseus-browser.jpg" alt="Odysseus interface">
+  <img src="assets/branding/odysseus-browser.jpg" alt="Odysseus interface">
 </p>
 
 ---
@@ -40,7 +36,7 @@ docker compose up -d --build
 
 Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`.
 
-Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md).
+Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](website/setup.md).
 
 ## Features
 
@@ -55,26 +51,31 @@ Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration
 
 ## Demo
 
-Explore the interface through the [interactive product tour](docs/index.html).
+A full hover-to-play tour lives on the [Odysseus landing page](https://odysseus-dev.github.io/odysseus/). Its source lives under [`website/`](website/).
 
 ## Contributing
 
-Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, documentation, and small focused refactors. Read the [contributing guide](CONTRIBUTING.md), review the [public roadmap](ROADMAP.md), and browse the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues).
+Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
 
 ## Security
 
-Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model or service ports publicly. Read the [security policy](SECURITY.md) and the [deployment security guidance](docs/setup.md#security-notes).
+Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly.
+
+- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
+- Keep `LOCALHOST_BYPASS=false` outside local development.
+
+Deployment details are in the [setup guide](website/setup.md#security-notes).
 
 ## Star History
 
-<a href="https://www.star-history.com/?repos=odysseus-dev%2Fodysseus&type=date&legend=top-left">
+<a href="https://star-history.dera.page/#odysseus-dev/odysseus&type=date&legend=top-left">
  <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&theme=dark&legend=top-left" />
-   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
-   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
+   <source media="(prefers-color-scheme: dark)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&theme=dark&legend=top-left" />
+   <source media="(prefers-color-scheme: light)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
+   <img alt="Star History Chart" src="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
  </picture>
 </a>
 
 ## License
 
-Licensed under AGPL-3.0-or-later. See the [license](LICENSE) and [acknowledgments](ACKNOWLEDGMENTS.md).
+AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).

ROADMAP.md

@@ -1,55 +1,87 @@
-# Roadmap
+# Roadmap / Help Wanted
 
-This document provides a high-level view of the areas Odysseus is currently improving.
+Odysseus is on a voyage, but not home yet. It works great for me (lol), but this ship is moving fast and feedback/help would be appreciated! (I don't know what I'm doing, help).
 
-It is directional rather than exhaustive. Priorities may change as the project evolves, defects are discovered, and maintainers learn more from implementation work and user feedback.
+If you see weird CSS, strange layout behavior, or a suspiciously murky corner of
+the codebase, you are probably right to stay away.
 
-For current implementation work, see the open [GitHub issues](https://github.com/odysseus-dev/odysseus/issues). Accepted behaviour should be documented in the repository alongside the code.
+## High Priority
 
-## Current priorities
+- SQUASH BUGS
+- Fresh install smoke tests on Linux, macOS, and Windows. Docker, native Python,
+  and WSL all need coverage.
 
-### Reliability and setup
+- Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. 
+- Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments.
+- Cookbook SGLang support across platforms. Make sure SGLang setup/serve works
+  predictably on Linux, Windows/WSL, macOS where possible, Docker, and common
+  NVIDIA/AMD hardware paths.
+- Deep Research model presets by hardware. Recommend approved model/parameter
+  profiles for small, medium, and large local setups so people with different
+  hardware can use Deep Research without guessing. Surface this either in Deep
+  Research settings or as a Cookbook scan/dropdown suggestion.
+- Cookbook model scan/download ranking. Prioritize newer architectures and
+  better hardware-fit models instead of scoring everything almost the same.
+  Ranking should account for architecture age, quant format, VRAM/RAM fit,
+  backend support, vision/mmproj requirements, and likely serve reliability.
+- Cookbook error feedback and logging. Failed downloads, dependency installs,
+  preflights, and serve jobs should show the actual command/output/error in the
+  UI, with copyable logs and clear next steps instead of just "crashed".
+- Agent prompt/context bloat. Agent mode is too heavy for smaller local models:
+  tool schemas, skills, memory, documents, and instructions can eat the context
+  before the user request really starts. We need slimmer prompts, better tool
+  selection, smaller default tool sets, and clearer guidance for models with
+  4k/8k/16k context windows.
+- Local model speculative decoding support. For Odysseus-tuned local models,
+  plan to ship or recommend a small same-tokenizer draft model when the serving
+  backend supports it. Early vLLM testing showed a generic `Qwen3-0.6B` draft
+  beside `Qwen3-8B` can materially reduce wall time, while an unsupported
+  DSpark conversion performed poorly. Treat this as a supported draft-model lane
+  first; keep MTP-specific packaging as future work only when the architecture
+  and runtime support are real. Judge this by time-to-success, tool correctness,
+  grammar, and unchanged target output, not tokens/sec alone.
+- Skill/tool prompt-injection audit. User-editable skills, notes, documents,
+  fetched pages, and memories should be treated as untrusted data. Keep testing
+  whether models follow malicious instructions from those surfaces.
+- Better degraded-state reporting for ChromaDB, SearXNG, email, ntfy, and provider probes.
+- Email performance audit. Fetching, searching, opening, deleting, and sending
+  email can feel slow, especially over IMAP/SMTP providers with high latency.
+  Need someone who knows mail performance to profile the current flow, identify
+  whether the bottleneck is IMAP folder select/fetch, cache invalidation,
+  attachment/body loading, SMTP handshakes, or frontend refresh behavior, then
+  propose safer caching/prefetch/batching without breaking multi-account state.
+- Provider setup/probing audit for Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, and DeepSeek.
 
-- Improve fresh-install and smoke-test coverage across supported environments.
-- Make provider setup, probing, and failure states more predictable.
-- Improve Cookbook reliability across hardware, operating systems, drivers, shells, and serving backends.
-- Improve degraded-state reporting and recovery guidance when optional services are unavailable.
+## Refactor Targets
+- CSS cleanup. `static/style.css` basically Calypso's island atm.
+- Tour core helper. The onboarding tours have too much copy-pasted scaffolding; promote a shared `tour-core.js` helper before adding more tours.
+- Modal/window positioning cleanup. Some window controls have improved, but the
+  underlying popup/dropdown/fixed-position behavior is still too fragile.
+- Mobile media override discoverability. A lot of "CSS did not move" bugs are mobile `@media` overrides of the same selector; comments or linting around desktop/mobile paired rules would help.
+- Dead code pass for old routes, stale feature flags, and unused UI states.
 
-### Local model workflows
+## Frontend
 
-- Improve hardware-aware model recommendations and compatibility guidance.
-- Evaluate serving optimizations, including speculative decoding, through reproducible benchmarks.
-- Improve installation, preflight checks, logging, and error reporting for local model serving.
-- Reduce prompt and context overhead for smaller local models.
+- Expand the Editor for quicker, more robust everyday use. Better file/document
+  handling, smoother window behavior, clearer save/export flows, stronger image
+  editing affordances, and fewer brittle edge cases.
+- Better AI integration for Notes and Todos. Notes should be easier for the
+  agent to read, update, summarize, and turn into actions. Todos should be
+  assignable to an agent from the UI, possibly through a button, task action,
+  or dedicated skill/tool flow.
+- Mobile gallery/editor polish. Easier to launch/download inpaint model or any missing pieces.
+- Accessibility pass: keyboard navigation, focus states, contrast, reduced motion.
+- Improve empty states and error messages on fresh installs.
+- Tighten first-run setup, hints, and tours so they do not repeat or fight each other.
+- Vendor CDN assets eventually for a more fully self-hosted/offline mode.
 
-### Safety and resilience
+## Backend
 
-- Continue hardening tool execution, filesystem access, credentials, networking, and destructive operations.
-- Treat content from documents, notes, memories, skills, and fetched pages as potentially untrusted.
-- Improve security-focused regression coverage and operational guidance.
-- Review integrations that expand access to sensitive data or privileged operations.
+- More tests around endpoint probing and provider setup.
+- Better task scheduler defaults and visibility.
+- Backup/restore guide and helper flow for `data/`.
+- Security hardening around admin-only tools and clear docs for their risk.
 
-### Product usability
+## Not The Focus Right Now
 
-- Improve first-run setup, onboarding, hints, and tours.
-- Improve accessibility, keyboard navigation, focus behaviour, contrast, and reduced-motion support.
-- Improve empty states, error messages, and recovery paths.
-- Strengthen Notes, Todos, Editor, mobile, and everyday workspace flows.
-
-### Architecture and maintainability
-
-- Reduce duplication and technical debt through focused, reviewable refactors.
-- Improve subsystem documentation as behaviour and architecture become stable.
-- Remove stale code, obsolete feature flags, and unsupported integrations.
-- Keep implementation decisions grounded in current code and verified behaviour.
-
-## Tracking work
-
-Concrete implementation tasks, defects, proposals, and technical investigations are tracked in:
-
-- [GitHub Issues](https://github.com/odysseus-dev/odysseus/issues)
-- [Contributing Guide](CONTRIBUTING.md)
-
-Maintainers may use additional private coordination tools for ownership, planning, and unresolved decisions.
-
-This roadmap is not a complete backlog or a guarantee that a particular item will be delivered.
+I prob shouldnt add more themes.

SECURITY.md

@@ -10,7 +10,7 @@ Security fixes are handled on the default branch until formal releases are cut.
 
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
-- Set `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
+- Leave `SECURE_COOKIES` unset unless you need to override it: session cookies are marked `Secure` whenever the request arrives over HTTPS. Set `SECURE_COOKIES=true` to force it on (for a proxy Odysseus cannot see the scheme of), or `SECURE_COOKIES=false` to force it off while you still serve plain HTTP alongside HTTPS.
 - Use HTTPS when exposing the app beyond localhost.
 - Put the authenticated Odysseus web/API entrypoint behind a trusted reverse proxy or private access layer such as Cloudflare Access, Tailscale, or a VPN.
 - Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only.
@@ -37,6 +37,4 @@ Only `.env.example`, docs, source, tests, and static assets should be committed.
 
 ## Reporting
 
-Report security vulnerabilities privately through [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new).
-
-Do not open a public issue or discussion, and do not disclose exploit details publicly.
+Please report vulnerabilities privately via GitHub security advisories if available, or by opening a minimal issue that does not disclose exploit details.

THREAT_MODEL.md

@@ -37,7 +37,7 @@ Non-admin defaults are in `core/auth.py:DEFAULT_PRIVILEGES`. Tool enforcement is
 
 - **Sessions:** bcrypt passwords, 7-day session tokens stored atomically in `data/sessions.json` via `core/atomic_io.py`.
 - **2FA:** TOTP with 8 single-use backup codes. Verified after password check, before session issuance.
-- **Reserved usernames:** `internal-tool`, `api`, `demo`, `system` cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
+- **Reserved usernames:** request sentinels and the Default/Local storage owner cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
   - `internal-tool` is security-critical: `core/middleware.py:require_admin` treats any request where `request.state.current_user == "internal-tool"` as the in-process tool loopback and grants admin unconditionally. A real account with that name would silently pass every `require_admin` check.
 - **Orphan sessions:** `validate_token` re-checks that the user record still exists on every call. A deleted user's cookie is dropped on next request rather than continuing to authenticate.
 
@@ -68,14 +68,14 @@ External content that reaches the LLM is treated as untrusted via `src/prompt_se
 - `X-Content-Type-Options: nosniff` and `Referrer-Policy: no-referrer` everywhere.
 - **CSP:** nonce-based `script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net`. `style-src 'unsafe-inline'` is intentionally kept — `static/index.html` ships inline `<style>` blocks and JS modules set `style=""` attributes at runtime. Inline styles do not execute script so the risk is visual-only. Removing this requires templating the HTML files and auditing all JS-set style attributes.
 
-## Token-Supplied Model Endpoints
-
-Direct `/api/v1/chat` requests with a token-supplied `base_url` must use a public HTTP(S) endpoint. This restriction applies only to untrusted direct values; administrator-configured endpoints may intentionally use local or LAN URLs for private model providers.
-
 ## Known Gaps
 
 These are open, acknowledged, and contributor help is welcome:
 
 1. **No shell/filesystem sandbox.** The agent `bash` and `read_file`/`write_file` tools run as the app process user with no network egress filtering or filesystem confinement. A successful prompt-injection reaching a shell-enabled admin session can make outbound requests to internal services. See #1058 for the sandbox proposal.
 
-2. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity.
+2. **SSRF via `/api/v1/chat` `base_url` parameter.** A chat-scoped API token can supply an arbitrary `base_url`; the server forwards the LLM request to that host without validating the scheme or address. PR #1039 fixes this.
+
+3. **`src/search/` partial consolidation.** `src.search.core` and `src.search.providers` correctly alias `services.search` via `sys.modules` replacement. `analytics`, `cache`, `content`, `query`, and `ranking` are still independent copies that can drift. The SSRF regression tests in `tests/test_webhook_ssrf_resilience.py` test `src.webhook_manager` directly (separate from search), so the safety net there is intact. See #1058.
+
+4. **Token scopes are coarse.** There is no way to grant a session a subset of the owning user's privileges. Companion/mobile tokens carry either `chat` or `admin` scope with no per-capability granularity.

app.py

@@ -4,6 +4,8 @@ import os
 import sys
 import asyncio
 import time
+import shutil
+import socket
 
 # On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
 # When started via `python -m uvicorn` from a terminal, uvicorn sets this
@@ -67,7 +69,13 @@ from core.constants import (
     REQUEST_TIMEOUT, OPENAI_API_KEY, AUTH_FILE,
 )
 from core.database import SessionLocal, ApiToken
-from core.middleware import SecurityHeadersMiddleware, is_cors_preflight
+from core.middleware import (
+    SecurityHeadersMiddleware,
+    get_application_route_path,
+    is_cors_preflight,
+    path_is_route_or_child,
+    with_asgi_root_path,
+)
 from core.auth import AuthManager, normalize_known_username
 from core.exceptions import (
     SessionNotFoundError, InvalidFileUploadError,
@@ -78,6 +86,7 @@ import bcrypt as _bcrypt
 
 from src.app_helpers import abs_join, serve_html_with_nonce
 from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
+from src.owner_identity import auth_disabled
 from starlette.responses import RedirectResponse
 
 # ========= LOGGING =========
@@ -153,7 +162,8 @@ app.add_middleware(
 # model-probe — all served with media_type="text/event-stream") are never
 # compressed or buffered; only complete bodies over minimum_size are. The
 # security-header middleware composes cleanly on top.
-app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
+if os.getenv("RESPONSE_COMPRESSION_ENABLED", "true").strip().lower() not in {"0", "false", "no", "off"}:
+    app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
 
 # ========= SECURITY HEADERS MIDDLEWARE =========
 app.add_middleware(SecurityHeadersMiddleware)
@@ -248,7 +258,7 @@ from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
 
 auth_manager = AuthManager()
 app.state.auth_manager = auth_manager
-AUTH_ENABLED = os.getenv("AUTH_ENABLED", "true").lower() != "false"
+AUTH_ENABLED = not auth_disabled()
 LOCALHOST_BYPASS = os.getenv("LOCALHOST_BYPASS", "false").lower() == "true"
 if LOCALHOST_BYPASS:
     logger.warning("LOCALHOST_BYPASS is enabled, loopback requests bypass authentication. Do not expose this instance to a network.")
@@ -284,7 +294,7 @@ if AUTH_ENABLED:
     def _is_auth_exempt(path: str) -> bool:
         if path in AUTH_EXEMPT_EXACT:
             return True
-        if any(path.startswith(p) for p in AUTH_EXEMPT_PREFIXES):
+        if any(path_is_route_or_child(path, p) for p in AUTH_EXEMPT_PREFIXES):
             return True
         return any(p.match(path) for p in AUTH_EXEMPT_PATTERNS)
 
@@ -355,7 +365,7 @@ if AUTH_ENABLED:
 
     class AuthMiddleware(BaseHTTPMiddleware):
         async def dispatch(self, request: Request, call_next):
-            path = request.url.path
+            path = get_application_route_path(request.scope)
             # A genuine CORS preflight (OPTIONS + Access-Control-Request-Method)
             # carries no credentials by design and must reach CORSMiddleware to be
             # answered. AuthMiddleware is the outermost middleware, so gating the
@@ -399,7 +409,10 @@ if AUTH_ENABLED:
             if not auth_manager.is_configured:
                 # No users yet — redirect to login for first-time setup
                 if not path.startswith("/api/"):
-                    return RedirectResponse(url="/login", status_code=302)
+                    return RedirectResponse(
+                        url=with_asgi_root_path(request.scope, "/login"),
+                        status_code=302,
+                    )
                 return JSONResponse(status_code=401, content={"error": "Setup required"})
 
             # --- Bearer token auth (API tokens for external integrations) ---
@@ -461,7 +474,10 @@ if AUTH_ENABLED:
             if not auth_manager.validate_token(token):
                 if path.startswith("/api/"):
                     return JSONResponse(status_code=401, content={"error": "Not authenticated"})
-                return RedirectResponse(url="/login", status_code=302)
+                return RedirectResponse(
+                    url=with_asgi_root_path(request.scope, "/login"),
+                    status_code=302,
+                )
 
             # Attach current username to request state for downstream routes
             request.state.current_user = auth_manager.get_username_for_token(token)
@@ -630,13 +646,24 @@ app.include_router(auth_router)
 
 @app.post("/api/activity/heartbeat")
 async def activity_heartbeat():
-    from src.interactive_gate import mark_browser_activity
+    from src.interactive_gate import (
+        mark_browser_activity,
+        maybe_stop_background_tasks_for_heartbeat,
+    )
+
     await mark_browser_activity()
+
     async def _stop_background():
         try:
-            await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
+            await maybe_stop_background_tasks_for_heartbeat(
+                task_scheduler.stop_background_tasks_for_foreground
+            )
         except Exception:
-            logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
+            logging.getLogger("app.foreground_gate").debug(
+                "heartbeat task stop failed",
+                exc_info=True,
+            )
+
     asyncio.create_task(_stop_background())
     return {"ok": True}
 
@@ -660,6 +687,7 @@ app.include_router(setup_session_routes(
     session_config,
     webhook_manager=webhook_manager,
     upload_handler=upload_handler,
+    skills_manager=skills_manager,
 ))
 
 # Admin Danger Zone wipes (Settings → System → Danger Zone)
@@ -692,7 +720,7 @@ from routes.history.history_routes import setup_history_routes
 app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler))
 
 # Search
-from routes.search_routes import setup_search_routes
+from routes.search.search_routes import setup_search_routes
 app.include_router(setup_search_routes(config))
 
 # Presets
@@ -739,7 +767,7 @@ app.include_router(setup_stt_routes(stt_service))
 logger.info("STT service initialized (provider managed via settings)")
 
 # Documents (artifacts/canvas)
-from routes.document_routes import setup_document_routes
+from routes.document.document_routes import setup_document_routes
 document_router = setup_document_routes(session_manager, upload_handler)
 app.include_router(document_router)
 
@@ -760,7 +788,7 @@ from src.task_scheduler import TaskScheduler
 task_scheduler = TaskScheduler(session_manager)
 from src.event_bus import set_task_scheduler
 set_task_scheduler(task_scheduler)
-from routes.task_routes import setup_task_routes
+from routes.task.task_routes import setup_task_routes
 app.include_router(setup_task_routes(task_scheduler))
 
 from routes.assistant_routes import setup_assistant_routes
@@ -805,7 +833,7 @@ app.include_router(setup_font_routes())
 # MCP (Model Context Protocol)
 from src.mcp_manager import McpManager
 from src.agent_tools import set_mcp_manager
-from routes.mcp_routes import setup_mcp_routes
+from routes.mcp.mcp_routes import setup_mcp_routes
 
 mcp_manager = McpManager()
 set_mcp_manager(mcp_manager)
@@ -820,7 +848,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
 logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
 
 # Webhooks
-from routes.webhook_routes import setup_webhook_routes
+from routes.webhook.webhook_routes import setup_webhook_routes
 app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
 
 # API Tokens
@@ -852,7 +880,7 @@ app.include_router(setup_codex_routes(
 ))
 app.include_router(setup_claude_routes())
 
-from routes.vault_routes import setup_vault_routes
+from routes.vault.vault_routes import setup_vault_routes
 app.include_router(setup_vault_routes())
 
 # Contacts (CardDAV)
@@ -925,8 +953,12 @@ async def serve_login(request: Request):
 
 @app.get("/api/version")
 async def get_version():
-    from core.constants import APP_VERSION
-    return {"version": APP_VERSION}
+    from core.constants import APP_BUILD_VERSION, APP_SOURCE_COMMIT, APP_VERSION
+    return {
+        "version": APP_VERSION,
+        "build": APP_BUILD_VERSION,
+        "source_commit": APP_SOURCE_COMMIT,
+    }
 
 @app.get("/api/health")
 async def health_check() -> Dict[str, str]:
@@ -986,11 +1018,76 @@ async def runtime_info() -> Dict[str, object]:
         or os.getenv("OLLAMA_URL")
         or ("http://host.docker.internal:11434/v1" if in_docker else "http://127.0.0.1:11434/v1")
     )
+    network_mode = os.getenv("ODYSSEUS_CONTAINER_NETWORK_MODE", "").strip()
+    host_gateway_reachable = False
+    host_gateway_address = ""
+    if in_docker and network_mode != "host":
+        try:
+            resolved = socket.getaddrinfo("host.docker.internal", None)
+            for item in resolved:
+                sockaddr = item[4] if len(item) >= 5 else ()
+                candidate = sockaddr[0] if sockaddr else ""
+                if candidate:
+                    host_gateway_address = str(candidate)
+                    break
+            host_gateway_reachable = True
+        except OSError:
+            host_gateway_reachable = False
+        if not host_gateway_address:
+            host_gateway_address = _docker_default_gateway_ip()
+    container: Dict[str, object] = {
+        "engine": "docker" if in_docker else "",
+        "networkMode": network_mode,
+        "hostAccess": bool(in_docker and network_mode == "host"),
+        "hostGatewayReachable": host_gateway_reachable,
+    }
+    if host_gateway_address:
+        container["hostGatewayAddress"] = host_gateway_address
+    command_names = (
+        "ip",
+        "ss",
+        "arp",
+        "nmap",
+        "ping",
+        "dig",
+        "ssh",
+        "git",
+        "docker",
+    )
+    commands = {name: bool(shutil.which(name)) for name in command_names}
+    capabilities = {
+        "networkInspection": bool(commands["ip"] and (commands["ss"] or commands["arp"])),
+        "lanScan": bool(commands["nmap"]),
+        "dnsLookup": bool(commands["dig"]),
+        "sshClient": bool(commands["ssh"]),
+        "git": bool(commands["git"]),
+        "dockerClient": bool(commands["docker"]),
+    }
     return {
         "in_docker": in_docker,
         "ollama_base_url": ollama_url,
+        "container": container,
+        "commands": commands,
+        "capabilities": capabilities,
     }
 
+
+def _docker_default_gateway_ip() -> str:
+    try:
+        with open("/proc/net/route", "r", encoding="utf-8", errors="ignore") as fh:
+            for line in fh.readlines()[1:]:
+                parts = line.split()
+                if len(parts) < 3 or parts[1] != "00000000":
+                    continue
+                raw = parts[2]
+                if len(raw) != 8:
+                    continue
+                octets = [str(int(raw[i:i + 2], 16)) for i in range(6, -1, -2)]
+                return ".".join(octets)
+    except Exception:
+        return ""
+    return ""
+
 # ========= LIFECYCLE =========
 
 @asynccontextmanager
@@ -1030,6 +1127,15 @@ async def _startup_event():
     # GC tasks created with `asyncio.create_task(...)` before they finish.
     _startup_tasks: list[asyncio.Task] = getattr(app.state, "_startup_tasks", [])
     app.state._startup_tasks = _startup_tasks
+    from src.background_tool_jobs import BackgroundToolJobs
+    from routes.chat_routes import _active_streams
+    from src import agent_runs
+    app.state.background_tool_jobs = BackgroundToolJobs(
+        is_busy=lambda sid: sid in _active_streams or agent_runs.is_active(sid),
+        session_manager=session_manager, research_handler=research_handler,
+    )
+    app.state.background_tool_delivery_task = asyncio.create_task(app.state.background_tool_jobs.run())
+    _startup_tasks.append(app.state.background_tool_delivery_task)
     if upload_cleanup_func:
         upload_cleanup_task = asyncio.create_task(upload_cleanup_func())
     # Always-on monitor that auto-continues the agent when a background bash
@@ -1056,23 +1162,34 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
 
-    # Startup warmups are opt-in. They make later requests a little warmer, but
-    # they also compete with the first seconds of real UI use on slow or busy
-    # machines. Default to clear/idle startup and let requests warm what they use.
-    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
-    if _startup_warmups_enabled:
+    # Semantic tool selection is part of the agent serving contract. Initialize
+    # it in a background thread by default so startup remains nonblocking while
+    # harness deployments can wait for the explicit readiness state.
+    from src.tool_index import prewarm_tool_index, tool_index_prewarm_enabled
+    if tool_index_prewarm_enabled():
         async def _warmup_tool_index():
-            try:
-                from src.tool_index import get_tool_index
-                idx = await asyncio.to_thread(get_tool_index)
-                if idx:
-                    await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
-                    logger.info("[startup] Tool index pre-warmed")
-            except Exception as e:
-                logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
+            status = await asyncio.to_thread(prewarm_tool_index)
+            if status.get("ready"):
+                logger.info(
+                    "[startup] Tool index pre-warmed lanes=%s tools=%s duration_ms=%s",
+                    [lane.get("name") for lane in status.get("lanes", [])],
+                    status.get("builtin_tools"),
+                    status.get("duration_ms"),
+                )
+            else:
+                logger.warning(
+                    "Tool index warmup degraded (non-critical): %s",
+                    status.get("error_type") or status.get("state"),
+                )
 
         _startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
+    else:
+        logger.info("Tool index prewarm disabled (ODYSSEUS_TOOL_INDEX_PREWARM=0)")
 
+    # Model endpoint pings remain opt-in. They can compete with the first seconds
+    # of UI use on slow or busy machines and are not required for local startup.
+    _startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
+    if _startup_warmups_enabled:
         async def _warmup_endpoints():
             try:
                 import httpx
@@ -1092,7 +1209,7 @@ async def _startup_event():
 
         _startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
     else:
-        logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
+        logger.info("Model endpoint warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
 
     # Keep-alive is opt-in. The ping path performs model discovery, and when
     # stale LAN endpoints are configured it can add periodic backend pressure
@@ -1160,6 +1277,14 @@ async def _startup_event():
     # Disk-backed skills are not covered by the DB legacy-owner sweep. Repair
     # ownerless or deleted/test-owner SKILL.md files so strict owner filtering
     # does not make an existing library look empty after auth/account changes.
+    try:
+        from services.memory.builtin_skills import install_builtin_skills
+        installed = install_builtin_skills(skills_manager, ())
+        if installed:
+            logger.info("Installed %s built-in skill file(s)", installed)
+    except Exception as e:
+        logger.debug(f"Built-in skill installation skipped: {e}")
+
     try:
         import json as _json
         auth_path = AUTH_FILE
@@ -1205,35 +1330,10 @@ async def _startup_event():
 
     _startup_tasks.append(asyncio.create_task(_null_owner_sweep_loop()))
 
-    # Nightly skill audit — at ~02:00 local, test + judge a batch of the
-    # least-recently-checked skills, auto-fixing/escalating weak ones (never
-    # deletes). Rotates through the library so each night covers different
-    # skills. Gated by the `skill_audit_nightly` setting (default on); hour via
-    # `skill_audit_hour` (default 2), batch size via `skill_audit_batch` (8).
-    async def _skill_audit_nightly_loop():
-        from datetime import timedelta
-        while True:
-            try:
-                from src.settings import get_setting
-                hour = int(get_setting("skill_audit_hour", 2) or 2)
-            except Exception:
-                hour = 2
-            now = datetime.now()
-            nxt = now.replace(hour=hour % 24, minute=0, second=0, microsecond=0)
-            if nxt <= now:
-                nxt += timedelta(days=1)
-            await asyncio.sleep(max(60, (nxt - now).total_seconds()))
-            try:
-                from src.settings import get_setting
-                if not get_setting("skill_audit_nightly", True):
-                    continue
-                batch = int(get_setting("skill_audit_batch", 8) or 8)
-                from routes.skills_routes import run_scheduled_skill_audit
-                await run_scheduled_skill_audit(skills_manager, owner=None, max_skills=batch)
-            except Exception as e:
-                logger.warning(f"Nightly skill audit failed: {e}")
-
-    _startup_tasks.append(asyncio.create_task(_skill_audit_nightly_loop()))
+    # Skills Audit is scheduled per owner by TaskScheduler. Do not also start
+    # an ownerless audit here: its sidecar results cannot be read back through
+    # an authenticated owner's skill namespace, and its model activity can
+    # defer the real per-owner task at the same time of night.
 
     # Cookbook serve lifecycle — kills scheduler-launched serves whose
     # window-end has passed. Paired with the cookbook_serve builtin
@@ -1248,6 +1348,18 @@ async def _startup_event():
 
 async def _shutdown_event():
     logger.info("Application shutting down...")
+    background_delivery = getattr(app.state, 'background_tool_delivery_task', None)
+    if background_delivery:
+        background_delivery.cancel()
+        try:
+            await background_delivery
+        except asyncio.CancelledError:
+            pass
+    try:
+        from src.agent_tools.web_tools import shutdown_private_browser_sessions
+        await shutdown_private_browser_sessions()
+    except Exception as e:
+        logger.warning(f"Private browser shutdown error: {e}")
     if upload_cleanup_task:
         upload_cleanup_task.cancel()
         try:
@@ -1276,6 +1388,6 @@ if __name__ == "__main__":
     import uvicorn
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
 
     uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

build-macos-app.sh

@@ -27,13 +27,13 @@ echo "  port:        $PORT"
 rm -rf "$APP"
 mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources"
 
-# ── Icon (best effort) — center-crop docs/odysseus.jpg to a square .icns ──
-if [ -f "$REPO_DIR/docs/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
+# ── Icon (best effort) — center-crop the branding image to a square .icns ──
+if [ -f "$REPO_DIR/assets/branding/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
   TMPIMG="$(mktemp -d)"
   # Center-crop to a square, scale to 512 (sips' icns encoder caps at 512), and
   # let sips emit the .icns directly — more robust across macOS versions than
   # building an .iconset by hand.
-  sips -c 720 720 "$REPO_DIR/docs/odysseus.jpg" --out "$TMPIMG/sq.png" >/dev/null 2>&1 || cp "$REPO_DIR/docs/odysseus.jpg" "$TMPIMG/sq.png"
+  sips -c 720 720 "$REPO_DIR/assets/branding/odysseus.jpg" --out "$TMPIMG/sq.png" >/dev/null 2>&1 || cp "$REPO_DIR/assets/branding/odysseus.jpg" "$TMPIMG/sq.png"
   sips -z 512 512 "$TMPIMG/sq.png" --out "$TMPIMG/icon.png" >/dev/null 2>&1
   if sips -s format icns "$TMPIMG/icon.png" --out "$APP/Contents/Resources/odysseus.icns" >/dev/null 2>&1; then
     echo "  icon:        odysseus.icns"
@@ -42,7 +42,7 @@ if [ -f "$REPO_DIR/docs/odysseus.jpg" ] && command -v sips >/dev/null 2>&1; then
   fi
   rm -rf "$TMPIMG"
 else
-  echo "  icon:        (skipped — no docs/odysseus.jpg)"
+  echo "  icon:        (skipped — no assets/branding/odysseus.jpg)"
 fi
 
 # ── Info.plist ──
@@ -73,6 +73,10 @@ cat > "$APP/Contents/MacOS/$APP_NAME.tmpl" <<'LAUNCHER'
 INSTALL_DIR="__INSTALL_DIR__"
 PORT="__PORT__"
 URL="http://127.0.0.1:${PORT}"
+# uvicorn is started with --port below, but APP_PORT is what the app itself
+# reads when it needs to build a URL for this instance (internal_api_base(),
+# companion pairing, the MCP OAuth callback), so export it as well.
+export APP_PORT="$PORT"
 export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
 
 UVICORN="$INSTALL_DIR/venv/bin/uvicorn"

companion/pairing.py

@@ -6,11 +6,14 @@ units so the route layer stays thin and the logic is directly testable.
 
 from __future__ import annotations
 
+import ipaddress
 import json
 import os
+import re
 import secrets
 import socket
 import uuid
+from urllib.parse import urlsplit
 
 import bcrypt
 
@@ -20,6 +23,102 @@ PAIRING_VERSION = 1
 COMPANION_SCOPE = "chat"
 
 
+_COMPANION_IPV4_NETWORKS = tuple(
+    ipaddress.ip_network(cidr)
+    for cidr in (
+        "10.0.0.0/8",
+        "100.64.0.0/10",
+        "127.0.0.0/8",
+        "169.254.0.0/16",
+        "172.16.0.0/12",
+        "192.168.0.0/16",
+    )
+)
+_DNS_LABEL_RE = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
+
+
+def _valid_companion_client_host(host: str) -> bool:
+    """Match the host forms supported by the current v1 Expo client."""
+    if not host or len(host) > 253 or not host.isascii() or "%" in host:
+        return False
+
+    try:
+        address = ipaddress.ip_address(host)
+    except ValueError:
+        labels = host.split(".")
+        if any(not _DNS_LABEL_RE.fullmatch(label) for label in labels):
+            return False
+        if any(label.startswith("xn--") for label in labels):
+            return False
+        # WHATWG URL parsers treat a decimal or ``0x`` single-label hostname
+        # as an IPv4 number even though Python's strict ``ipaddress`` parser
+        # rejects that spelling.  The v1 client interpolates this host back
+        # into a URL, so accepting e.g. ``134744072`` would make the phone send
+        # its bearer token to public 8.8.8.8.  Keep DNS labels unambiguous.
+        if len(labels) == 1 and (
+            labels[0].isdigit()
+            or re.fullmatch(r"0x[0-9a-f]*", labels[0]) is not None
+        ):
+            return False
+        return len(labels) == 1 or (len(labels) >= 2 and labels[-1] == "local")
+
+    return isinstance(address, ipaddress.IPv4Address) and any(
+        address in network for network in _COMPANION_IPV4_NETWORKS
+    )
+
+
+def parse_companion_base_url(value: str) -> tuple[str, int]:
+    """Validate a v1 companion address and return its legacy (host, port).
+
+    The deployed client understands only HTTP plus a LAN-style host and port.
+    Reject anything outside that exact contract instead of advertising a URL
+    the client would reject, downgrade, or interpret differently.
+    """
+    if not isinstance(value, str) or not value:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    if not value.isascii():
+        raise ValueError("COMPANION_BASE_URL must contain only ASCII characters")
+    if any(
+        ord(char) <= 32 or ord(char) == 127 or char in {"\\", "%"}
+        for char in value
+    ):
+        raise ValueError(
+            "COMPANION_BASE_URL contains a forbidden character"
+        )
+
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except ValueError as exc:
+        raise ValueError("COMPANION_BASE_URL must be a valid HTTP LAN origin") from exc
+
+    host = parsed.hostname
+    if parsed.scheme.lower() != "http" or not parsed.netloc or not host:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    if parsed.username is not None or parsed.password is not None:
+        raise ValueError("COMPANION_BASE_URL must not contain credentials")
+    if parsed.path or parsed.query or parsed.fragment:
+        raise ValueError("COMPANION_BASE_URL must not contain a path, query, or fragment")
+    if port is not None and not 1 <= port <= 65535:
+        raise ValueError("COMPANION_BASE_URL port must be between 1 and 65535")
+    if not _valid_companion_client_host(host):
+        raise ValueError("COMPANION_BASE_URL host is not supported by companion v1")
+
+    netloc = f"{host}:{port}" if port is not None else host
+    origin = f"http://{netloc}"
+    if value != origin:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    return host, port or 80
+
+
+def configured_companion_origin() -> tuple[str, int] | None:
+    """Return the validated operator-configured v1 address, if any."""
+    value = os.environ.get("COMPANION_BASE_URL")
+    if value is None or value == "":
+        return None
+    return parse_companion_base_url(value)
+
+
 def default_port() -> int:
     """Best guess at the port the server is reachable on. Callers that know the
     real request port should pass it explicitly."""

companion/routes.py

@@ -23,7 +23,7 @@ from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import HTMLResponse
 
 from core.middleware import require_admin
-from src.auth_helpers import get_current_user
+from src.auth_helpers import _auth_disabled, get_current_user
 
 from companion import pairing as _pairing
 
@@ -113,8 +113,9 @@ def setup_companion_routes() -> APIRouter:
         The stock /api/models route scopes to get_current_user, which for a
         bearer token is the sandboxed pseudo-user "api" (owns nothing). Here we
         scope to the token's real owner instead, plus legacy null-owner shared
-        rows -- the same rule as owner_filter. Read-only; never returns api_key
-        material.
+        rows -- the same rule as owner_filter. Explicit auth-disabled mode keeps
+        the stock route's single-user all-endpoints view. Read-only; never
+        returns api_key material.
         """
         require_models_scope(request)
         import json as _json
@@ -123,6 +124,11 @@ def setup_companion_routes() -> APIRouter:
         from src.endpoint_resolver import build_chat_url
 
         owner = token_owner(request)
+        single_user_mode = (
+            owner is None
+            and not getattr(request.state, "api_token", False)
+            and _auth_disabled()
+        )
         out = []
         db = SessionLocal()
         try:
@@ -133,7 +139,7 @@ def setup_companion_routes() -> APIRouter:
             if owner:
                 q = q.filter((ModelEndpoint.owner == owner) | (ModelEndpoint.owner == None))  # noqa: E711
             for ep in q.all():
-                if not owner_can_see(ep.owner, owner):
+                if not single_user_mode and not owner_can_see(ep.owner, owner):
                     continue
                 try:
                     model_ids = _json.loads(ep.cached_models) if ep.cached_models else []
@@ -194,19 +200,27 @@ def setup_companion_routes() -> APIRouter:
         the code works immediately, no restart. `?format=json` returns the
         payload for an in-app pairing screen."""
         require_admin(request)
+        try:
+            configured_origin = _pairing.configured_companion_origin()
+        except ValueError as exc:
+            raise HTTPException(500, str(exc)) from None
         owner = get_current_user(request)
         invalidate = getattr(request.app.state, "invalidate_token_cache", None)
         token_id, raw_token = mint_pairing_token(owner, invalidate)
 
-        hosts = _pairing.lan_ip_candidates()
-        host = hosts[0] if hosts else "127.0.0.1"
-        port = request.url.port or _pairing.default_port()
+        if configured_origin:
+            host, port = configured_origin
+            hosts = [host]
+        else:
+            hosts = _pairing.lan_ip_candidates()
+            host = hosts[0] if hosts else "127.0.0.1"
+            port = request.url.port or _pairing.default_port()
         payload = _pairing.pairing_payload(host, port, raw_token)
         qr = _pairing.pairing_qr_png_data_uri(payload)
         qr_ok = bool(qr and qr.startswith("data:image/png;base64,"))
 
         if (request.query_params.get("format") or "").lower() == "json":
-            return {
+            response = {
                 "host": host,
                 "port": port,
                 "token": raw_token,
@@ -215,6 +229,7 @@ def setup_companion_routes() -> APIRouter:
                 "payload": payload,
                 "qr": qr if qr_ok else None,
             }
+            return response
 
         import json as _json
         payload_json = _json.dumps(payload, separators=(",", ":"))

core/atomic_io.py

@@ -15,31 +15,53 @@ from __future__ import annotations
 
 import json
 import os
+import uuid
 from typing import Any, Optional
 
 
 def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> None:
     """Atomically persist `data` as JSON at `path`.
 
-    The temp file uses the live PID as a suffix so two processes saving the
-    same file (e.g. unit tests) don't collide on the rename target.
+    The temp file uses a random suffix so two concurrent writers saving the
+    same file don't collide on the rename target. A PID suffix does not do
+    this: the PID is constant for the life of a process, so two writers on
+    the same path within one process (or one single-process container, where
+    the PID never changes at all) still race for the same temp file.
     """
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{os.getpid()}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=indent)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=indent)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
 
 
 def atomic_write_text(path: str, text: str) -> None:
     if not isinstance(text, str):
         raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{os.getpid()}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        f.write(text)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            f.write(text)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass

core/auth.py

@@ -20,7 +20,6 @@ logger = logging.getLogger(__name__)
 
 
 from core.atomic_io import atomic_write_json as _atomic_write_json  # noqa: E402
-from core.middleware import INTERNAL_TOOL_USER  # noqa: E402
 
 DEFAULT_PRIVILEGES = {
     "can_use_agent": True,
@@ -49,24 +48,18 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False
 ADMIN_PRIVILEGES["block_all_models"] = False
 
 from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH
+from src.owner_identity import RESERVED_AUTH_USERNAMES
 DEFAULT_AUTH_PATH = AUTH_FILE
 TOKEN_TTL = 60 * 60 * 24 * 7  # 7 days
 
-# Usernames the auth + middleware layer reserve as internal "synthetic owner"
-# sentinels; they must never belong to a real account. The most dangerous is
-# "internal-tool": `core.middleware.require_admin` treats any request whose
-# `current_user == "internal-tool"` as the in-process tool loopback and grants
-# admin, and because the cookie auth path sets `current_user` to the raw
-# username, an account literally named "internal-tool" would be silently
-# treated as an admin by every `require_admin`-gated route. "api" collides with
-# the bearer-token owner-attribution sentinel. "demo"/"system" round out the
-# synthetic-owner set the rest of the codebase already special-cases (see
-# `_SYNTHETIC_OWNERS` in routes/assistant_routes.py and the matching guards in
-# src/task_scheduler.py / routes/research_routes.py) — a real account with one
-# of those names would be denied an assistant and inconsistently owner-scoped.
-# Refuse to create or rename into any of them so the sentinels can't be
-# impersonated. (Keep this in sync with that synthetic-owner set.)
-RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"})
+# Usernames the auth + middleware layer reserves for request sentinels and
+# internal storage owners; they must never belong to a real login account.
+# "internal-tool" is the most dangerous because `core.middleware.require_admin`
+# treats it as the in-process tool loopback. "api" collides with bearer-token
+# attribution. "demo"/"system" are synthetic owners already special-cased by
+# scheduler/assistant/research paths. The Default/Local owner is a storage
+# bucket for explicit auth-disabled no-login mode, not a login username.
+RESERVED_USERNAMES = frozenset(RESERVED_AUTH_USERNAMES)
 
 
 def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]:

core/database.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.parse import unquote, urlparse
-from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
+from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, Float, ForeignKey, JSON, Index, func, inspect, text
 from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
@@ -75,7 +75,7 @@ DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database
 # Create engine
 engine = create_engine(
     DATABASE_URL,
-    connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
+    connect_args={"check_same_thread": False, "timeout": 30} if "sqlite" in DATABASE_URL else {}
 )
 
 
@@ -144,6 +144,8 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     if isinstance(dbapi_connection, sqlite3.Connection):
         cursor = dbapi_connection.cursor()
         cursor.execute("PRAGMA foreign_keys=ON")
+        cursor.execute("PRAGMA busy_timeout=30000")
+        cursor.execute("PRAGMA journal_mode=WAL")
         cursor.close()
 
 
@@ -191,9 +193,15 @@ class Session(TimestampMixin, Base):
     # Configuration flags
     rag = Column(Boolean, default=False)
     archived = Column(Boolean, default=False)
+    memory_extraction_enabled = Column(Boolean, default=True)
+    skill_injection_enabled = Column(Boolean, default=True)
+    thinking_mode = Column(String, nullable=True, default="off")
+    temperature_override = Column(Float, nullable=True, default=None)
+    max_tokens_override = Column(Integer, nullable=True, default=None)
 
     # Organization
     folder = Column(String, nullable=True, default=None)
+    cwd = Column(String, nullable=True, default=None)
     
     # Headers stored as JSON
     headers = Column(JSON, default=dict)
@@ -219,6 +227,7 @@ class Session(TimestampMixin, Base):
     message_count = Column(Integer, default=0)
     total_input_tokens = Column(Integer, default=0)
     total_output_tokens = Column(Integer, default=0)
+    total_cost_usd = Column(Float, default=0.0)
     mode = Column(String, nullable=True)  # 'agent', 'chat', or 'research'
     crew_member_id = Column(String, nullable=True)  # links to crew_members.id
 
@@ -239,6 +248,11 @@ class Session(TimestampMixin, Base):
             'endpoint_url': self.endpoint_url,
             'rag': self.rag,
             'archived': self.archived,
+            'memory_extraction_enabled': self.memory_extraction_enabled is not False,
+            'skill_injection_enabled': self.skill_injection_enabled is not False,
+            'thinking_mode': self.thinking_mode or '',
+            'temperature_override': self.temperature_override,
+            'max_tokens_override': self.max_tokens_override,
             'created_at': self.created_at.isoformat() if self.created_at else None,
             'updated_at': self.updated_at.isoformat() if self.updated_at else None,
             'last_accessed': self.last_accessed.isoformat() if self.last_accessed else None,
@@ -248,6 +262,7 @@ class Session(TimestampMixin, Base):
             'folder': self.folder,
             'total_input_tokens': self.total_input_tokens or 0,
             'total_output_tokens': self.total_output_tokens or 0,
+            'total_cost_usd': self.total_cost_usd or 0.0,
             'crew_member_id': self.crew_member_id,
         }
 
@@ -280,6 +295,22 @@ class ChatMessage(Base):
         Index('ix_messages_session_time', 'session_id', 'timestamp'),  # Composite for efficient message retrieval
     )
 
+class BackgroundToolJob(Base):
+    """Durable origin and once-only chat delivery for background tool work."""
+    __tablename__ = "background_tool_jobs"
+    id = Column(String, primary_key=True)
+    session_id = Column(String, ForeignKey("sessions.id", ondelete="CASCADE"), nullable=False, index=True)
+    owner = Column(String, nullable=False, index=True)
+    tool = Column(String, nullable=False)
+    query = Column(Text, nullable=False)
+    rounds = Column(Integer, nullable=True)
+    status = Column(String, nullable=False, default="running", index=True)
+    payload = Column(Text, nullable=True)
+    summary = Column(Text, nullable=True)
+    message_id = Column(String, nullable=True)
+    created_at = Column(DateTime, default=utcnow_naive)
+
+
 class Document(TimestampMixin, Base):
     """Living document that the AI can create and edit in-place."""
     __tablename__ = "documents"
@@ -430,6 +461,93 @@ class EmailAccount(TimestampMixin, Base):
     )
 
 
+class EmailAccountOwnerLock(Base):
+    """Durable per-owner mutex for email-account default mutations.
+
+    Row-locking databases serialize mutations by locking this row before they
+    inspect or stage EmailAccount changes.  SQLite uses ``BEGIN IMMEDIATE``
+    instead, because it ignores ``SELECT ... FOR UPDATE``; keeping the table in
+    the shared metadata still makes the non-SQLite path available without a
+    separate migration.  The empty key represents the normalized legacy /
+    unconfigured scope shared by ``owner IS NULL`` and ``owner = ''`` rows.
+    """
+    __tablename__ = "email_account_owner_locks"
+
+    owner_key = Column(String, primary_key=True)
+
+
+_EMAIL_ACCOUNT_DEFAULT_INDEX = "ux_email_accounts_one_default_per_owner"
+_EMAIL_ACCOUNT_DEFAULT_INDEX_DDL = {
+    "sqlite": (
+        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
+        "ON email_accounts (COALESCE(owner, '')) WHERE is_default = 1"
+    ),
+    "postgresql": (
+        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
+        "ON email_accounts ((COALESCE(owner, ''))) WHERE is_default IS TRUE"
+    ),
+}
+
+
+# SQLAlchemy cannot express one portable partial, functional index across the
+# two supported database families.  Register dialect-specific DDL so fresh
+# databases get the invariant as part of create_all(); the startup migration
+# below installs the same index on existing databases after normalizing legacy
+# duplicate rows.
+for _dialect_name, _index_ddl in _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.items():
+    event.listen(
+        EmailAccount.__table__,
+        "after_create",
+        DDL(_index_ddl).execute_if(dialect=_dialect_name),
+    )
+
+
+def lock_email_account_owner_mutations(db, *owners: str) -> None:
+    """Lock normalized email-account owner scopes in canonical order.
+
+    ``NULL`` and the empty string are one legacy/single-user owner partition,
+    matching the unique default-account index.  SQLite has only a database
+    writer reservation, while row-locking databases use durable mutex rows.
+    Sorting all requested owner keys keeps multi-owner operations such as user
+    rename from deadlocking with another mutation that requests the same keys
+    in the opposite order.
+    """
+    from sqlalchemy.exc import IntegrityError
+
+    owner_keys = sorted({owner or "" for owner in owners} or {""})
+    if db.get_bind().dialect.name == "sqlite":
+        db.execute(text("BEGIN IMMEDIATE"))
+        return
+
+    for owner_key in owner_keys:
+        lock_row = db.get(
+            EmailAccountOwnerLock,
+            owner_key,
+            with_for_update=True,
+        )
+        if lock_row is not None:
+            continue
+
+        inserted = False
+        try:
+            with db.begin_nested():
+                db.add(EmailAccountOwnerLock(owner_key=owner_key))
+                db.flush()
+                inserted = True
+        except IntegrityError:
+            # A competing transaction created the mutex row first.  Once its
+            # insert commits, lock that durable row before touching accounts.
+            pass
+
+        if not inserted:
+            (
+                db.query(EmailAccountOwnerLock)
+                .filter(EmailAccountOwnerLock.owner_key == owner_key)
+                .with_for_update()
+                .one()
+            )
+
+
 class ModelEndpoint(TimestampMixin, Base):
     """Admin-configured model endpoints. Models are auto-discovered via /v1/models."""
     __tablename__ = "model_endpoints"
@@ -457,6 +575,9 @@ class ModelEndpoint(TimestampMixin, Base):
     # can be toggled per-endpoint in the UI. NULL = unknown, falls
     # back to the model-name keyword heuristic in agent_loop.py.
     supports_tools = Column(Boolean, nullable=True, default=None)
+    # JSON object: model id -> native tool schema surface preference.
+    # Values: none, compact, full. Missing key = legacy automatic behavior.
+    model_tool_modes = Column(Text, nullable=True)
     # Per-user ownership. NULL = legacy/shared (visible to every user) — this
     # is the historical default. When non-null, the model picker only shows
     # the endpoint to that user (admins always see everything).
@@ -743,6 +864,23 @@ class TaskRun(Base):
     )
 
 
+class NotificationLog(Base):
+    """Persisted task notifications, including completion and error text."""
+    __tablename__ = "notification_logs"
+
+    id         = Column(String, primary_key=True, index=True)
+    owner      = Column(String, nullable=True, index=True)
+    task_name  = Column(String, nullable=False)
+    task_id    = Column(String, nullable=True, index=True)
+    status     = Column(String, nullable=False, default="success")
+    body       = Column(Text, nullable=True)
+    timestamp  = Column(DateTime, nullable=False, default=utcnow_naive, index=True)
+
+    __table_args__ = (
+        Index('ix_notification_logs_owner_time', 'owner', 'timestamp'),
+    )
+
+
 class Memory(Base):
     """
     SQLAlchemy model for Memory table.
@@ -823,6 +961,74 @@ def _migrate_add_last_message_at_column():
         except Exception:
             pass
 
+def _migrate_add_memory_extraction_enabled_column():
+    """Add per-session auto memory extraction toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "memory_extraction_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN memory_extraction_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added memory_extraction_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"memory_extraction_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_skill_injection_enabled_column():
+    """Add per-session skill injection toggle."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()]
+        if "skill_injection_enabled" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN skill_injection_enabled BOOLEAN DEFAULT 1")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added skill_injection_enabled to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"skill_injection_enabled migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+def _migrate_add_session_generation_settings_columns():
+    """Add per-chat model generation controls."""
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = {row[1] for row in conn.execute("PRAGMA table_info(sessions)").fetchall()}
+        additions = {
+            "thinking_mode": "VARCHAR DEFAULT 'off'",
+            "temperature_override": "FLOAT",
+            "max_tokens_override": "INTEGER",
+        }
+        for name, sql_type in additions.items():
+            if name not in columns:
+                conn.execute(f"ALTER TABLE sessions ADD COLUMN {name} {sql_type}")
+        conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"session generation settings migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
 def _migrate_add_document_archived_column():
     """Add `archived` to documents (soft-archive flag). Guarded + idempotent."""
     import sqlite3
@@ -1072,6 +1278,30 @@ def _migrate_add_supports_tools_column():
             pass
 
 
+def _migrate_add_model_tool_modes_column():
+    """Add per-model tool-surface preferences to model_endpoints if missing."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(model_endpoints)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if columns and "model_tool_modes" not in columns:
+            conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_tool_modes TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'model_tool_modes' column to model_endpoints")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"model_tool_modes migration failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
+
 def _migrate_add_cached_models_column():
     """Add cached_models column to model_endpoints if it doesn't exist."""
     import sqlite3
@@ -1195,6 +1425,29 @@ def _migrate_add_folder_column():
         except Exception:
             pass
 
+def _migrate_add_session_cwd_column():
+    """Add cwd column to sessions table if it doesn't exist."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "cwd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN cwd TEXT")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added 'cwd' column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for cwd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_token_columns():
     """Add cumulative token tracking columns to sessions table."""
     import sqlite3
@@ -1219,6 +1472,29 @@ def _migrate_add_token_columns():
         except Exception:
             pass
 
+def _migrate_add_total_cost_usd():
+    """Add cumulative USD cost column to sessions table."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.execute("PRAGMA table_info(sessions)")
+        columns = [row[1] for row in cursor.fetchall()]
+        if "total_cost_usd" not in columns:
+            conn.execute("ALTER TABLE sessions ADD COLUMN total_cost_usd REAL DEFAULT 0.0")
+            conn.commit()
+            logging.getLogger(__name__).info("Migrated: added total_cost_usd column to sessions")
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"Migration check for total_cost_usd failed: {e}")
+    finally:
+        try:
+            conn.close()
+        except Exception:
+            pass
+
 def _migrate_add_owner_to_table(table_name: str, index_name: str):
     """Generic helper: add owner TEXT column + index to a table if missing."""
     import sqlite3
@@ -1404,8 +1680,25 @@ def _migrate_assign_legacy_owner():
             with open(prefs_path, "r", encoding="utf-8") as f:
                 prefs = _json.load(f)
             if "_users" not in prefs and prefs:
-                # Flat format → nest under admin user
-                new_prefs = {"_users": {admin_user: prefs}}
+                # Flat format → nest ordinary preferences under the admin
+                # user. Foreground fallback is an explicit per-owner opt-in,
+                # so auth-disabled consent must remain inert at the flat root
+                # rather than becoming consent for the first named owner.
+                foreground_keys = {
+                    "foreground_fallback_enabled",
+                    "foreground_model_fallbacks",
+                }
+                named_prefs = {
+                    key: value
+                    for key, value in prefs.items()
+                    if key not in foreground_keys
+                }
+                new_prefs = {
+                    key: prefs[key]
+                    for key in foreground_keys
+                    if key in prefs
+                }
+                new_prefs["_users"] = {admin_user: named_prefs}
                 with open(prefs_path, "w", encoding="utf-8") as f:
                     _json.dump(new_prefs, f, indent=2)
                 logger.info(f"Migrated user_prefs.json to per-user format under '{admin_user}'")
@@ -1720,6 +2013,7 @@ class Note(TimestampMixin, Base):
     session_id = Column(String, nullable=True)
     sort_order = Column(Integer, default=0)
     image_url  = Column(String, nullable=True)      # uploaded image URL (relative path)
+    gallery_id = Column(String, nullable=True, index=True)  # stable Gallery image for drawings
     repeat     = Column(String, default="none")     # none, daily, weekly, monthly, yearly
     # Auto-AI fields — populated by /api/notes/{id}/classify. The classification
     # JSON shape is { kind, solvable, confidence, task_prompt, tools, items?: [...] }.
@@ -1812,72 +2106,142 @@ class Integration(TimestampMixin, Base):
 
 
 
-def _migrate_seed_email_account():
-    """If email_accounts is empty and settings.json has legacy flat imap_host/smtp_host
-    keys, create a single default account from them so nothing breaks for users who
-    upgraded. Safe to run repeatedly — it short-circuits once any row exists."""
+def _migrate_email_account_default_invariant():
+    """Normalize legacy duplicates and install durable at-most-one enforcement.
+
+    Older databases only had a non-unique ``(owner, is_default)`` lookup index.
+    Keep the oldest default deterministically in each normalized owner scope,
+    then add the same partial functional unique index used for fresh schemas.
+    """
+    dialect_name = engine.dialect.name
+    index_ddl = _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.get(dialect_name)
+    if index_ddl is None:
+        logger.warning(
+            "Email-account default uniqueness is not available for database "
+            "dialect %s; mutations remain serialized but are not protected by "
+            "a database constraint",
+            dialect_name,
+        )
+        return
+
     try:
-        with engine.connect() as conn:
-            tables = [r[0] for r in conn.execute(text(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='email_accounts'"
-            ))]
-            if "email_accounts" not in tables:
-                return
-            existing = conn.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
-            if existing > 0:
+        with engine.begin() as conn:
+            if not inspect(conn).has_table(EmailAccount.__tablename__):
                 return
+            default_rows = conn.execute(text("""
+                SELECT id, owner
+                FROM email_accounts
+                WHERE is_default IS TRUE
+                ORDER BY
+                    COALESCE(owner, ''),
+                    CASE WHEN created_at IS NULL THEN 1 ELSE 0 END,
+                    created_at,
+                    id
+            """)).mappings()
+            seen_owner_keys = set()
+            duplicate_ids = []
+            for row in default_rows:
+                owner_key = row["owner"] or ""
+                if owner_key in seen_owner_keys:
+                    duplicate_ids.append(row["id"])
+                else:
+                    seen_owner_keys.add(owner_key)
 
-        import json as _json
-        import uuid as _uuid
-        from pathlib import Path
-        settings_file = Path(SETTINGS_FILE)
-        if not settings_file.exists():
-            return
-        try:
-            s = _json.loads(settings_file.read_text(encoding="utf-8"))
-        except Exception:
-            return
+            for account_id in duplicate_ids:
+                conn.execute(
+                    text("UPDATE email_accounts SET is_default = :value WHERE id = :id"),
+                    {"value": False, "id": account_id},
+                )
+            conn.execute(text(index_ddl))
 
-        imap_host = (s.get("imap_host") or "").strip()
-        smtp_host = (s.get("smtp_host") or "").strip()
-        if not imap_host and not smtp_host:
-            return  # nothing to migrate
+        if duplicate_ids:
+            logger.warning(
+                "Normalized %d duplicate default email account(s) before "
+                "installing %s",
+                len(duplicate_ids),
+                _EMAIL_ACCOUNT_DEFAULT_INDEX,
+            )
+    except Exception:
+        # Starting without the constraint would silently retain the race this
+        # migration is intended to close.  Fail startup so an operator sees and
+        # can repair an incompatible schema instead of accepting unsafe writes.
+        logger.exception("Failed to enforce the email-account default invariant")
+        raise
+
+
+def _migrate_seed_email_account():
+    """Atomically seed one legacy default account when no account exists.
+
+    Reading settings is intentionally done before taking the owner mutex.  The
+    decisive emptiness check and insert share one locked transaction, so two
+    application workers starting together cannot both seed a default row.
+    """
+    import json as _json
+    import uuid as _uuid
+
+    settings_file = Path(SETTINGS_FILE)
+    if not settings_file.exists():
+        return
+    try:
+        s = _json.loads(settings_file.read_text(encoding="utf-8"))
+    except Exception:
+        return
+
+    imap_host = (s.get("imap_host") or "").strip()
+    smtp_host = (s.get("smtp_host") or "").strip()
+    if not imap_host and not smtp_host:
+        return
+
+    db = None
+    try:
+        if not inspect(engine).has_table(EmailAccount.__tablename__):
+            return
+        db = SessionLocal()
+        lock_email_account_owner_mutations(db, "")
+        existing = db.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
+        if existing > 0:
+            return
 
         now = utcnow_naive()
-        with engine.begin() as conn:
-            conn.execute(text("""
-                INSERT INTO email_accounts
-                  (id, owner, name, is_default, enabled,
-                   imap_host, imap_port, imap_user, imap_password, imap_starttls,
-                   smtp_host, smtp_port, smtp_user, smtp_password,
-                   from_address, created_at, updated_at)
-                VALUES
-                  (:id, :owner, :name, :is_default, :enabled,
-                   :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
-                   :smtp_host, :smtp_port, :smtp_user, :smtp_password,
-                   :from_address, :created_at, :updated_at)
-            """), {
-                "id": _uuid.uuid4().hex,
-                "owner": None,
-                "name": "Default",
-                "is_default": True,
-                "enabled": True,
-                "imap_host": imap_host,
-                "imap_port": int(s.get("imap_port") or 993),
-                "imap_user": s.get("imap_user") or "",
-                "imap_password": s.get("imap_password") or "",
-                "imap_starttls": bool(s.get("imap_starttls", True)),
-                "smtp_host": smtp_host,
-                "smtp_port": int(s.get("smtp_port") or 465),
-                "smtp_user": s.get("smtp_user") or "",
-                "smtp_password": s.get("smtp_password") or "",
-                "from_address": s.get("email_from") or "",
-                "created_at": now,
-                "updated_at": now,
-            })
-            logging.getLogger(__name__).info("Seeded email_accounts 'Default' from settings.json")
+        db.execute(text("""
+            INSERT INTO email_accounts
+              (id, owner, name, is_default, enabled,
+               imap_host, imap_port, imap_user, imap_password, imap_starttls,
+               smtp_host, smtp_port, smtp_user, smtp_password,
+               from_address, created_at, updated_at)
+            VALUES
+              (:id, :owner, :name, :is_default, :enabled,
+               :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
+               :smtp_host, :smtp_port, :smtp_user, :smtp_password,
+               :from_address, :created_at, :updated_at)
+        """), {
+            "id": _uuid.uuid4().hex,
+            "owner": None,
+            "name": "Default",
+            "is_default": True,
+            "enabled": True,
+            "imap_host": imap_host,
+            "imap_port": int(s.get("imap_port") or 993),
+            "imap_user": s.get("imap_user") or "",
+            "imap_password": s.get("imap_password") or "",
+            "imap_starttls": bool(s.get("imap_starttls", True)),
+            "smtp_host": smtp_host,
+            "smtp_port": int(s.get("smtp_port") or 465),
+            "smtp_user": s.get("smtp_user") or "",
+            "smtp_password": s.get("smtp_password") or "",
+            "from_address": s.get("email_from") or "",
+            "created_at": now,
+            "updated_at": now,
+        })
+        db.commit()
+        logger.info("Seeded email_accounts 'Default' from settings.json")
     except Exception as e:
-        logging.getLogger(__name__).warning(f"seed email account migration: {e}")
+        if db is not None:
+            db.rollback()
+        logger.warning("seed email account migration: %s", e)
+    finally:
+        if db is not None:
+            db.close()
 
 
 # WARNING: Foreign-key enforcement is enabled globally for all SQLite connections.
@@ -1935,12 +2299,18 @@ def init_db():
     _migrate_add_model_endpoint_owner_column()
     _migrate_add_provider_auth_id_column()
     _migrate_add_supports_tools_column()
+    _migrate_add_model_tool_modes_column()
     _migrate_add_task_run_model_column()
     _migrate_add_owner_column()
     _migrate_add_document_archived_column()
     _migrate_add_last_message_at_column()
+    _migrate_add_memory_extraction_enabled_column()
+    _migrate_add_skill_injection_enabled_column()
+    _migrate_add_session_generation_settings_columns()
     _migrate_add_folder_column()
+    _migrate_add_session_cwd_column()
     _migrate_add_token_columns()
+    _migrate_add_total_cost_usd()
     _migrate_add_mode_column()
     _migrate_add_multiuser_owner_columns()
     _migrate_add_gallery_caption_column()
@@ -1960,6 +2330,7 @@ def init_db():
     _migrate_add_crew_member_id()
     _migrate_add_assistant_columns()
     _migrate_add_email_smtp_security()
+    _migrate_email_account_default_invariant()
     _migrate_seed_email_account()
     _migrate_add_calendar_metadata()
     _migrate_add_calendar_is_utc()
@@ -1967,6 +2338,7 @@ def init_db():
     _migrate_add_calendar_account_id()
     _migrate_add_caldav_sync_columns()
     _migrate_add_calendar_recurrence_exdates()
+    _migrate_add_note_gallery_id()
     _migrate_chat_messages_fts()
     _migrate_encrypt_email_passwords()
     _migrate_encrypt_signatures()
@@ -2064,17 +2436,33 @@ def _migrate_chat_messages_fts():
             END;
             """
         )
-        conn.execute(
-            f"""
-            INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-            SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
-            FROM chat_messages cm
-            WHERE NOT EXISTS (
-                SELECT 1 FROM chat_messages_fts fts
-                WHERE fts.message_id = cm.id
+        # message_id is deliberately UNINDEXED in the FTS table.  A correlated
+        # NOT EXISTS against it therefore becomes quadratic once the transcript
+        # grows large, even when there is nothing left to backfill.  Build a
+        # temporary indexed set only when the row counts show that reconciliation
+        # is needed.  Normal inserts/updates/deletes stay synchronized by the
+        # triggers above.
+        chat_count = conn.execute("SELECT COUNT(*) FROM chat_messages").fetchone()[0]
+        fts_count = conn.execute("SELECT COUNT(*) FROM chat_messages_fts").fetchone()[0]
+        if chat_count != fts_count:
+            conn.execute(
+                "CREATE TEMP TABLE IF NOT EXISTS _odysseus_fts_message_ids "
+                "(message_id TEXT PRIMARY KEY) WITHOUT ROWID"
+            )
+            conn.execute("DELETE FROM temp._odysseus_fts_message_ids")
+            conn.execute(
+                "INSERT OR IGNORE INTO temp._odysseus_fts_message_ids(message_id) "
+                "SELECT message_id FROM chat_messages_fts"
+            )
+            conn.execute(
+                f"""
+                INSERT INTO chat_messages_fts(content, message_id, session_id, role)
+                SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
+                FROM chat_messages cm
+                LEFT JOIN temp._odysseus_fts_message_ids known ON known.message_id = cm.id
+                WHERE known.message_id IS NULL
+                """
             )
-            """
-        )
         _scrub_legacy_chat_message_fts_media(conn)
         conn.commit()
     except Exception as e:
@@ -2390,6 +2778,27 @@ def _migrate_add_calendar_recurrence_exdates():
         except Exception:
             pass
 
+def _migrate_add_note_gallery_id():
+    """Keep a drawn note linked to one Gallery image across edits."""
+    import sqlite3
+    db_path = DATABASE_URL.replace("sqlite:///", "")
+    if not os.path.exists(db_path):
+        return
+    conn = None
+    try:
+        conn = sqlite3.connect(db_path)
+        columns = [row[1] for row in conn.execute("PRAGMA table_info(notes)").fetchall()]
+        if columns and "gallery_id" not in columns:
+            conn.execute("ALTER TABLE notes ADD COLUMN gallery_id VARCHAR")
+            conn.execute("CREATE INDEX IF NOT EXISTS ix_notes_gallery_id ON notes(gallery_id)")
+            conn.commit()
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"notes gallery_id migration failed: {e}")
+    finally:
+        if conn is not None:
+            conn.close()
+
+
 def get_db():
     """
     Dependency to get a database session.

core/middleware.py

@@ -3,10 +3,14 @@
 
 import os
 import secrets
+from collections.abc import Mapping
 
 from fastapi import HTTPException, Request
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import Response
+from starlette.routing import get_route_path
+
+from src.owner_identity import INTERNAL_TOOL_USER, auth_disabled
 
 
 # Per-process token that lets the in-app tool layer hit admin-gated
@@ -15,8 +19,30 @@ from starlette.responses import Response
 # same value from this module. Never persisted or exposed externally.
 INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32)
 INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token"
-# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved.
-INTERNAL_TOOL_USER = "internal-tool"
+
+
+def get_application_route_path(scope: Mapping[str, object]) -> str:
+    """Return the application-relative path used by Starlette routing.
+
+    Uvicorn prefixes ``scope["path"]`` with a configured ASGI ``root_path``;
+    Starlette removes that prefix before matching routes. Middleware policy
+    must use the same path form or a deployment prefix can change which policy
+    applies to an otherwise unchanged application route.
+    """
+    return get_route_path(scope)
+
+
+def with_asgi_root_path(scope: Mapping[str, object], path: str) -> str:
+    """Prefix an application path for a client-facing redirect target."""
+    root_path = scope.get("root_path", "")
+    if not isinstance(root_path, str) or not root_path:
+        return path
+    return f"{root_path.rstrip('/')}{path}"
+
+
+def path_is_route_or_child(path: str, prefix: str) -> bool:
+    """Return whether ``path`` is exactly ``prefix`` or below that route."""
+    return path == prefix or path.startswith(prefix + "/")
 
 
 def is_cors_preflight(method: str, headers) -> bool:
@@ -47,7 +73,7 @@ def require_admin(request: Request):
         pass
 
     auth_mgr = getattr(request.app.state, "auth_manager", None)
-    if os.getenv("AUTH_ENABLED", "true").lower() == "false":
+    if auth_disabled():
         return
     if not auth_mgr or not auth_mgr.is_configured:
         raise HTTPException(403, "Admin only")

core/models.py

@@ -8,6 +8,11 @@ These are simple datacontainers. All persistence is handled by SessionManager.
 from dataclasses import dataclass
 from typing import Dict, List, Any, Optional, TYPE_CHECKING
 
+from src.tool_approval_scopes import (
+    CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
+    CHAT_SESSION_APPROVAL_DECISION,
+)
+
 if TYPE_CHECKING:
     from .session_manager import SessionManager
 
@@ -31,6 +36,35 @@ set_session_manager = set_session_manager_instance
 get_session_manager = get_session_manager_instance
 
 
+def _history_grants_chat_session_approval(
+    history: List["ChatMessage"],
+    session_id: str,
+) -> bool:
+    """Return whether this exact chat has a resolved session-scope grant."""
+
+    expected_session = str(session_id or "")
+    if not expected_session:
+        return False
+    for message in reversed(history or []):
+        metadata = getattr(message, "metadata", None)
+        if not isinstance(metadata, dict):
+            continue
+        tool_events = metadata.get("tool_events")
+        if not isinstance(tool_events, list):
+            continue
+        for event in reversed(tool_events):
+            ask_user = event.get("ask_user") if isinstance(event, dict) else None
+            if not isinstance(ask_user, dict):
+                continue
+            if (
+                ask_user.get("kind") == "tool_approval"
+                and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
+                and str(ask_user.get("session_id") or "") == expected_session
+            ):
+                return True
+    return False
+
+
 @dataclass
 class ChatMessage:
     """A single chat message."""
@@ -74,6 +108,12 @@ class Session:
     owner: Optional[str] = None
     is_important: bool = False
     message_count: int = 0
+    memory_extraction_enabled: bool = True
+    skill_injection_enabled: bool = True
+    thinking_mode: str = "off"
+    temperature_override: Optional[float] = None
+    max_tokens_override: Optional[int] = None
+    cwd: Optional[str] = None
 
     def __post_init__(self):
         if self.headers is None:
@@ -116,11 +156,45 @@ class Session:
         the model. Display/history-load paths use the raw ``history`` and are
         unaffected.
         """
-        return [
+        messages = [
             msg.to_dict()
             for msg in self.history
             if (msg.metadata or {}).get("source") != "slash"
         ]
+        from src.background_tool_jobs import background_result_context
+        messages = [part for message in messages for part in (
+            *background_result_context(message.get('metadata')), message,
+        )]
+        # Resume an interrupted thinking-only response from its actual model
+        # reasoning channel. Restrict this to the latest assistant message so
+        # old traces do not accumulate in context or cause reasoning loops.
+        for index in range(len(messages) - 1, -1, -1):
+            message = messages[index]
+            if message.get("role") != "assistant":
+                continue
+            metadata = message.get("metadata") or {}
+            thinking = str(metadata.get("thinking") or "").strip()
+            if metadata.get("stopped") and thinking:
+                resumed = dict(message)
+                resumed["reasoning_content"] = thinking
+                messages[index] = resumed
+            break
+        if not _history_grants_chat_session_approval(self.history, self.id):
+            return messages
+
+        # Keep the grant close to the latest user request so route-neutral
+        # compaction/trimming preserves it. Copy the metadata instead of
+        # mutating the durable transcript object.
+        for index in range(len(messages) - 1, -1, -1):
+            if messages[index].get("role") != "user":
+                continue
+            message = dict(messages[index])
+            metadata = dict(message.get("metadata") or {})
+            metadata[CHAT_SESSION_APPROVAL_CONTEXT_MARKER] = True
+            message["metadata"] = metadata
+            messages[index] = message
+            break
+        return messages
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""

core/session_manager.py

@@ -14,6 +14,8 @@ import logging
 from datetime import datetime, timezone, timedelta
 from typing import Dict, Optional
 
+from sqlalchemy import func
+
 from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
 from .models import Session, ChatMessage
 from src.attachment_refs import persistable_message_content
@@ -92,14 +94,28 @@ class SessionManager:
         try:
             db_sessions = db.query(DbSession).filter(
                 DbSession.archived == False,
-                DbSession.message_count > 0,
+                DbSession.messages.any(),
             ).order_by(DbSession.last_accessed.desc()).limit(100).all()
 
+            # message_count is derived metadata and can drift after interrupted
+            # or legacy writes. Count only the bounded discovery set so startup
+            # remains metadata-only while lazy hydration sees an authoritative
+            # positive count for every discovered non-empty session.
+            message_counts = {}
+            if db_sessions:
+                message_counts = dict(
+                    db.query(DbChatMessage.session_id, func.count(DbChatMessage.id))
+                    .filter(DbChatMessage.session_id.in_([row.id for row in db_sessions]))
+                    .group_by(DbChatMessage.session_id)
+                    .all()
+                )
+
             loaded_count = 0
             for db_session in db_sessions:
                 try:
                     session = self._db_to_session_meta(db_session)
                     if session is not None:
+                        session.message_count = message_counts[db_session.id]
                         self.sessions[db_session.id] = session
                         loaded_count += 1
                 except Exception as e:
@@ -134,6 +150,12 @@ class SessionManager:
             history=[],
             owner=getattr(db_session, "owner", None),
             is_important=getattr(db_session, "is_important", False) or False,
+            memory_extraction_enabled=getattr(db_session, "memory_extraction_enabled", True) is not False,
+            skill_injection_enabled=getattr(db_session, "skill_injection_enabled", True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
         session.message_count = getattr(db_session, "message_count", 0) or 0
         return session
@@ -192,9 +214,20 @@ class SessionManager:
             history=history,
             owner=getattr(db_session, 'owner', None),
             is_important=getattr(db_session, 'is_important', False) or False,
+            memory_extraction_enabled=getattr(db_session, 'memory_extraction_enabled', True) is not False,
+            skill_injection_enabled=getattr(db_session, 'skill_injection_enabled', True) is not False,
+            thinking_mode=getattr(db_session, "thinking_mode", "") or "off",
+            temperature_override=getattr(db_session, "temperature_override", None),
+            max_tokens_override=getattr(db_session, "max_tokens_override", None),
+            cwd=getattr(db_session, "cwd", None) or None,
         )
 
-        session.message_count = getattr(db_session, 'message_count', len(history))
+        # The rows just loaded are the whole transcript, so they — not the
+        # denormalized sessions.message_count column — are the truth for this
+        # cached object. get_session's hydration gate compares against this
+        # number; seeding it from a drifted column would ask for a reload that
+        # can never close the gap.
+        session.message_count = len(history)
         return session
 
     # ------------------------------------------------------------------
@@ -398,30 +431,50 @@ class SessionManager:
     # ------------------------------------------------------------------
 
     def get_session(self, session_id: str) -> Session:
-        """Get a session by ID, loading from DB if needed.
+        """Get a session by ID, loading complete DB history when needed.
 
-        Sessions seeded by `load_sessions` start with empty history. The
-        first read here hydrates them with the message rows.
+        Sessions seeded by ``load_sessions`` start with empty history, and a
+        cached session can also become partially stale. Refresh metadata first,
+        then hydrate whenever the cached transcript is short of the stored rows.
+        Model-send routes enter through this method before building context,
+        while paginated display history reads SQLite directly.
+
+        The gate compares against ``sync_session_metadata``'s reconciled count
+        (the real ``chat_messages`` total), never the denormalized column, so a
+        hydrate always closes the gap and the next read is a cache hit.
         """
         if session_id not in self.sessions:
             self._load_session_from_db(session_id)
-        else:
-            cached = self.sessions[session_id]
-            # Lazy hydrate: metadata-only entries get their messages on first read.
-            if not cached.history and getattr(cached, "message_count", 0) > 0:
-                self._load_session_from_db(session_id)
 
         # Keep model/endpoint metadata fresh. Endpoint deletion can clear the
-        # DB row while a session object is still cached in RAM.
+        # DB row while a session object is still cached in RAM. Refreshing first
+        # also exposes the authoritative message count before completeness is
+        # checked.
         self.sync_session_metadata(session_id)
 
+        cached = self.sessions[session_id]
+        cached_count = len(cached.history or [])
+        stored_count = int(getattr(cached, "message_count", 0) or 0)
+        if cached_count < stored_count:
+            self._load_session_from_db(session_id)
+
         # Update last_accessed
         self._touch_session(session_id)
 
         return self.sessions[session_id]
 
     def sync_session_metadata(self, session_id: str) -> bool:
-        """Refresh non-message session fields from the DB into the cached object."""
+        """Refresh non-message session fields from the DB into the cached object.
+
+        ``message_count`` is reconciled against the real ``chat_messages`` rows
+        rather than copied from the denormalized ``sessions.message_count``
+        column. That column drifts in normal operation — ``_persist_message``
+        swallows a failed insert but ``add_message`` has already appended in
+        memory, so the next successful persist writes rows+1, and a persist for
+        an uncached session writes 0. Hydration keys off this number: a
+        drifted-high column would reload the whole transcript on every warm
+        read, and a drifted-low one would leave the model a truncated one.
+        """
         session = self.sessions.get(session_id)
         if session is None:
             return False
@@ -444,7 +497,12 @@ class SessionManager:
             session.archived = db_session.archived
             session.owner = getattr(db_session, "owner", None)
             session.is_important = getattr(db_session, "is_important", False) or False
-            session.message_count = getattr(db_session, "message_count", session.message_count) or 0
+            session.cwd = getattr(db_session, "cwd", None) or None
+            session.message_count = (
+                db.query(DbChatMessage)
+                .filter(DbChatMessage.session_id == session_id)
+                .count()
+            )
             return True
         except Exception as e:
             logger.error(f"Error syncing session metadata {session_id}: {e}")
@@ -500,9 +558,12 @@ class SessionManager:
         endpoint_url: str,
         model: str,
         rag: bool = False,
-        owner: str = None
+        owner: str = None,
+        cwd: str = None,
+        headers: Optional[Dict[str, str]] = None,
     ) -> Session:
         """Create a new session and save to database."""
+        session_headers = dict(headers or {})
         db = SessionLocal()
         try:
             db_session = DbSession(
@@ -511,8 +572,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
                 created_at=datetime.now(timezone.utc),
                 updated_at=datetime.now(timezone.utc)
             )
@@ -525,8 +587,9 @@ class SessionManager:
                 endpoint_url=endpoint_url,
                 model=model,
                 rag=rag,
-                headers={},
+                headers=session_headers,
                 owner=owner,
+                cwd=cwd or None,
             )
 
             self.sessions[session_id] = session

discovery/README.md

@@ -1,34 +0,0 @@
-# Odysseus discovery maps
-
-Compact, code-grounded discovery maps of cross-cutting systems in the checked-in Odysseus codebase. They preserve investigation context and open factual questions; they are not canonical subsystem specifications, a feature certification, or a substitute for normal testing.
-
-> [!IMPORTANT]
-> Checked-in code is the source of truth for current behaviour. Mature subsystem specifications, where they exist, are the canonical documentation of accepted subsystem behaviour. Check code, tests, and configuration before reconciling a discovery finding. Discovery remains non-canonical.
-
-## Explore the maps
-
-| Document | Purpose |
-|---|---|
-| [Current system map](system-map.md) | Records evidence locations, confirmed local observations, and factual open questions about subsystem boundaries. |
-| [Safety boundaries](safety-boundaries.md) | Records evidence about broad authority, safeguards, confirmed risks or gaps, and unverified behaviour. |
-
-## Working rules
-
-- **Trace the code first.** Confirm the current path in source before recording a claim.
-- **Promote selectively.** When an owning mature specification exists, add a fact only when it is verified, useful, and not already represented there.
-- **Record missing ownership.** When no owning specification exists, retain the verified finding in discovery and record missing documentation ownership as a follow-up.
-- **Retain uncertainty here.** Keep unresolved questions and useful investigation context in discovery rather than treating them as canonical truth.
-- **Keep specifications current-state only.** Do not record intentions, design direction, refactor plans, decision history, priority, ownership, or sequencing here.
-- **Investigate with cause.** Do not exhaustively revalidate existing functionality without a report, visible failure, relevant change, or high-authority review need.
-- **Review authority carefully.** Give execution, data access, external tools, credentials, destructive operations, and unattended work focused review.
-- **Use stable locations.** Cite modules, routes, classes, and functions instead of fragile line ranges or generated evidence tables.
-
-## Reconciliation flow
-
-1. Start with the relevant map and trace the cited code.
-2. Classify the finding against current source evidence and an owning mature specification where one exists.
-3. Promote only verified, useful facts that are missing from an existing owning specification.
-4. When no owning specification exists, retain the verified finding here and record missing documentation ownership as a follow-up; otherwise retain unresolved context here and correct stale wording.
-
-> [!NOTE]
-> This package intentionally contains no generator, validator, maturity scale, feature database, or parallel work tracker. The [architecture runtime inventory](./architecture-runtime-inventory.md) preserves dated structural metrics, investigation context, and historical planning as an explicitly non-canonical snapshot.

discovery/architecture-runtime-inventory.md

@@ -1,331 +0,0 @@
-# Architecture runtime inventory
-
-> [!WARNING]
-> This document is a dated structural snapshot, not a canonical runtime specification.
-> Counts, paths, and implementation details may drift as the repository changes.
-> Verify implementation-sensitive claims against the current code and tests.
-
-- **Branch:** `discovery`
-- **Commit:** `c762efe1c97a`
-- **Generated:** `2026-07-28T05:38:14+01:00`
-- **Historical context:** readability and refactor planning in [#4071](https://github.com/odysseus-dev/odysseus/issues/4071) and [#4082](https://github.com/odysseus-dev/odysseus/issues/4082)
-
-## Disposition
-
-> [!NOTE]
-> Reviewed for documentation classification. Stable runtime structure and
-> subsystem ownership have been transferred to the proposed canonical
-> destination, [`docs/ARCHITECTURE.md`](../docs/ARCHITECTURE.md), for
-> maintainer review.
->
-> This document retains dated metrics, rankings, investigation context,
-> refactor-sensitive observations, and historical planning. Those contents are
-> non-canonical and belong under `discovery/`.
-
-| Content | Authority and destination |
-|---|---|
-| Stable runtime structure | Proposed canonical destination: `docs/ARCHITECTURE.md` |
-| Stable subsystem boundaries | Proposed canonical destination: `docs/ARCHITECTURE.md` |
-| Frontend module organization | `static/js/MODULE_SUMMARY.md` |
-| Counts, line totals, and rankings | This non-canonical inventory |
-| Investigation context and open questions | `discovery/` |
-| Refactor options and prioritization | Issues, Plane, or non-canonical discovery material |
-
-The transfer preserves the stable facts without promoting generated metrics or
-historical prioritization into canonical documentation.
-
-## Purpose
-
-This inventory provides a reviewable map of the current repository structure,
-large runtime modules, major subsystem boundaries, and refactor-sensitive areas.
-
-It does not:
-
-- define accepted subsystem behaviour;
-- certify runtime correctness;
-- prescribe a committed refactor sequence;
-- replace focused specifications, tests, or source review.
-
-For cross-cutting implementation evidence, see the [discovery maps](./README.md).
-
-## Top-level runtime structure
-
-| Area | Role |
-|---|---|
-| `app.py` | FastAPI application composition and entry point |
-| `launcher.py` | Application launch support |
-| `setup.py` | Native setup workflow |
-| `core/` | Authentication, middleware, persistence, sessions, and platform primitives |
-| `routes/` | HTTP and API route handlers |
-| `src/` | Application services, orchestration, tools, providers, and runtime helpers |
-| `services/` | Domain-oriented service packages |
-| `mcp_servers/` | Built-in MCP server implementations |
-| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers |
-| `static/` | No-build browser frontend and bundled assets |
-| `tests/` | Automated test suite and supporting test infrastructure |
-
-## Directory snapshot
-
-| Directory | Tracked files | Tracked Python files | Direct subdirectories |
-|---|---:|---:|---|
-| `src/` | 143 | 143 | `agent_tools/`, `model_capability_readers/`, `search/`, `tools/` |
-| `routes/` | 73 | 73 | `admin_wipe/`, `cleanup/`, `compare/`, `contacts/`, `gallery/`, `history/`, `memory/`, `note/`, `research/` |
-| `core/` | 11 | 11 | None |
-| `services/` | 42 | 40 | `docs/`, `faces/`, `hwfit/`, `memory/`, `research/`, `search/`, `shell/`, `stt/`, `tts/`, `youtube/` |
-| `mcp_servers/` | 5 | 5 | None |
-| `scripts/` | 44 | 17 | `_completion/`, `_lib/`, `demo_email/` |
-| `static/js/` | 154 | 0 | `calendar/`, `color/`, `compare/`, `editor/`, `emailLibrary/`, `markdown/`, `model/`, `research/`, `util/` |
-| `tests/` | 768 | 758 | `cli/`, `helpers/`, `streaming/`, `tools/` |
-
-> [!NOTE]
-> Counts in this table use `git ls-files`, so generated caches, virtual
-> environments, and other untracked local files are excluded.
-
-## Largest backend modules
-
-Large files are review signals, not proof that a module should be split.
-Coupling, ownership, import compatibility, tests, and runtime authority matter more
-than line count alone.
-
-| Rank | File | Lines | Classes | Top-level functions | Review signal |
-|---:|---|---:|---:|---:|---|
-| 1 | `routes/email_routes.py` | 6032 | 1 | 58 | High |
-| 2 | `src/agent_loop.py` | 5248 | 0 | 63 | High |
-| 3 | `routes/cookbook_routes.py` | 4545 | 0 | 16 | High |
-| 4 | `mcp_servers/email_server.py` | 2920 | 0 | 77 | High |
-| 5 | `src/llm_core.py` | 2895 | 3 | 85 | High |
-| 6 | `src/builtin_actions.py` | 2845 | 2 | 27 | High |
-| 7 | `routes/model_routes.py` | 2743 | 0 | 65 | High |
-| 8 | `src/task_scheduler.py` | 2627 | 1 | 8 | Medium |
-| 9 | `core/database.py` | 2562 | 28 | 67 | High |
-| 10 | `routes/gallery/gallery_routes.py` | 2325 | 0 | 16 | Medium |
-| 11 | `routes/chat_routes.py` | 2063 | 0 | 18 | Medium |
-| 12 | `routes/shell_routes.py` | 1971 | 1 | 21 | Medium |
-| 13 | `src/visual_report.py` | 1933 | 0 | 11 | Medium |
-| 14 | `routes/email_helpers.py` | 1888 | 3 | 48 | Medium |
-| 15 | `routes/document_routes.py` | 1810 | 0 | 5 | Medium |
-| 16 | `src/tools/cookbook.py` | 1705 | 0 | 34 | Medium |
-| 17 | `routes/calendar_routes.py` | 1667 | 2 | 19 | Medium |
-| 18 | `routes/skills_routes.py` | 1662 | 3 | 19 | Medium |
-| 19 | `src/tool_schemas.py` | 1595 | 0 | 3 | Medium |
-| 20 | `routes/email_pollers.py` | 1551 | 0 | 23 | Medium |
-
-The largest current backend concentrations include:
-
-- email routing and helper logic;
-- agent-loop orchestration;
-- Cookbook lifecycle and serving logic;
-- provider and model routing;
-- task scheduling;
-- shared database models and persistence helpers.
-
-These areas require focused ownership and compatibility analysis before structural
-changes are attempted.
-
-## Largest frontend modules
-
-| Rank | File | Lines |
-|---:|---|---:|
-| 1 | `static/style.css` | 41132 |
-| 2 | `static/js/document.js` | 11200 |
-| 3 | `static/js/emailLibrary.js` | 8505 |
-| 4 | `static/js/slashCommands.js` | 6520 |
-| 5 | `static/js/chat.js` | 6001 |
-| 6 | `static/js/settings.js` | 5819 |
-| 7 | `static/js/notes.js` | 5365 |
-| 8 | `static/app.js` | 4681 |
-| 9 | `static/js/cookbookRunning.js` | 4433 |
-| 10 | `static/js/galleryEditor.js` | 4386 |
-| 11 | `static/js/cookbookServe.js` | 4305 |
-| 12 | `static/js/calendar.js` | 3722 |
-| 13 | `static/js/cookbook.js` | 3677 |
-| 14 | `static/js/sessions.js` | 3665 |
-| 15 | `static/js/documentLibrary.js` | 3422 |
-| 16 | `static/js/tasks.js` | 3187 |
-| 17 | `static/js/admin.js` | 3144 |
-| 18 | `static/js/gallery.js` | 2958 |
-| 19 | `static/js/cookbook-hwfit.js` | 2826 |
-| 20 | `static/js/chatRenderer.js` | 2808 |
-
-The browser frontend remains a no-build ES-module application. Its current source
-tree is authoritative; the maintained structural summary is available in
-[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md).
-
-CSS modularization remains tracked separately in
-[#2617](https://github.com/odysseus-dev/odysseus/issues/2617).
-
-## Major subsystem boundaries
-
-| Subsystem | Primary implementation locations |
-|---|---|
-| Application startup | `app.py`, `src/app_initializer.py`, `core/` |
-| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` |
-| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` |
-| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` |
-| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` |
-| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` |
-| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` |
-| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` |
-| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` |
-| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` |
-| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` |
-| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` |
-| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` |
-
-For a broader evidence map, see
-[`system-map.md`](./system-map.md).
-
-## Refactor-sensitive areas
-
-### Shared persistence
-
-`core/database.py` is a central dependency containing models and shared persistence
-helpers. Changes can affect routes, services, background work, tests, migrations,
-and import compatibility.
-
-A split should not begin from file size alone. It requires:
-
-- an importer inventory;
-- model and helper ownership decisions;
-- migration compatibility checks;
-- stable re-export or migration strategy;
-- focused and full-suite validation.
-
-### Agent orchestration
-
-`src/agent_loop.py` coordinates model interaction, tool selection, policy decisions,
-multi-round execution, and background behaviour. Extraction work must preserve tool
-event semantics, policy enforcement, cancellation, and test patch points.
-
-Historical agent-loop modularization discussion is tracked in
-[#3266](https://github.com/odysseus-dev/odysseus/issues/3266).
-
-### Tool implementation boundaries
-
-Tool implementation is no longer represented by one proposed future package alone.
-Current responsibilities are distributed across:
-
-- `src/tool_execution.py`;
-- `src/tool_schemas.py`;
-- `src/tool_index.py`;
-- `src/tool_policy.py`;
-- `src/tool_security.py`;
-- `src/agent_tools/`;
-- `src/tools/`;
-- remaining compatibility surfaces such as `src/tool_implementations.py`.
-
-Historical tool modularization work is tracked in
-[#3629](https://github.com/odysseus-dev/odysseus/issues/3629).
-
-### Route ownership
-
-`routes/` now contains both flat modules and domain packages. Existing package
-boundaries should be extended only through focused changes. Broad mechanical route
-movement would affect registration, imports, tests, monkeypatch targets, and
-compatibility paths.
-
-### Frontend concentration
-
-The no-build frontend contains several large JavaScript modules and one central CSS
-file. Refactors should preserve module load order, global compatibility exports,
-DOM contracts, deep-link handling, and browser behaviour.
-
-## Non-implemented architecture options
-
-> [!NOTE]
-> The paths below are historical or possible design directions. They do not describe
-> the current repository and are not approved implementation plans.
-
-Earlier planning discussed:
-
-- renaming `app.py` to `main.py`;
-- moving agent orchestration into a new `src/agent/` package;
-- introducing broad `src/domain/`, `src/infra/`, `src/api/`, or `src/pkg/` layers;
-- moving all routes into domain subpackages;
-- splitting database models into a new infrastructure hierarchy.
-
-These options should be reconsidered against the current tree rather than copied
-forward as assumed targets.
-
-## Refactor guardrails
-
-- Keep structural changes behaviour-preserving.
-- Change one ownership boundary at a time.
-- Do not mix file movement with unrelated feature work.
-- Preserve existing import and monkeypatch paths where compatibility is required.
-- Identify focused tests before modifying high-authority modules.
-- Validate startup, imports, and affected runtime paths.
-- Avoid repository-wide package reorganizations without maintainer agreement.
-- Treat generated metrics as snapshots, not architectural decisions.
-
-## Reproduce the snapshot
-
-Run these commands from the repository root.
-
-```bash
-# Tracked directory totals
-for dir in src routes core services mcp_servers scripts static/js tests; do
-  files="$(git ls-files "$dir" | wc -l)"
-  python_files="$(git ls-files "$dir" '*.py' | wc -l)"
-
-  printf '%-14s tracked=%-5s python=%-5s\n' \
-    "$dir" \
-    "$files" \
-    "$python_files"
-done
-
-# Largest tracked backend files
-git ls-files \
-  'app.py' \
-  'launcher.py' \
-  'setup.py' \
-  'core/*.py' \
-  'core/**/*.py' \
-  'routes/*.py' \
-  'routes/**/*.py' \
-  'services/*.py' \
-  'services/**/*.py' \
-  'src/*.py' \
-  'src/**/*.py' \
-  'mcp_servers/*.py' \
-  'scripts/*.py' \
-  'scripts/**/*.py' |
-xargs wc -l |
-sort -nr |
-head -31
-
-# Largest tracked frontend source files
-git ls-files \
-  'static/*.js' \
-  'static/*.css' \
-  'static/*.html' \
-  'static/**/*.js' \
-  'static/**/*.css' \
-  'static/**/*.html' |
-grep -vE '\.min\.js$' |
-xargs wc -l |
-sort -nr |
-head -31
-```
-
-## Validation for architecture changes
-
-Use the smallest relevant checks first, then expand according to risk:
-
-```bash
-python3 -m compileall -q app.py core routes services src
-venv/bin/python -m pytest tests/<focused-test-file>.py -q
-venv/bin/python -m pytest -q
-```
-
-Startup, browser, Docker, and integration checks may also be required depending on
-the affected boundary.
-
-## Related documentation
-
-- [Documentation style](../docs/STYLE.md)
-- [Discovery maps](../discovery/README.md)
-- [Current system map](../discovery/system-map.md)
-- [Safety boundaries](../discovery/safety-boundaries.md)
-- [Frontend module summary](../static/js/MODULE_SUMMARY.md)
-- [Testing standard](../tests/TESTING_STANDARD.md)

discovery/safety-boundaries.md

@@ -1,142 +0,0 @@
-# Safety boundaries
-
-> [!IMPORTANT]
-> This non-canonical discovery map records code-grounded safeguards, confirmed risks or gaps, and unverified behaviour. Broad authority does not by itself establish a vulnerability. Verify the cited source before relying on a finding. No destructive test, external connection, or real credential was used for this map.
-
-## Navigate the boundaries
-
-- [Shell and subprocess execution](#shell-and-subprocess-execution)
-- [Filesystem access and workspace confinement](#filesystem-access-and-workspace-confinement)
-- [Agent-controlled tool dispatch](#agent-controlled-tool-dispatch)
-- [MCP and external tool servers](#mcp-and-external-tool-servers)
-- [Outbound network requests and URL validation](#outbound-network-requests-and-url-validation)
-- [Secrets, credentials, and vault sessions](#secrets-credentials-and-vault-sessions)
-- [Authentication and privileged administration](#authentication-and-privileged-administration)
-- [Deletion, wipe, backup, and restore](#deletion-wipe-backup-and-restore)
-- [Background jobs and unattended task execution](#background-jobs-and-unattended-task-execution)
-
-## Shell and subprocess execution
-
-- **Boundary:** Shell routes, agent `bash` and `python` tools, local model serving, and detached background jobs.
-
-- **Available authority:** Commands run as the application process user and can create child processes.
-
-- **User-controlled inputs:** Direct shell requests, model-produced tool arguments, scheduled-task prompts, and model-serving configuration.
-
-- **Current safeguards:** Agent dispatch applies owner/admin checks and tool policy; process helpers use timeouts or bounded background-job lifecycle where implemented.
-
-- **Confirmed risks or gaps:** Intentional authority with a confirmed gap: the agent shell starts in its workspace but is not sandboxed to it, and has no egress sandbox. This is documented in source and the threat model; it is not a newly demonstrated bypass.
-
-- **Unverified behaviour:** Role-gate and disabled-tool outcomes, direct shell-route behaviour, and timeout, cancellation, and output handling for foreground and detached processes remain unverified.
-
-## Filesystem access and workspace confinement
-
-- **Boundary:** Agent read, write, patch, listing, glob, and grep tools.
-
-- **Available authority:** Read and modify files within active workspace confinement or fallback allowlisted roots.
-
-- **User-controlled inputs:** Tool paths, patches, file contents, search patterns, and workspace selection passed into the tool dispatcher.
-
-- **Current safeguards:** [`src/tool_execution.py`](../src/tool_execution.py) resolves paths, blocks sensitive subpaths, applies allowlist containment, and tightens paths to the active workspace when one is bound. File tools use those resolvers.
-
-- **Confirmed risks or gaps:** Intentional authority with safeguards. The file-tool policy does not sandbox the shell; treating a workspace as a whole-process containment boundary would be incorrect.
-
-- **Unverified behaviour:** Traversal, symlink, sensitive-name, absolute-path, and workspace-switch behaviour remains unverified.
-
-## Agent-controlled tool dispatch
-
-- **Boundary:** Model output becomes native or parsed tool calls and is dispatched by the agent loop.
-
-- **Available authority:** The authority of every enabled tool, including privileged built-ins and external tools.
-
-- **User-controlled inputs:** Chat content, attached/retrieved content that may influence the model, tool arguments, per-request tool selection, and policy toggles.
-
-- **Current safeguards:** [`src/tool_security.py`](../src/tool_security.py) blocks protected tools for non-admin users and fails closed for malformed tool names; [`src/tool_policy.py`](../src/tool_policy.py) supports disabled and guide-only policy; prompt-security helpers label untrusted context.
-
-- **Confirmed risks or gaps:** Credible risk requiring verification: aliases, legacy text tools, native function calls, and MCP-qualified names must all reach the same policy outcome. The code has specific alias handling for email/MCP names, which makes this a sensitive compatibility seam.
-
-- **Unverified behaviour:** The current policy outcomes for owner role, request mode, disabled state, native versus parsed invocation, qualified aliases, and external-content entry points remain unverified.
-
-## MCP and external tool servers
-
-- **Boundary:** Configured MCP servers and their tools are exposed to the agent through the MCP manager and routes.
-
-- **Available authority:** Depends on the server: external network access, local process access, messaging, or data mutation may be delegated outside the application.
-
-- **User-controlled inputs:** Server configuration, remote OAuth completion, tool arguments, and model-selected MCP calls.
-
-- **Current safeguards:** MCP routes are registered through [`routes/mcp_routes.py`](../routes/mcp_routes.py); MCP-qualified tools are denied to non-admin users by [`src/tool_security.py`](../src/tool_security.py). OAuth state and token persistence are handled in [`src/mcp_oauth.py`](../src/mcp_oauth.py).
-
-- **Confirmed risks or gaps:** Credible risk requiring verification: an MCP server authority is broader than the application can infer from its tool name. This map does not establish a trust or approval model for server installation and individual tool invocation.
-
-- **Unverified behaviour:** Server onboarding, credential storage, server-origin trust, OAuth callback deployment, tool disablement, and invocation audit behaviour remain unverified.
-
-## Outbound network requests and URL validation
-
-- **Boundary:** Search/content fetch, research, webhooks, skill import, provider endpoints, and other HTTP clients.
-
-- **Available authority:** The application can make outbound requests from its network position.
-
-- **User-controlled inputs:** Search/fetch URLs, imported skill URLs, webhook configuration, and some endpoint settings.
-
-- **Current safeguards:** [`src/url_security.py`](../src/url_security.py) validates untrusted public HTTP URLs and fails closed on unsuitable schemes or private addresses. [`services/search/content.py`](../services/search/content.py) resolves and rejects non-public hosts, pins resolved addresses for fetches, caps bodies, and limits redirects.
-
-- **Confirmed risks or gaps:** Intentional split: administrator-created model endpoints may target private providers, while untrusted URLs use public-address checks. That distinction is required for self-hosted deployments but needs explicit call-site review.
-
-- **Unverified behaviour:** The URL-source classification for outbound clients and the current handling of redirects and DNS changes remain unverified.
-
-## Secrets, credentials, and vault sessions
-
-- **Boundary:** Application-managed encrypted secrets, API keys, provider credentials, and Bitwarden/Vaultwarden CLI sessions.
-
-- **Available authority:** Credentials unlock remote providers and connected personal services.
-
-- **User-controlled inputs:** Administrative configuration, login/unlock requests, imported settings, and agent vault tool arguments.
-
-- **Current safeguards:** [`src/secret_storage.py`](../src/secret_storage.py) uses a locally stored Fernet key with restrictive permissions for supported database secrets. Vault routes require an administrator, avoid passing master passwords in command arguments, and set restrictive permissions on the vault-session file.
-
-- **Confirmed risks or gaps:** Confirmed current boundary: vault session data is persisted through the vault path, not through [`src/secret_storage.py`](../src/secret_storage.py). This is an unresolved question about current security semantics, not a confirmed exposure.
-
-- **Unverified behaviour:** Current encryption-at-rest, owner scope, rotation, lock/logout, backup/restore, and log/tool-result exposure behaviour remains unverified.
-
-## Authentication and privileged administration
-
-- **Boundary:** Session authentication, API tokens, privileged routes, and internal tool loopback.
-
-- **Available authority:** Administrative identity can access execution, settings, integrations, data deletion, and secrets.
-
-- **User-controlled inputs:** Login/signup data, session cookies, API tokens, authentication configuration, and requests to privileged routes.
-
-- **Current safeguards:** [`core/auth.py`](../core/auth.py), [`core/middleware.py`](../core/middleware.py), and route-level checks establish identity and administrator gates. [`app.py`](../app.py) warns when localhost bypass is configured; [`SECURITY.md`](../SECURITY.md) documents deployment requirements.
-
-- **Confirmed risks or gaps:** Intentional authority with safeguards. Security depends on deployments keeping authentication enabled and internal services private; this map does not audit reverse-proxy or environment configuration.
-
-- **Unverified behaviour:** Setup, anonymous, non-admin, admin, token, and internal-loopback behaviour, including privileged-route gate consistency, remains unverified.
-
-## Deletion, wipe, backup, and restore
-
-- **Boundary:** Administrative wipe, cleanup, backup import/export, and the backup restore command.
-
-- **Available authority:** Delete or replace user data and credentials.
-
-- **User-controlled inputs:** Administrative HTTP requests, cleanup choices, backup payloads, archive paths, and restore command options.
-
-- **Current safeguards:** Administrative wipe routes use the administrative boundary. Cleanup exposes a preview route before mutation. The documented backup tool requires explicit restore confirmation, stages the old data directory, and validates archive members before extraction.
-
-- **Confirmed risks or gaps:** Intentional destructive authority. Backup archives contain secrets by design, as documented in [`docs/backup-restore.md`](../docs/backup-restore.md); this is an operator confidentiality responsibility, not a code defect established here.
-
-- **Unverified behaviour:** Role-gate, confirmation, archive-rejection, staged-recovery, and owner-isolation behaviour remains unverified. No destructive runtime test was performed.
-
-## Background jobs and unattended task execution
-
-- **Boundary:** Scheduled tasks, background-job monitor, startup tasks, and notification/delivery work that continue without an active browser request.
-
-- **Available authority:** Scheduled agent work can obtain model access and, for eligible owners, shell and file tools; task output can interact with connected services.
-
-- **User-controlled inputs:** Stored task prompt, schedule, model/crew selection, enabled-tool configuration, output target, and prior persisted state.
-
-- **Current safeguards:** [`src/task_scheduler.py`](../src/task_scheduler.py) serializes execution, records task runs, associates work with an owner, and applies the agent owner-based tool gate. [`src/bg_jobs.py`](../src/bg_jobs.py) keeps bounded state and can terminate overlong subprocess jobs.
-
-- **Confirmed risks or gaps:** Credible risk requiring verification: authority is inherited and exercised later, so changes to roles, task configuration, and disabled tools must be checked at execution time rather than assumed from task creation.
-
-- **Unverified behaviour:** Creation, editing, role-change, scheduling, cancellation, restart-recovery, and execution behaviour remains unverified, including whether current policy is re-evaluated before privileged action.

discovery/system-map.md

@@ -1,139 +0,0 @@
-# Current system map
-
-> [!NOTE]
-> This non-canonical discovery map is an evidence guide, not an exhaustive feature catalog or runtime certification. Verify the cited source before relying on a finding. Each section records local implementation observations, evidence locations, confirmed current problems, and unresolved factual questions.
-
-## Navigate the system
-
-- [Startup and application composition](#startup-and-application-composition)
-- [Frontend shell and browser interaction](#frontend-shell-and-browser-interaction)
-- [Chat, sessions, and streaming](#chat-sessions-and-streaming)
-- [Agents, tools, and execution](#agents-tools-and-execution)
-- [Models, providers, and local serving](#models-providers-and-local-serving)
-- [Search and research](#search-and-research)
-- [Documents, retrieval, and personal knowledge](#documents-retrieval-and-personal-knowledge)
-- [Memory and skills](#memory-and-skills)
-- [Email, calendar, contacts, notes, and tasks](#email-calendar-contacts-notes-and-tasks)
-- [Media, speech, and image work](#media-speech-and-image-work)
-- [Authentication, secrets, and privileged administration](#authentication-secrets-and-privileged-administration)
-- [Persistence, background work, and operations](#persistence-background-work-and-operations)
-
-## Startup and application composition
-
-- **How it works:** [`app.py`](../app.py) creates the application, mounts static assets, constructs shared services, registers route factories, and owns lifespan startup and shutdown. [`src/app_initializer.py`](../src/app_initializer.py) prepares application state; [`core/`](../core/) provides persistence, authentication, middleware, sessions, and platform helpers.
-
-- **Evidence locations:** [`app.py`](../app.py); [`src/app_initializer.py`](../src/app_initializer.py); [`core/database.py`](../core/database.py); [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/`](../routes/).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** Which component currently owns startup and shutdown for each long-lived service?
-
-## Frontend shell and browser interaction
-
-- **How it works:** [`static/index.html`](../static/index.html) is served by the root and SPA deep-link routes in [`app.py`](../app.py); [`static/app.js`](../static/app.js), [`static/style.css`](../static/style.css), and [`static/js/`](../static/js/) implement the client surface.
-
-- **Evidence locations:** [`static/index.html`](../static/index.html); [`static/app.js`](../static/app.js); [`static/js/`](../static/js/); [`static/style.css`](../static/style.css); [`app.py`](../app.py) deep-link handlers.
-
-- **Known problems:** The `/backgrounds` route in [`app.py`](../app.py) calls `serve_html_with_nonce` for `static/backgrounds.html`, but that file is absent from [`static/`](../static/). This is a confirmed broken prototype route, not evidence about the rest of the frontend.
-
-- **Open question:** Is `/backgrounds` currently an intentionally supported route or an obsolete prototype?
-
-## Chat, sessions, and streaming
-
-- **How it works:** [`routes/chat_routes.py`](../routes/chat_routes.py) and [`routes/chat_helpers.py`](../routes/chat_helpers.py) coordinate requests, session state, and SSE delivery. [`src/chat_handler.py`](../src/chat_handler.py), [`src/chat_processor.py`](../src/chat_processor.py), [`src/llm_core.py`](../src/llm_core.py), and [`src/session_actions.py`](../src/session_actions.py) provide message preparation, provider interaction, and session operations.
-
-- **Evidence locations:** [`routes/chat_routes.py`](../routes/chat_routes.py); [`routes/chat_helpers.py`](../routes/chat_helpers.py); [`routes/session_routes.py`](../routes/session_routes.py); [`src/chat_handler.py`](../src/chat_handler.py); [`src/chat_processor.py`](../src/chat_processor.py); [`src/llm_core.py`](../src/llm_core.py); [`core/session_manager.py`](../core/session_manager.py).
-
-- **Known problems:** [`src/agent_loop.py`](../src/agent_loop.py) annotates `_resolved_tool_event_name` with `Any` but imports no `Any` and does not enable postponed annotation evaluation. Python evaluates that annotation while importing the module, so this is an import-time defect at the checked baseline.
-
-- **Open question:** No end-to-end provider or browser streaming run was performed for this map.
-
-## Agents, tools, and execution
-
-- **How it works:** [`src/agent_loop.py`](../src/agent_loop.py) drives multi-round tool use. [`src/tool_execution.py`](../src/tool_execution.py) dispatches calls and binds workspace context. [`src/agent_tools/`](../src/agent_tools/) contains individual implementations; [`src/tool_security.py`](../src/tool_security.py) and [`src/tool_policy.py`](../src/tool_policy.py) apply role and request policies. Long-running command work is represented by [`src/bg_jobs.py`](../src/bg_jobs.py).
-
-- **Evidence locations:** [`src/agent_loop.py`](../src/agent_loop.py); [`src/tool_execution.py`](../src/tool_execution.py); [`src/agent_tools/`](../src/agent_tools/); [`src/tool_security.py`](../src/tool_security.py); [`src/tool_policy.py`](../src/tool_policy.py); [`src/tool_schemas.py`](../src/tool_schemas.py); [`src/bg_jobs.py`](../src/bg_jobs.py).
-
-- **Known problems:** The import-time annotation defect above blocks the main agent/tool path. The shell is intentionally not a filesystem or network sandbox; that is an authority boundary, not by itself a vulnerability claim.
-
-- **Open question:** Which native, legacy, and MCP-qualified invocation paths reach each policy gate?
-
-## Models, providers, and local serving
-
-- **How it works:** Model routes delegate to discovery, capabilities, endpoint resolution, and LLM core modules. Cookbook routes and hardware-fit services handle model lifecycle and local-serving support.
-
-- **Evidence locations:** [`routes/model_routes.py`](../routes/model_routes.py); [`src/model_discovery.py`](../src/model_discovery.py); [`src/model_capabilities.py`](../src/model_capabilities.py); [`src/endpoint_resolver.py`](../src/endpoint_resolver.py); [`src/llm_core.py`](../src/llm_core.py); [`routes/cookbook_routes.py`](../routes/cookbook_routes.py); [`src/cookbook_serve_lifecycle.py`](../src/cookbook_serve_lifecycle.py); [`services/hwfit/`](../services/hwfit/).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** Which endpoint inputs are administrator-created and permitted to use private provider addresses?
-
-## Search and research
-
-- **How it works:** HTTP search routes use [`services/search/`](../services/search/); research is exposed through [`routes/research/`](../routes/research/) and implemented in [`services/research/`](../services/research/), [`src/deep_research.py`](../src/deep_research.py), and related helpers. [`src/search/`](../src/search/) remains an import-compatibility layer for callers not yet moved to `services.search`.
-
-- **Evidence locations:** [`routes/search_routes.py`](../routes/search_routes.py); [`services/search/`](../services/search/); [`routes/research/research_routes.py`](../routes/research/research_routes.py); [`services/research/`](../services/research/); [`src/deep_research.py`](../src/deep_research.py); [`src/search/`](../src/search/).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** No live provider request was made; provider configuration and network access remain unverified.
-
-## Documents, retrieval, and personal knowledge
-
-- **How it works:** Document routes coordinate upload handling, document processing, and editor actions. Personal-document and RAG modules use Chroma and embedding clients. PDF viewing uses the optional-dependency loader in [`src/pdf_runtime.py`](../src/pdf_runtime.py); form extraction and filling live separately in [`src/pdf_forms.py`](../src/pdf_forms.py) and [`src/pdf_form_doc.py`](../src/pdf_form_doc.py).
-
-- **Evidence locations:** [`routes/document_routes.py`](../routes/document_routes.py); [`src/upload_handler.py`](../src/upload_handler.py); [`src/document_processor.py`](../src/document_processor.py); [`src/document_actions.py`](../src/document_actions.py); [`src/personal_docs.py`](../src/personal_docs.py); [`src/rag_manager.py`](../src/rag_manager.py); [`src/embeddings.py`](../src/embeddings.py); [`src/pdf_runtime.py`](../src/pdf_runtime.py); [`src/pdf_forms.py`](../src/pdf_forms.py); [`src/pdf_form_doc.py`](../src/pdf_form_doc.py).
-
-- **Known problems:** PDF viewing/runtime loading and PDF form processing are separate implementations. That separation is confirmed and intentional in the source; it is not a defect without a reported behavioural failure.
-
-- **Open question:** Optional PDF dependencies and representative uploaded documents were not exercised.
-
-## Memory and skills
-
-- **How it works:** Memory routes use [`services/memory/`](../services/memory/) and vector helpers. Skills are exposed through [`routes/skills_routes.py`](../routes/skills_routes.py), stored and managed in [`services/memory/skills.py`](../services/memory/skills.py), and may be imported through [`services/memory/skill_importer.py`](../services/memory/skill_importer.py).
-
-- **Evidence locations:** [`routes/memory/memory_routes.py`](../routes/memory/memory_routes.py); [`services/memory/`](../services/memory/); [`src/memory.py`](../src/memory.py); [`src/memory_vector.py`](../src/memory_vector.py); [`routes/skills_routes.py`](../routes/skills_routes.py); [`services/memory/skills.py`](../services/memory/skills.py); [`services/memory/skill_importer.py`](../services/memory/skill_importer.py).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** Which imported skill content can reach execution-capable paths, and which validation occurs before that point?
-
-## Email, calendar, contacts, notes, and tasks
-
-- **How it works:** Dedicated route modules own email, CalDAV calendar, CardDAV contacts, notes, and tasks. Supporting modules include email helpers and pollers, CalDAV sync and writeback, and the task scheduler.
-
-- **Evidence locations:** [`routes/email_routes.py`](../routes/email_routes.py); [`routes/calendar_routes.py`](../routes/calendar_routes.py); [`routes/contacts/contacts_routes.py`](../routes/contacts/contacts_routes.py); [`routes/note/note_routes.py`](../routes/note/note_routes.py); [`routes/task_routes.py`](../routes/task_routes.py); [`routes/assistant_routes.py`](../routes/assistant_routes.py); [`src/caldav_sync.py`](../src/caldav_sync.py); [`src/caldav_writeback.py`](../src/caldav_writeback.py); [`src/task_scheduler.py`](../src/task_scheduler.py).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** External account behaviour, writeback, and delivery require controlled credentials and are not runtime-validated here.
-
-## Media, speech, and image work
-
-- **How it works:** Gallery and image routes coordinate media features. Service modules own speech and media integrations; [`src/generated_images.py`](../src/generated_images.py) and [`src/visual_report.py`](../src/visual_report.py) support artifact handling and presentation.
-
-- **Evidence locations:** [`routes/gallery/gallery_routes.py`](../routes/gallery/gallery_routes.py); [`routes/stt_routes.py`](../routes/stt_routes.py); [`routes/tts_routes.py`](../routes/tts_routes.py); [`src/generated_images.py`](../src/generated_images.py); [`services/stt/`](../services/stt/); [`services/tts/`](../services/tts/); [`services/faces/`](../services/faces/); [`src/visual_report.py`](../src/visual_report.py).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** Hardware- and provider-dependent media workflows were not exercised.
-
-## Authentication, secrets, and privileged administration
-
-- **How it works:** [`core/auth.py`](../core/auth.py) and [`core/middleware.py`](../core/middleware.py) provide identity and request gates. [`src/secret_storage.py`](../src/secret_storage.py) encrypts application-managed database secrets with a local Fernet key. Vault handling is separate: [`routes/vault_routes.py`](../routes/vault_routes.py) and [`src/tools/vault.py`](../src/tools/vault.py) invoke the Bitwarden CLI and persist its session data in the application data area.
-
-- **Evidence locations:** [`core/auth.py`](../core/auth.py); [`core/middleware.py`](../core/middleware.py); [`routes/auth_routes.py`](../routes/auth_routes.py); [`routes/api_token_routes.py`](../routes/api_token_routes.py); [`src/secret_storage.py`](../src/secret_storage.py); [`routes/vault_routes.py`](../routes/vault_routes.py); [`src/tools/vault.py`](../src/tools/vault.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py).
-
-- **Known problems:** Vault-command handling and local application secret storage are distinct paths with different storage mechanisms. This is a source-confirmed boundary, not evidence that either path is compromised.
-
-- **Open question:** What are the current confidentiality, ownership, rotation, and backup semantics for vault session data?
-
-## Persistence, background work, and operations
-
-- **How it works:** SQLite models and persistence are centred in [`core/database.py`](../core/database.py); managers use application data paths. The scheduler and background-job monitor can continue work outside a live browser request. Operational routes cover cleanup, backup, and administrative wipe; the repository also provides a backup script and user documentation.
-
-- **Evidence locations:** [`core/database.py`](../core/database.py); [`src/runtime_paths.py`](../src/runtime_paths.py); [`src/task_scheduler.py`](../src/task_scheduler.py); [`src/bg_jobs.py`](../src/bg_jobs.py); [`src/bg_monitor.py`](../src/bg_monitor.py); [`routes/backup_routes.py`](../routes/backup_routes.py); [`routes/cleanup/cleanup_routes.py`](../routes/cleanup/cleanup_routes.py); [`routes/admin_wipe/admin_wipe_routes.py`](../routes/admin_wipe/admin_wipe_routes.py); [`scripts/odysseus-backup`](../scripts/odysseus-backup); [`docs/backup-restore.md`](../docs/backup-restore.md).
-
-- **Known problems:** None recorded by this mapping.
-
-- **Open question:** What current behaviour applies to background execution, cancellation, retries, and authority inheritance?

docker-compose.gpu-amd.yml

@@ -14,7 +14,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -46,10 +46,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -58,6 +59,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -65,14 +71,27 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -128,12 +147,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.gpu-nvidia.yml

@@ -13,7 +13,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -45,10 +45,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -57,6 +58,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -64,14 +70,27 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -131,12 +150,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.yml

@@ -2,7 +2,7 @@ services:
   odysseus:
     build: .
     ports:
-      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
+      - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7011}:7000"
     volumes:
       - ${APP_DATA_DIR:-./data}:/app/data:z
       - ${APP_LOGS_DIR:-./logs}:/app/logs:z
@@ -34,10 +34,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -46,6 +47,11 @@ services:
       - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
       - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
       - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
+      - ODYSSEUS_UNATTENDED_MODE=${ODYSSEUS_UNATTENDED_MODE:-false}
+      - ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS=${ODYSSEUS_QWEN_NATIVE_COMPACT_BUILTINS:-1}
+      - ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT=${ODYSSEUS_QWEN_SUPPRESS_LOCAL_CONTEXT:-0}
+      - ODYSSEUS_CAPTURE_MODEL_REQUESTS=${ODYSSEUS_CAPTURE_MODEL_REQUESTS:-0}
+      - ODYSSEUS_MCP_EMAIL_OWNER=${ODYSSEUS_MCP_EMAIL_OWNER:-}
       - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
       - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
       - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
@@ -53,14 +59,27 @@ services:
       - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
       - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
+      - ODYSSEUS_EDITOR_DRAFT_MAX_BYTES=${ODYSSEUS_EDITOR_DRAFT_MAX_BYTES:-268435456}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
+      # Host workspace translation is opt-in. Keep the public compose file
+      # user-neutral; configure these in a local .env or use the host-workspace
+      # overlay with ODYSSEUS_HOST_WORKSPACE_DIR.
+      - ODYSSEUS_WORKSPACE_HOST_ROOT=${ODYSSEUS_WORKSPACE_HOST_ROOT:-}
+      - ODYSSEUS_WORKSPACE_CONTAINER_ROOT=${ODYSSEUS_WORKSPACE_CONTAINER_ROOT:-/workspace}
+      - ODYSSEUS_WORKSPACE_DEFAULT=${ODYSSEUS_WORKSPACE_DEFAULT:-}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
       - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
       - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
       - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -109,12 +128,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker/host-network.yml

@@ -0,0 +1,21 @@
+# High-trust host network access. Enable only when the Odysseus agent needs
+# host-native LAN/VPN/mDNS behavior that Docker bridge networking cannot
+# provide. Linux only; Docker Desktop does not provide equivalent host
+# networking semantics.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml:docker/host-network.yml
+#   APP_PORT=7011
+services:
+  odysseus:
+    network_mode: host
+    ports: !reset []
+    environment:
+      - APP_PORT=${APP_PORT:-7011}
+      - APP_BIND=${APP_BIND:-0.0.0.0}
+      - SEARXNG_INSTANCE=${ODYSSEUS_HOST_NETWORK_SEARXNG_INSTANCE:-http://127.0.0.1:8080}
+      - CHROMADB_HOST=${ODYSSEUS_HOST_NETWORK_CHROMADB_HOST:-127.0.0.1}
+      - CHROMADB_PORT=${ODYSSEUS_HOST_NETWORK_CHROMADB_PORT:-8100}
+      - ODYSSEUS_CONTAINER_NETWORK_MODE=host
+    command:
+      - sh
+      - -c
+      - exec uvicorn app:app --host "$${APP_BIND:-0.0.0.0}" --port "$${APP_PORT:-7011}"

docker/host-workspace.yml

@@ -0,0 +1,11 @@
+# High-trust host workspace access. Enable only when the Odysseus agent should
+# work on a host directory outside the container's normal /app/data sandbox.
+#   COMPOSE_FILE=docker-compose.yml:docker/host-workspace.yml
+#   ODYSSEUS_HOST_WORKSPACE_DIR=/absolute/host/path
+#   ODYSSEUS_HOST_WORKSPACE_MOUNT=/host/workspace
+services:
+  odysseus:
+    volumes:
+      - ${ODYSSEUS_HOST_WORKSPACE_DIR:?set ODYSSEUS_HOST_WORKSPACE_DIR}:${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}:rw,z
+    environment:
+      - ODYSSEUS_HOST_WORKSPACE_MOUNT=${ODYSSEUS_HOST_WORKSPACE_MOUNT:-/host/workspace}

docs/AGENT_TURN_CONTRACT.md

@@ -0,0 +1,75 @@
+# Agent turn contract
+
+Scope: product Agent turns on 7011. Environment-owned native/TUI bridges retain
+their existing execution contract. No model weights or training settings change.
+
+## Boundaries
+
+1. `src/turn_contract.py` classifies capabilities, including explicit compound
+   requests and referential follow-ups. Classification is selection, not permission.
+2. `routes/chat_routes.py` resolves toggles, privileges, global/plan/incognito
+   restrictions, fixture restrictions and available schema inventory before
+   freezing the offered set. Web enabled alone does not select web tools.
+3. `TurnContract` checks `required <= offered <= executable`, stores immutable
+   serialized schema copies, and records unavailable requirements. An unavailable
+   request stops without inference or substitution; unknown actions ask for clarity.
+   Exact account-discovery requests narrow selection to account metadata only;
+   compounds retain their declared family scope. Media operations declare their
+   existing tool dependencies rather than falling back to shell generation.
+4. The agent's prompt/schema route and fallback use that same logical scope.
+   Native versus textual serialization remains model-specific. Answer-only phases
+   can suppress tool calls without granting a different scope.
+   Contract turns preserve the already-compacted conversation and tool-call/result
+   IDs. The standalone specialist prompt's latest-message-only behavior is not used
+   for these product turns. Prompt domains also come from the contract.
+   Accepted in-scope calls retain their model-provided arguments and native IDs;
+   the explicit-intent fallback must not overwrite them with the whole user turn.
+5. The context-bound dispatcher checks membership **and** existing runtime policy,
+   owner restrictions and exact-action approvals. A contract is not authorization
+   to bypass those gates. Contract work bypasses terminating legacy shortcuts.
+6. `_AgentRenderState` explicitly identifies streamed versus canonical output.
+   Later synthesis transfers ownership with turn-scoped replacement. The frontend
+   reconciles visible DOM, not just accumulated strings; tool evidence is retained.
+   Ownership is included in saved metrics and `message_saved` events.
+   History and resume honor replacement scope. Single-capability turns retain
+   canonical output: an always-synthesize trial caused a live notes loop and was
+   reverted. Compound turns cannot terminate after only one capability's result.
+
+## Verification
+
+Use the project's configured Python environment, not an unrelated system Python:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python -m pytest -q \
+  tests/test_turn_contract.py tests/test_turn_contract_integration.py \
+  tests/test_agent_turn_contract_boundaries.py tests/test_turn_rendering_js.py \
+  tests/test_contract_prompt_conversation.py tests/test_product_turn_contract_route.py \
+  tests/test_contract_explicit_fallback.py \
+  tests/test_history_resume_rendering_js.py \
+  tests/test_chat_route_tool_policy.py tests/test_tool_policy.py \
+  tests/test_frontend_module_version_parity.py
+node scripts/verify_agent_turn_contract.mjs --max-turns 80 --total-ms 900000
+```
+
+The browser verifier uses `sft_alex_creator` and actual 7011 Agent controls. It
+captures request toggles, SSE contract/tool events, visible output and persisted
+history. Ten families have four initial/follow-up Web-toggle combinations.
+Blocked or unrun cases are not passes. Email requires verified fixture isolation;
+do not enable global fixture mode on the user's live service to make a test pass.
+
+## Remaining limits
+
+- Classification is deterministic and vocabulary-based, not a proof of semantic
+  understanding. Add independent behavior examples for confirmed misses.
+- Schema registration and policy permission do not guarantee a remote provider
+  stays healthy throughout a turn. Runtime failure must remain visible.
+- Separate tool/argument errors, tool-service failures, rendering failures and
+  verifier defects in reports. Do not infer model accuracy from routing alone.
+- Canonical summaries can still ignore presentation constraints such as a
+  requested item count. Do not count those as full functional passes. Forcing an
+  extra model round is not a validated general repair for this deployed model.
+- Keep all imports of a local JS module on the same URL identity. Distinct query
+  versions instantiate separate module state even when source files are identical.
+
+Live baseline and current matrix results are in `reports/agent-turn-contract-*`.
+The implementation is not a claim that every family has passed live verification.

docs/ARCHITECTURE.md

@@ -1,87 +0,0 @@
-# Architecture
-
-> [!NOTE]
-> This document is the proposed canonical destination for stable high-level
-> architecture facts. It remains subject to maintainer review. Source code,
-> tests, and configuration remain authoritative for implementation-sensitive
-> behaviour.
-
-## Purpose
-
-This document identifies the stable runtime boundaries and primary ownership
-locations used to navigate and extend Odysseus.
-
-It intentionally excludes generated metrics, file-size rankings, refactor
-priorities, unresolved investigation findings, and proposed package layouts.
-
-## Runtime structure
-
-| Area | Responsibility |
-|---|---|
-| `app.py` | FastAPI application composition and primary application entry point |
-| `launcher.py` | Application launch support |
-| `setup.py` | Native setup workflow |
-| `core/` | Authentication, middleware, persistence, sessions, and platform primitives |
-| `routes/` | HTTP and API route handlers |
-| `src/` | Application orchestration, tools, providers, and runtime helpers |
-| `services/` | Domain-oriented service implementations |
-| `mcp_servers/` | Built-in MCP server implementations |
-| `scripts/` | CLI tools, diagnostics, maintenance, and migration helpers |
-| `static/` | No-build browser frontend and bundled assets |
-| `tests/` | Automated tests and supporting test infrastructure |
-
-## Subsystem boundaries
-
-| Subsystem | Primary implementation locations |
-|---|---|
-| Application startup | `app.py`, `src/app_initializer.py`, `core/` |
-| Authentication and sessions | `core/auth.py`, `core/middleware.py`, `core/session_manager.py`, `routes/auth_routes.py` |
-| Chat and streaming | `routes/chat_routes.py`, `routes/chat_helpers.py`, `src/chat_handler.py`, `src/chat_processor.py`, `src/llm_core.py` |
-| Agents and tools | `src/agent_loop.py`, `src/tool_execution.py`, `src/agent_tools/`, `src/tools/`, `src/tool_policy.py`, `src/tool_security.py` |
-| Models and providers | `routes/model_routes.py`, `src/model_discovery.py`, `src/model_capabilities.py`, `src/endpoint_resolver.py`, `src/llm_core.py` |
-| Cookbook and hardware fit | `routes/cookbook_routes.py`, `routes/cookbook_helpers.py`, `src/cookbook_serve_lifecycle.py`, `services/hwfit/` |
-| Search and research | `routes/search_routes.py`, `services/search/`, `routes/research/`, `services/research/`, `src/deep_research.py` |
-| Documents and retrieval | `routes/document_routes.py`, `src/document_processor.py`, `src/personal_docs.py`, `src/rag_manager.py`, `src/pdf_runtime.py` |
-| Memory and skills | `routes/memory/`, `services/memory/`, `routes/skills_routes.py` |
-| Email | `routes/email_routes.py`, `routes/email_helpers.py`, `routes/email_pollers.py`, `mcp_servers/email_server.py` |
-| Calendar, contacts, notes, and tasks | `routes/calendar_routes.py`, `routes/contacts/`, `routes/note/`, `routes/task_routes.py`, `src/task_scheduler.py` |
-| Media and speech | `routes/gallery/`, `routes/stt_routes.py`, `routes/tts_routes.py`, `services/stt/`, `services/tts/` |
-| Persistence and operations | `core/database.py`, `src/runtime_paths.py`, `src/bg_jobs.py`, `routes/backup_routes.py`, `routes/cleanup/` |
-
-## Architectural constraints
-
-- Preserve established import and compatibility paths unless a focused change
-  explicitly migrates them.
-- Keep HTTP concerns in route modules and reusable domain behaviour in runtime
-  or service modules.
-- Treat shared persistence, agent orchestration, tool execution, and application
-  startup as high-authority boundaries.
-- Change one ownership boundary at a time.
-- Do not mix structural movement with unrelated feature behaviour.
-- Validate affected imports, startup paths, compatibility surfaces, and tests.
-
-## Frontend
-
-The browser frontend is a no-build ES-module application under `static/`.
-
-Its maintained module-level structure is documented in
-[`static/js/MODULE_SUMMARY.md`](../static/js/MODULE_SUMMARY.md).
-
-## Investigation and snapshots
-
-Non-canonical investigation material is maintained under [`discovery/`](../discovery/).
-
-The following documents may contain dated observations, metrics, unresolved
-questions, or historical planning and must not be treated as specifications:
-
-- [`discovery/system-map.md`](../discovery/system-map.md)
-- [`discovery/architecture-runtime-inventory.md`](../discovery/architecture-runtime-inventory.md)
-
-## Documentation authority
-
-- Code, tests, and configuration define implemented behaviour.
-- Mature specifications define accepted subsystem behaviour where they exist.
-- Following maintainer acceptance, this document will define the high-level
-  architecture map.
-- Discovery documents preserve evidence and uncertainty but remain
-  non-canonical.

docs/BACKGROUND_TOOL_JOBS.md

@@ -0,0 +1,55 @@
+# Background research → originating chat
+
+Chat `trigger_research` calls carry a **dispatcher-supplied** `origin_chat_id`.
+The research start route verifies chat ownership before registering a durable
+`background_tool_jobs` row and starting the existing research service. Panel
+jobs have no origin and never inject a chat reply.
+
+- Chat default: **2 rounds**, 120-second *soft* research budget. Explicit
+  deeper/Auto rounds regain the normal research time budget. Panel defaults
+  remain unchanged. This is not a guaranteed two-minute wall-clock deadline.
+- A completion callback stores the report and sources. A startup worker also
+  reconciles missed callbacks and research errors/restarts.
+- When the origin has no active foreground/detached run, its model summarizes
+  the report with thinking off and no tools. An outer 75-second deadline also
+  bounds model-slot waits. If synthesis is unavailable, deliver an honest
+  notice plus the report link; preserve the evidence for follow-ups.
+- Message and delivery marker commit in one transaction with a deterministic
+  message ID. Report context is stored in server message metadata and injected
+  as untrusted evidence in regular and compact model history. Long excerpts
+  are explicitly marked; the saved full research report remains accessible.
+- The browser polls owner-scoped `/api/research/chat-jobs/{chat_id}`, appending
+  unseen message IDs only when that chat is current and not streaming. No
+  transcript replacement or forced navigation. Reloaded history deduplicates.
+- Chat uses the existing agent-thread rail and expandable rows. The compact
+  header shows status and a right-aligned BG task label with the shared whirlpool
+  while running; expanding reveals topic, phase/round, source count and report
+  link. Rows update in place, preserving expansion/focus while chat streams.
+  Completed rows remain visible; zero-source runs show a warning, not success.
+  Progress polling excludes reports and internal fields.
+
+Other tools are **not automatically backgrounded**. The durable handoff can be
+reused, but each future producer needs explicit launch/result/permission wiring.
+
+## Verification
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_background_tool_jobs.py tests/test_research_chat_runtime.py
+node --test tests/backgroundToolJobs.test.mjs
+node scripts/verify_background_delivery_isolation.mjs
+node scripts/verify_background_research_cards.mjs
+node scripts/verify_background_research_chat.mjs
+```
+
+The last script uses disposable `sft_alex_creator` chats and real research/model
+calls, then removes only its own reports/chats. Do not use real-user mutations.
+It checks two-round launch, continued chat, automatic arrival, no transcript
+rebuild/duplicates, reload, and a follow-up. Inspect retained report excerpts
+and generated summary when it fails; do not equate job launch with good research.
+
+Initial live runs verified delivery/navigation/follow-ups but exposed a summary
+attempt-count bug (fixed: helper requires **1 attempt**, not `max_retries=0`).
+A later full run was interrupted by an inference endpoint outage. The corrected
+summary path separately passed a real-model evidence/limitations/citation probe.
+All targeted Python tests passed (441); real DOM isolation checks passed. A clean
+full live run with useful retrieved evidence remains to be recorded.

docs/CLEAN_LOOP_V3_EXPERIMENT_20260909.md

@@ -0,0 +1,99 @@
+# No-RAG clean loop: first diagnostic
+
+## Setup
+
+No live UI, service configuration, or weights changed. The standalone loop sends
+conversation history, native assistant calls and matching tool results directly
+to the served pre-Heretic model. It never rewrites queries, invents calls, swaps
+families, or strips output. Invalid calls return errors. Six executions per turn
+and seven model rounds bound the test.
+
+Both arms use temperature 0, thinking disabled, 768 output tokens, and the
+original tool-work evaluator's `tools_for_mode(..., 'compact_contract_v3')`.
+This matters: the app's plain compact scrubber deletes descriptions, whereas v3
+retains empirically tested micro-hints. Previous plain-compact tests were not
+exact reproductions of the passing benchmark setup.
+
+The 76 tools come from the current app's ten-family inventory, transformed by
+the original v3 builder. This is not a byte-identical frozen 99-tool benchmark
+inventory or proof of training-data identity. The report records schema and
+builder hashes. No schemas are invented for this experiment.
+
+- **Stable:** same compact inventory on every turn, irrespective of spelling.
+- **Routed:** same loop, but existing `requested_capabilities` chooses inventory
+  each turn. This isolates that selector; it is not the complete production RAG
+  or Agent UI path. Other production normalizers are absent in both arms.
+- Private records are synthetic. No real private dispatcher is imported.
+  Only fixture reads and optional public SearXNG calls execute. Other operations
+  return explicit errors, so this does not validate their functionality.
+- Live search sends the exact model query to local SearXNG Bing/Yep, bypassing
+  app query rewriting/filtering. Source results may vary between arms.
+
+## Observations, not a blind score
+
+| Case | Stable compact inventory | Selector arm |
+|---|---|---|
+| `whats the current stock mraket` | Selected `web_search`, query `current stock market` | Offered zero tools; declined live lookup |
+| Exact seeded failed exchange, then `can you look up` | Searched with corrected query | Also searched with corrected query |
+| Summarize search, explicitly no tools | Answered without tools or permission failure | Same |
+| Calendar → email → calendar | Recalled second event at 14:30 | Same |
+| Notes → second note → what does it say | Correct `view` ID and content | Same after fixture correction |
+| Deliberately irrelevant search result | Did not automatically retry | Did not automatically retry |
+| User asks for a better source | Refined and executed another search | Proposed search was not offered and was rejected |
+| Web-disabled lookup | Attempted network access via bash; sandbox rejected it | Invented unsupported current market news without tools |
+
+The initial stable stock answer listed sources, not current index values. It
+does not establish that the market question was fully answered. Its subsequent
+`can you look up` elicited clarification after it had already searched. The
+separate seeded replay removes that differing-history confound.
+
+The first notes fixture incorrectly accepted `get/read`, not the real `view`
+action. Both models selected the correct action, but the fixture rejected it.
+Those six original turns are invalid for execution comparison. A corrected
+six-turn rerun succeeded in both arms; the failed evidence is retained.
+
+Web-off results are a release blocker: removing named web tools alone does not
+enforce network denial across general-purpose tools. The fixture prevented real
+execution, but any UI integration must use the real cross-tool permissions and
+clearly communicate unavailable capabilities. Neither arm is ready for a live
+switch. Source recovery and grounded completion also remain weak.
+
+## What this changes
+
+There is direct evidence that the selector can withhold needed tools, and that
+the model can repair the misspelled query itself when offered the tool. Clean
+history also supports the tested topic switches without synthetic substitutions.
+This supports continuing the clean-path experiment, not retraining or declaring
+the UI fixed. Full inventory is slower in these requests; overlapping runs and
+different source content prevent a controlled latency conclusion.
+
+Next: integrate the clean loop behind a test-only UI profile with real permission
+enforcement and one renderer, preserving the v3 contract. Test live read-only
+follow-ups and explicit Web-off behavior before any rollout. Separately compare
+a generic evidence-check/retry instruction on the weak-result fixture; do not
+manufacture a retry query in the harness.
+
+## Reproduce
+
+Eight boundary tests pass:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_clean_tool_loop.py
+```
+
+Run with a fresh report filename (existing evidence is never overwritten):
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/python scripts/test_clean_tool_loop.py --live-search --report reports/clean-loop-v3-new-run.json
+```
+
+Evidence:
+
+- `reports/clean-loop-v3-20260909.json`: original 24 turns; notes fixture caveat above.
+- `reports/clean-loop-v3-stock-seeded-20260909.json`: four matched seeded follow-up turns.
+- `reports/clean-loop-v3-notes-fixture-corrected-20260909.json`: corrected six notes turns.
+
+Each report retains model requests, responses, offered inventory and execution
+results. The `completed` status means the request loop finished, **not** that
+the answer passed functional evaluation. These are synthetic/public traces, not
+private user conversations. This test does not measure UI rendering or streaming.

docs/CLEAN_V3_UI_PREVIEW.md

@@ -0,0 +1,220 @@
+# Tools v3 — No-RAG preview
+
+Select this endpoint in the 7011 model picker, with model
+`odysseus-qwen3.5-tools-pre-heretic`. This endpoint owns its complete tool loop
+and enters Agent mode server-side on every turn, including ambiguous follow-ups;
+it does not depend on the legacy per-message intent classifier. Start a new chat
+for an uncontaminated comparison. Enable Web for searches. Clean routing is
+owned by the exact model identity, so both the normal `preheret` endpoint and
+the `cleanv3` alias use this runtime. Every other model remains on legacy RAG.
+
+Endpoint ID: `cleanv3`. Its base URL uses the same inference server's Tailscale
+DNS name, `http://odysseus.tailb895f4.ts.net:18182/v1`, to distinguish it from
+the original IP-address route when existing chats omit endpoint IDs.
+
+## Implementation
+
+- `src/clean_agent_preview.py` is a separate streamed native-tool loop, entered
+  before legacy routing and substitutions. It uses real authenticated tool
+  dispatch, the tool-work `compact_contract_v5` builder, temperature 0,
+  and thinking disabled. No weights change or inference server was started.
+- The offered tool inventory is stable except for permissions/toggles. Safe,
+  explicit personal creates/updates are enabled for notes, tasks, calendar,
+  memory, skills and documents. Destructive operations, shell/code, outbound
+  email, browser interaction, deployment/admin changes and unrelated-family
+  write substitution remain blocked. No tool or argument substitution is
+  applied by the loop.
+- Native calls and matching results persist in `clean_v3_turn` metadata so
+  follow-ups use actual evidence. History retains at most eight complete turns,
+  trimming oldest whole turns for size; individual outputs cap at 8000 chars.
+- Real search still uses the existing search backend and its provider handling;
+  this does not claim that provider quality or every backend transform is fixed.
+- All routing, privileges and default settings outside this exact Odysseus model
+  remain unchanged. The loop has six execution/eight-round limits.
+- Write completion is evidence-bound: affirmative success text is replaced
+  unless a private-write tool succeeded during the turn. Proposed call batches
+  are policy-preflighted atomically, so a batch containing a blocked operation
+  cannot partially execute before denial.
+
+## Verification
+
+399 focused Python tests passed after route integration. Browser runs r1/r2
+accidentally exercised the old loop and are not preview evidence. The runner
+now explicitly asserts `selection_mode=clean_compact_v3_preview`.
+
+`reports/clean-v3-live-ui-r3-20260909.json` confirms the preview route, real notes
+execution, correct repetition from history, successful search and no-tool
+summary, plus visible incremental growth. Its notes assertions were for the
+old routed contract: they prohibited offering web tools even with Web enabled,
+and required another notes call for a verbatim repeat. The updated preview
+checks permit stable offers and accept an exact match to the preceding saved
+answer without re-execution; execution permissions are still asserted.
+
+`reports/clean-v3-live-ui-r4-20260909.json` is the corrected four-turn check,
+including notes with Web off and search with Web on: **4/4 passed**, with the
+preview selection mode explicitly confirmed on every turn.
+These are UI smoke tests, not all-family or factual-answer benchmark scores.
+
+## Disable
+
+Disabling only endpoint `cleanv3` removes the duplicate picker alias; it does
+not disable this model-owned runtime. To roll back the runtime, revert the exact
+model route in `routes/chat_routes.py`. Do not delete weights, adapters, or user
+chats. The v3 schema builder dependency is
+`/home/pewds/odysseus-tool-work/scripts/eval_alltools_unseen_compare.py` and its
+schema-dropout helper; preserve those with this deployment.
+
+## Expanded UI checks — 2026-09-09
+
+24 additional turns completed through the preview: 23 automated passes and one
+checker false alarm. The Cookbook follow-up correctly shortened the previous
+six-server result to the first three requested names without another call. The
+checker required either a fresh call or a verbatim repeat; manual inspection
+confirmed the requested subset. Raw failure evidence is retained, not rescored.
+
+Covered notes/misspellings/second-note selection, calendar/second-event time,
+tasks, documents, memory, skills, Cookbook listing, misspelled search, and Web
+toggle changes. Cross-family flows passed: Germany news → “whats my notes”,
+notes → “seach current stock mraket news”, and calendar → “now show my noes”.
+The model chose `current stock market news` itself. Every completed turn's
+audit confirmed the preview mode. Search source factual accuracy is not graded
+by this suite, and successful reads do not establish mutation coverage.
+
+Email was separately attempted but the test guard stopped it because the
+stable offered inventory exceeded its metadata-only verified scope. Email
+therefore remains unverified in this expanded run; the guard was not weakened.
+No production code, service settings or weights changed during these tests.
+
+Evidence under `reports/`:
+
+- `clean-v3-broader-ui-20260909.json`: 16 turns, 15 automatic passes, Cookbook caveat.
+- `clean-v3-topic-switch-ui-20260909.json`: 6/6 passed.
+- `clean-v3-second-note-ui-20260909.json`: 2/2 passed.
+- `clean-v3-email-notes-ui-20260909.json`: blocked email attempt; notes not run in that file.
+
+## Picker route fix
+
+The previous tests selected sessions through the API, missing a real picker
+bug: local entries were deduplicated by model ID, hiding alternative endpoints
+with the same weights. The picker now uses endpoint+model identity for local
+routes too, displays the endpoint name, and scopes its last-picked send override
+to the current chat. `/api/sessions` returns owner-filtered endpoint identity
+for unambiguous saved URLs, so reload labels do not depend on loading the model
+catalog. Ambiguous identical URLs are not guessed.
+
+The user-authorized chat `ec0683a2-015f-41d7-aa1f-34135c9640cb` was switched to
+`cleanv3` using the authenticated session PATCH API; no messages were inserted
+and no tool actions ran in that chat. Defaults and other chats were unchanged.
+
+The runner's `--picker-route true` starts on the original route, clicks the
+preview in the real picker, sends a greeting, reloads the chat permalink, then
+asks for notes. Early picker/reload reports are incomplete, not passes: their
+label check exposed the unloaded-catalog issue. Focused route/picker/history
+tests: 16 passed.
+
+Final picker test: `reports/clean-v3-picker-reload-r5-20260909.json`, **2/2
+passed**. Real picker click, greeting, permalink reload, and notes follow-up
+all confirmed the preview route. The label survived reload. R4 retained a
+history/DOM mismatch from sending before restored history was ready; the final
+driver explicitly waits for the saved first answer to render before sending.
+This does not claim a general fix for sending during unfinished history loading.
+
+## Native image/VL status
+
+The inference launcher previously set `--limit-mm-per-prompt` to zero images,
+so vLLM rejected attachments before the model saw them. The durable Odysseus
+launcher now permits up to three images per prompt; video remains disabled.
+
+`reports/clean-v3-vl-live-r4-20260909.json` proves the real 7011 attachment
+path, clean compact route, object/color/spatial recognition, permalink reload,
+and ambiguous image follow-up. Those checks pass. Exact OCR of the deterministic
+`ODYSSEUS 42` heading fails in both the untouched Qwen 3.5 9B base and the
+fine-tune, so it remains a base/runtime capability limitation rather than a
+fine-tune regression or harness failure.
+
+The same native path also passes JPEG and lossless WebP transport, object
+recognition, reload, and follow-up grounding. Evidence:
+`reports/clean-v3-vl-jpeg-r1-20260909.json` and
+`reports/clean-v3-vl-webp-r1-20260909.json`. Both remain `partial` only because
+the shared OCR check fails.
+
+## Reversible write check
+
+`scripts/verify_clean_v3_write.mjs` runs against only `sft_alex_creator`. It
+creates one UUID-named note through the real 7011 UI, verifies that exact row,
+requests a destructive bulk deletion, verifies the row still exists, and then
+deletes only its own test row through the authenticated API. The cleanup is
+verified by a 404 lookup.
+
+Final evidence: `reports/clean-v3-write-ui-r8-20260909.json`, **passed**. Both
+turns reported `selection_mode=clean_compact_v3_preview`; creation executed via
+`manage_notes(action=add)`, the destructive action did not execute, and the
+canonical response was “No changes were made.” The earlier r3/r5 files are
+startup/placement failures, while r4/r6/r7 retained genuine intermediate
+harness and verifier failures; none should be interpreted as passes.
+
+## Stateful, search, and email checks
+
+The reversible stateful runner passes all six mutation families in one run:
+calendar, notes, tasks, documents, memory, and skills (**6/6**). Each flow
+creates a UUID-only artifact through the real Agent UI, verifies it by
+owner-scoped API, applies a noun-free correction, verifies persistence, and
+removes only that artifact. A direct database audit found zero active synthetic
+calendar, note, task, or document rows afterward.
+
+The document failure was harness-owned. Compact description dropout left a
+vague free-form `command` field, error envelopes defaulted to exit code 0, and
+the clean loop dropped the active document ID. Compact v5 now exposes only
+required structured `edits`, reports errors truthfully, and executes against
+the request's explicit active document. Fresh document and combined stateful
+runs pass.
+
+Search Web-toggle combinations `00`, `01`, `10`, and `11` pass **8/8** across
+two turns. A web question can no longer silently enable Bash because it says
+“official source”, and an unavailable Web capability exposes no unrelated
+fallback family. The quality suite passes **3/3**: evidence reuse without a
+second call, explicit official-page inspection with `web_fetch`, correction of
+“stock mraket” in actual search arguments, and a truthful unsupported result
+for a synthetic company.
+
+Production-path email reads pass **3/3** through the running email MCP: account
+list, latest inbox list, and referential read of the first result. The report
+retains no account names, addresses, subjects, bodies, prompts, or answers.
+
+Post-fix representative direct/follow-up coverage also passes for every family:
+notes/calendar 4/4, tasks/documents/memory/skills/Cookbook/search/shell 14/14,
+and email 3/3 in its privacy-preserving runner. The combined legacy verifier's
+metadata-only email guard correctly refused its broader stable inventory; that
+stopped report is not counted as a model failure.
+
+Evidence:
+
+- `reports/clean-v3-stateful-all-r3-20260909.json`
+- `reports/clean-v3-stateful-documents-r2-20260909.json`
+- `reports/clean-v3-search-toggle-final-r6-20260909.json`
+- `reports/clean-v3-search-quality-r3-20260909.json`
+- `reports/clean-v3-email-read-r1-20260909.json`
+- `reports/clean-v3-ten-family-tail-postfix-r1-20260909.json`
+
+These checks verify routing, execution, persistence, follow-up, and selected
+answer-quality invariants. They are not yet the sealed all-action ship score.
+
+## Compact v5 and corrected contract evidence
+
+Compact v5 keeps the compact-v3 surface and adds only development-positive
+field hints for Email, Search/Hugging Face quant selection, and Shell/files.
+A Calendar date hint regressed development and was excluded. The Python tool now
+emits one final bare expression, REPL-style, without duplicating explicit
+`print(...)`; this turns otherwise correct computation calls into visible tool
+evidence for all models.
+
+Under frozen scorer `odysseus.contract.v2.5`, development is 327/344 raw
+(95.06%) and 327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%)
+and 311/336 scorable (92.56%), with zero reasoning leakage. Calendar, Shell,
+and Tasks remain below the 90% family ship floor, so the model is not yet a
+full benchmark ship candidate.
+
+Fresh post-deploy real-UI evidence passes: stateful flows 6/6, Email 3/3,
+Search quality/recovery 3/3, private browser 3/3, and VL workflow 3/3. The
+Search check accepts a failed attempt only when a later tool succeeds and the
+final answer remains grounded.

docs/REGULAR_MODEL_TOOL_COMPATIBILITY.md

@@ -0,0 +1,125 @@
+# Regular-model tool compatibility
+
+Last verified: 2026-09-09 through the authenticated 7011 Agent UI as
+`sft_alex_creator`.
+
+This is the legacy-RAG track. The exact model
+`odysseus-qwen3.5-tools-pre-heretic` is excluded and remains on its model-owned
+clean compact runtime.
+
+## Current baseline
+
+| Endpoint | Model | Ten-family result | State |
+|---|---|---:|---|
+| DeepSeek | `deepseek-v4-flash` | 10/10 | passed |
+| DeepSeek | `deepseek-v4-pro` | 10/10 | passed |
+| OpenAI | `gpt-5.5` | 10/10 | passed |
+| OpenAI | `gpt-5.6-sol` | 10/10 | passed |
+| OpenAI | `gpt-5.6-terra` | 10/10 | passed |
+| OpenAI | `gpt-5.6-luna` | 10/10 | passed |
+| OpenRouter | `moonshotai/kimi-k3` | 10/10 | passed |
+| OpenRouter | `x-ai/grok-4.5` | 10/10 | passed |
+| OpenRouter | `qwen/qwen3-vl-235b-a22b-instruct` | 10/10 | passed |
+| OpenRouter | `openai/gpt-5-image` | n/a | image generation; chat tools unsupported |
+| Local `100.69.120.65:8062` | `Qwen/Qwen3.5-9B` | not run | endpoint unavailable |
+| Local `100.69.120.65:8062` | `GLM-5.3-Flash-Alis-MLX-4bit` | not run | endpoint unavailable |
+
+The ten-family baseline covers one read-only functional turn each for notes,
+calendar, email accounts, tasks, documents, memory, skills, Cookbook/admin,
+web search, and shell. It verifies the legacy route, expected native tool call,
+execution result, visible UI answer, and absence of reasoning leakage. It is not
+yet a claim that every mutation/action variant, typo, or follow-up passes.
+
+## Typo and follow-up profile
+
+The stricter real-UI profile sends one misspelled read-only request to every
+family, followed immediately by a noun-free reference to the returned result.
+Read-only follow-ups must not call any tool; search follow-ups may either use
+the existing evidence or fetch the prior link. Across the nine chat-capable API
+models, the composited post-repair result is **178/180 turns (98.89%)**:
+
+| Model | Conversation result |
+|---|---:|
+| `deepseek-v4-flash` | 20/20 |
+| `deepseek-v4-pro` | 20/20 |
+| `gpt-5.5` | 20/20 |
+| `gpt-5.6-sol` | 20/20 |
+| `gpt-5.6-terra` | 20/20 |
+| `gpt-5.6-luna` | 18/20 |
+| `moonshotai/kimi-k3` | 20/20 |
+| `x-ai/grok-4.5` | 20/20 |
+| `qwen/qwen3-vl-235b-a22b-instruct` | 20/20 |
+
+Luna's only remaining family miss is a deliberately misspelled Shell request.
+The correct-spelling baseline passes. The harness does not auto-execute a shell
+command to hide that model-owned limitation.
+
+The shared repair recognizes a uniquely misspelled action verb and family noun,
+then seals only declared safe private reads with immutable canonical arguments.
+This repaired Tasks/Documents/Memory and adjacent read families across providers
+without widening mutation or Shell authority. A compact native-tool instruction
+also tells regular API models to map clear typos to a currently offered tool.
+
+Conversation evidence:
+
+- `reports/regular-model-conversation-flash-r3-20260909.json`
+- `reports/regular-model-conversation-remaining-r1-20260909.json`
+- `reports/regular-model-conversation-repair-r1-20260909.json`
+- `reports/regular-model-conversation-shell-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-repair-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-tail-r1-20260909.json`
+- `reports/regular-model-conversation-qwen-search-r1-20260909.json`
+
+Evidence:
+
+- `reports/regular-model-tools-provider-final-r4-20260909.json` — Flash, GPT-5.5, Kimi: 30/30.
+- `reports/regular-model-tools-repair-r3-20260909.json` — Pro and Sol: 20/20; retained Qwen pre-final 9/10 miss.
+- `reports/regular-qwen-vl-full-r4-20260909.json` — Qwen-VL final family-switch run: 10/10.
+- `reports/regular-model-tools-remaining-20260909.json` — Terra, Luna, Grok: 30/30; records unavailable/unsupported models and pre-repair failures.
+- `reports/regular-model-tools-postfix-r1-20260909.json` — post-hardening
+  rerun: nine chat-capable API models passed 90/90 family turns with zero model
+  failures. Its overall status is non-passing only because the two configured
+  local endpoints were offline; the image-only model remains unsupported.
+
+## Family switch and page inspection
+
+The six-turn switch/back flow covers notes → calendar → notes from prior
+evidence → web search → explicit `web_fetch` → calendar from prior evidence.
+All nine API models have a clean 6/6 reproduction (**54/54**). Kimi skipped
+search once in the retained first run and passed a fresh reproduction; that
+variability remains visible instead of being erased.
+
+Evidence:
+
+- `reports/regular-model-switchback-flash-r2-20260909.json`
+- `reports/regular-model-switchback-remaining-r1-20260909.json`
+- `reports/regular-model-switchback-kimi-r1-20260909.json`
+
+## Repair that produced the clean baseline
+
+Regular models no longer inherit up to three stale tool families into every
+explicit new request. Referential follow-ups still resolve from typed recent
+tool evidence, while explicit family switches receive the current family only.
+Safe required reads use `active_capabilities`, so stale offered context cannot
+disable their immutable operation. The stream layer also stops an exact long
+block repeated twice instead of waiting for a provider's full timeout.
+
+The composer no longer treats generic words such as “source”, “system”, “app”,
+or “review” as authority to silently enable Bash. Explicit shell, terminal,
+repository, code-file, and direct coding requests retain workspace
+auto-escalation. This is a shared UI authority fix, not a model-name exception.
+
+Run a bounded subset with:
+
+```sh
+MODELS='deepseek-v4-flash,gpt-5.5' \
+FAMILIES='notes,calendar' WORKERS=2 \
+REPORT_PATH=reports/regular-model-check.json \
+node scripts/verify_regular_model_tools.mjs
+```
+
+Set `PROFILE=conversation` to run the typo plus follow-up profile.
+
+The runner discovers only enabled pinned models (visible cached local models
+when no pins exist), retains no tool outputs or private rows, and deletes only
+the exact sessions it creates.

docs/SEARCH_HARNESS_AUDIT_20260909.md

@@ -0,0 +1,92 @@
+# Search and compact-tool experiment — 2026-09-09
+
+## Decision
+
+Keep the normal routed profile on 7011. The all-tools compact experiment is
+implemented but **disabled**: direct routing success did not translate into a
+working Agent UI. Do not retrain or promote a profile on these measurements.
+
+## Changes
+
+- Short public-web lookups on the target model have an execution budget: two
+  distinct token-normalized searches, one fetch, and up to three browser calls
+  after the two searches. This bounds attempts, not just recovery prose. Existing
+  permissions still apply; this does not make unavailable tools executable.
+- Failed/weak searches reach the model for evaluation and query refinement,
+  instead of the earlier unconditional terminal evidence veto. Some legacy
+  heuristics and official-site shortcuts remain; this is not a completed rewrite.
+- Search providers retain query/engine/date provenance. Unconfigured credentialed
+  fallbacks are skipped. When SearXNG is the sole configured usable provider, Yep
+  on the same instance is an additional fallback.
+- An unavailable warm-only family no longer vetoes an otherwise ordinary reply.
+- The all-tools experiment offers the trained compact inventory subject to
+  permissions. It requires the exact test-owner environment flag and exact model
+  match. The temporary service flag was removed after failed UI testing.
+
+## Evidence and limits
+
+| Measurement | Result | What it establishes |
+|---|---|---|
+| Focused Python regression suite | 401 passed | Covered policy, contract, provider and recovery-budget behavior |
+| Direct family-only compact schemas | 10/10 tool routing | Small public smoke test, not functional or blind accuracy |
+| Direct all-family compact schemas | 10/10 tool routing | Inventory did not break these first calls; roughly 3–4x slower in this run |
+| Full compact Agent UI, revision 3 | All eight turns failed one or more checks | Not suitable for activation; leaks, duplicate/incorrect rendering or missing expected calls |
+| Normal-profile final UI control | Five passed checks, two product failures, one capture error | Not accepted; suite status incomplete |
+| Clean synthetic notes tool-result continuation | Clean answer with both schema sizes | Model can continue correctly on that isolated input, not proof that UI failure is solely harness |
+
+Direct probes used temperature 0; the UI target-model sampling path can cap at
+0.2. Prompts, history and tool-result serialization also differ. Match those
+before attributing UI failures to weights versus harness. Existing UI checks are
+not a grounded factual-answer benchmark. Unit tests are not UI acceptance.
+
+Normal-profile control details: notes initial, both calendar turns, AI search
+initial, and history initial passed the automated checks. Notes follow-up hit a
+Playwright `Network.getResponseBody` capture error and is inconclusive. AI search
+follow-up explicitly requested a summary with no tools, but the contract still
+required `search_browser` and returned a permission failure. The history search
+follow-up failed the visible-leak check. These are separate from source relevance;
+the earlier warm-only fix did not cover classification as an active requirement.
+There is no matched pre-change control establishing a net improvement.
+
+Provider isolation bypassed app relevance filters. Bing general often returned
+broad or unrelated results despite the full query. Google/Mojeek returned no
+results, DDG hit CAPTCHA, and Presearch timed out. Yep returned useful PostgreSQL
+documentation, but was weak or empty for several other questions. Engine health
+and source quality remain unresolved. Fallback cannot help when an earlier weak
+result survives filtering; there is no claim of universal relevance here.
+
+## Reproduce and inspect
+
+- `scripts/audit_search_pipeline.py`: raw provider comparison, no model.
+- `scripts/compare_compact_tool_inventory.py`: read-only model schema comparison;
+  proposed calls are never executed.
+- `scripts/verify_agent_turn_contract.mjs`: real 7011 Agent UI and persisted-history
+  checks. Use the dedicated test account; reports can contain private tool data.
+- `reports/search-provider-isolation.json`, `reports/search-yep-isolation.json`:
+  public provider evidence.
+- `reports/compact-inventory-ablation.json`: direct routing probe.
+- `reports/full-compact-ui-audit-r3.json`: completed rejected UI experiment.
+  Earlier experiment reports include an initialization error and an aborted run;
+  do not combine them into an accuracy score.
+- `reports/routed-control-ui-audit-final.json`: normal-profile control replay;
+  eight attempts, incomplete because of the capture error; failures retained.
+
+Focused suite:
+
+```sh
+/home/pewds/odysseus-cookbook-fresh/.venv/bin/pytest -q tests/test_turn_contract.py tests/test_turn_contract_integration.py tests/test_service_search_provider_guards.py tests/test_web_recovery_budget.py tests/test_tool_policy.py
+```
+
+UI replay (read-only prompts, creates test chats):
+
+```sh
+node scripts/verify_agent_turn_contract.mjs --families notes,calendar,search_ai,search_history --pairs notes:11,calendar:11,search_ai:11,search_history:11 --max-turns 8 --total-ms 360000 --turn-ms 45000 --report reports/routed-control-ui-audit-final.json
+```
+
+## Next discriminating test
+
+Replay the same captured UI request directly, preserving sampling, compact
+schemas, history and tool results. Then change one layer at a time. Separately
+score retrieved-source relevance and supported answers. Replace failing generic
+boundaries only when the replay identifies them; do not add rules for individual
+user phrasings or treat successful tool routing as successful execution.

docs/STYLE.md

@@ -1,72 +0,0 @@
-# Documentation style
-
-This guide defines the shared structure and writing conventions for Odysseus documentation.
-
-## Principles
-
-- Write for a clear audience and purpose.
-- State whether a document is canonical, informational, a snapshot, or planning material.
-- Prefer current behaviour over historical explanation.
-- Link to source files, tests, issues, or other documentation when useful.
-- Separate verified behaviour from assumptions, open questions, and future work.
-- Keep headings descriptive and consistent.
-- Use Markdown callouts where status or risk must be visible.
-- Do not use emojis.
-
-## Document status
-
-Use a status callout near the top when the document is not normal canonical guidance.
-
-### Canonical documentation
-
-> [!IMPORTANT]
-> This document describes accepted current behaviour. Verify implementation-sensitive details against the current code and tests.
-
-### Discovery material
-
-> [!NOTE]
-> This is non-canonical discovery material. It records code-grounded observations and open questions.
-
-### Snapshot or inventory
-
-> [!WARNING]
-> This document is a dated snapshot. Counts, paths, and implementation details may drift as the codebase changes.
-
-### Planning material
-
-> [!NOTE]
-> This document records planning context. It does not define current runtime behaviour or guarantee future implementation.
-
-## Recommended structure
-
-Use the following sections where relevant:
-
-1. Title
-2. Purpose or status callout
-3. Scope
-4. Current behaviour or guidance
-5. Safety, limitations, or known gaps
-6. Validation or evidence
-7. Related documentation
-
-Not every document needs every section.
-
-## Writing style
-
-- Use concise sentences.
-- Prefer direct language.
-- Avoid jokes, filler, and informal warnings.
-- Avoid repeating the same guidance across several files.
-- Link to the owning document instead of duplicating large sections.
-- Use lists for procedures, requirements, and comparisons.
-- Use tables only when they improve scanning.
-- Use fenced code blocks with an appropriate language identifier.
-- Use relative repository links for internal files.
-
-## Authority
-
-The current code, tests, and configuration are the source of truth for implemented behaviour.
-
-Canonical documentation describes accepted behaviour and supported workflows.
-
-Discovery, inventory, and planning documents must identify themselves explicitly and must not silently become behavioural specifications.

docs/TYPO_ROUTING_AUDIT_20260909.md

@@ -0,0 +1,73 @@
+# Typo-tolerant tool routing audit
+
+The 9B SFT model was not retrained. This audit targets the earlier harness
+stage that decides which complete tool families the model is allowed to see.
+
+## Method
+
+- Source prompts: real `sft_alex_creator` sessions from `a37dcb3b-...` onward.
+- Labels: recorded single-family tool calls, excluding mixed/ambiguous traces.
+- Variants: deletion, adjacent transposition, duplicated character,
+  keyboard-neighbor substitution, and accidental word split.
+- Split: deterministic SHA-256 assignment before scoring (75% dev, 25% blind).
+- Safety: static routing only; no historical mutation or send action is replayed.
+- Acceptance: at least 95% blind exact-family accuracy and below 1% blind
+  wrong-family authorization. Abstention is measured separately.
+
+## Results
+
+| Router | Dev family supplied | Blind family supplied | Blind exact | Blind wrong-family |
+|---|---:|---:|---:|---:|
+| Previous exact rules | 63.64% | 65.69% | — | — |
+| Conservative fuzzy fallback r4 | 96.31% | 98.31% | 96.62% | 0.00% |
+| Final router + safe-read repair | 98.31% | 98.73% | 97.05% | 0.00% |
+
+The fallback runs only for action/lookup-shaped requests, resolves exactly one
+nearby family term, and abstains on ambiguity. Conceptual questions remain
+tool-free. Complete family schemas are still selected by the immutable turn
+contract; fuzzy matching never chooses an individual tool or its arguments.
+
+Authoritative machine reports:
+
+- `reports/typo-tool-routing-baseline-20260909.json`
+- `reports/typo-tool-routing-fuzzy-r4-20260909.json`
+- `reports/typo-tool-routing-final-20260909.json`
+- `reports/post-followup-agent-80-20260909.json`
+- `reports/post-typo-routing-agent-80-20260909.json`
+- `reports/live-typo-agent-20-20260909.json`
+- `reports/live-typo-unresolved-r3-20260909.json`
+- `reports/live-typo-agent-final-20-20260909.json`
+- `reports/post-typo-safe-read-agent-final-80-20260909.json`
+
+## Live 7011 findings
+
+The post-deployment standard matrix passed 80/80 through the real Agent UI.
+The first read-only typo matrix then attempted 17 of 20 planned turns before
+its total-time limit. Initial Notes, Calendar, Email, Tasks, Documents, and
+Cookbook calls passed. Completed failing turns still had the correct family
+and required tool in `turn_contract.offered`; the 9B model sometimes answered
+without calling that offered tool. Memory and Search also exposed timeouts.
+
+This separates three failure classes:
+
+1. **Tool injection:** addressed by conservative fuzzy family routing; blind
+   exact routing is 96.62% with zero blind wrong-family authorizations.
+2. **Required read execution:** a correctly offered safe list/refresh tool can
+   still be skipped by the model, especially after a typo or on “list those
+   again” follow-ups. This should be handled by the generic deterministic
+   safe-read path, not additional prompt-specific hints.
+3. **Runtime timeout:** Search and one Memory follow-up require loop/backend
+   diagnosis. A timeout is not counted as a model-accuracy or routing result.
+
+The generic safe-read parser and search-family precedence were then repaired.
+The previously unresolved Calendar, Email, Search, and Shell/Files cases passed
+8/8. The complete typo matrix passed 20/20, including initial requests and
+follow-ups for all ten families. The final standard Agent UI compatibility
+matrix passed 80/80 across family, Web-toggle, and follow-up combinations.
+
+The broad routing regression suite passed 458 tests. The model was not
+retrained and no DeepSeek API was used: the measured defect was in harness
+family selection and deterministic safe-read execution, upstream of the
+model. All 1,535 unique labeled historical turns were statically audited to
+mine failure categories. Historical write/send/delete actions were not replayed
+against live data; live verification used the deduplicated read-only matrices.

docs/skills-lifecycle.md

@@ -0,0 +1,26 @@
+# Skills lifecycle
+
+The UI exposes All, Built-in, Approved, and Draft. Draft includes archived
+records so they remain inspectable and recoverable. Built-ins are not audited.
+Approved means published, passing, at the configured confidence threshold,
+and not marked unnecessary. Baseline speed measurements remain evidence, not
+an additional hidden UI approval gate.
+
+Automatic audits process at most eight eligible records at a time, oldest first.
+New records are eligible immediately; inconclusive checks retry after a day;
+failed repairs retry after a week. Passed, duplicate-skipped, and archived records
+are excluded. Existing daily Skills Audit tasks drive this queue. Their quiet
+window deferrals propagate to the scheduler rather than becoming task failures.
+Automatic runs use background model scheduling. Existing self-repair and teacher
+repair stages remain in place; failed candidates remain drafts.
+
+The skill index advertises short descriptions; the agent loads a relevant full
+procedure on demand and applies already-injected procedures directly. Extraction
+prefers verified discoveries and specific workarounds over routine tool usage.
+
+Reference reviewed: NousResearch/hermes-agent, MIT license, commit
+cfdbbb6e35010ace89fbe8243ee82fa4de143e10, cloned to
+/home/pewds/hermes-skills-reference. In particular tools/skills_tool.py and
+agent/prompt_builder.py use progressive disclosure and task-triggered procedure
+loading. These changes adapt that approach to Odysseus's existing registry;
+no Hermes implementation code was copied.

launch-windows.ps1

@@ -163,6 +163,10 @@ if (Test-Path $cudaBase) {
 }
 
 # 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
+# -Port only reaches uvicorn as a flag. Everything that builds a URL for this
+# instance - internal_api_base(), companion pairing, the MCP OAuth callback -
+# reads APP_PORT, so set it too or they all assume 7000.
+$env:APP_PORT = $Port
 Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port)
 Write-Host "Press Ctrl+C to stop."
 Write-Host ""

launcher.py

@@ -130,7 +130,7 @@ if __name__ == "__main__":
     from app import app
 
     bind_host = os.getenv("APP_BIND", "127.0.0.1")
-    bind_port = int(os.getenv("APP_PORT", "7000"))
+    bind_port = int(os.getenv("APP_PORT", "7011"))
     url = f"http://{bind_host}:{bind_port}"
 
     if getattr(sys, 'frozen', False):

licenses/KaTeX-MIT-LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013-2020 Khan Academy and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/Mermaid-MIT-LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 - 2022 Knut Sveidqvist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mcp_servers/memory_server.py

@@ -17,6 +17,8 @@ from mcp.types import Tool, TextContent
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
+from src.memory import MemoryStoreUnreadable
+
 server = Server("memory")
 
 # Late-initialized managers (set during first tool call)
@@ -29,6 +31,10 @@ _OWNER_SCOPE_ERROR = (
     "Error: Memory MCP owner is not configured for an owner-scoped memory store. "
     "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
 )
+_UNREADABLE_STORE_ERROR = (
+    "Error: Memory store is temporarily unreadable — nothing was saved. "
+    "Repair or restore memory.json, then retry."
+)
 
 
 def _configured_owner() -> str | None:
@@ -51,9 +57,21 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
     return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
 
 
-def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
-    """Return configured owner, all entries, visible entries, and optional error."""
-    entries = _memory_manager.load_all()
+def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
+    """Return configured owner, all entries, visible entries, and optional error.
+
+    ``for_update=True`` is for read-modify-write callers. They save the ``all
+    entries`` list back, so an unreadable store must be reported as an error
+    instead of degrading to ``[]`` — otherwise the save writes their one new
+    entry over the whole store (issue #5673).
+    """
+    if for_update:
+        try:
+            entries = _memory_manager.load_all_for_update()
+        except MemoryStoreUnreadable as e:
+            return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
+    else:
+        entries = _memory_manager.load_all()
     owner = _configured_owner()
     if owner is None and _owner_scoped_store(entries):
         return None, entries, [], _OWNER_SCOPE_ERROR
@@ -161,7 +179,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
         category = arguments.get("category", "fact")
         if not text:
             return _text_result("Error: Memory text cannot be empty")
-        owner, memories, _visible, scope_error = _scope_entries()
+        owner, memories, _visible, scope_error = _scope_entries(for_update=True)
         if scope_error:
             return _text_result(scope_error)
         entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)

package-lock.json

@@ -4,19 +4,84 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "odysseus",
       "devDependencies": {
-        "@antithesishq/bombadil": "^0.6.1"
+        "@antithesishq/bombadil": "^0.7.0",
+        "@playwright/test": "^1.62.1"
       }
     },
     "node_modules/@antithesishq/bombadil": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
-      "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.7.0.tgz",
+      "integrity": "sha512-alJmnphJ/iUoL5mCsnV3DwtajGy/sEQ3NJJCiMhgjqXshSq2BUtAs0vqdXEiiSkB8HbsOX5CLrAcaogYdwfAJg==",
       "dev": true,
       "license": "MIT",
       "bin": {
         "bombadil": "bin/bombadil.js"
       }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
+      "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
+      "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.62.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.62.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
+      "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=20"
+      }
     }
   }
 }

package.json

@@ -1,9 +1,18 @@
 {
+  "name": "odysseus",
+  "private": true,
   "repository": {
     "type": "git",
     "url": "https://github.com/odysseus-dev/odysseus.git"
   },
+  "scripts": {
+    "test:photo-editor": "playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:install": "playwright install chromium firefox webkit",
+    "test:photo-editor:firefox": "PHOTO_EDITOR_E2E_BROWSER=firefox playwright test --config tests/e2e/playwright.config.js",
+    "test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js"
+  },
   "devDependencies": {
-    "@antithesishq/bombadil": "^0.6.1"
+    "@antithesishq/bombadil": "^0.7.0",
+    "@playwright/test": "^1.62.1"
   }
 }

plans/ODYSSEUS_TOOL_HARDENING_PLAN.md

@@ -0,0 +1,326 @@
+# Odysseus Tool Runtime Hardening Plan
+
+## Objective
+
+Ship `odysseus-qwen3.5-tools-pre-heretic` with one compact, model-specific tool
+runtime that supports realistic multi-turn use. Keep the existing RAG runtime
+unchanged for every other model. Prove routing, execution, answer quality,
+follow-ups, safety, rendering, latency, and native image/VL understanding through
+the real 7011 Agent UI.
+
+Current evidence is a baseline, not a ship claim:
+
+- Corrected v2.5 + compact-v5 development is 327/344 raw (95.06%) and
+  327/336 scorable (97.32%). Sealed blind is 311/344 raw (90.41%) and
+  311/336 scorable (92.56%), with zero reasoning leakage.
+- Notes, Skills, and Cookbook/admin clear 95% scorable blind. Calendar 87.5%,
+  Shell/files 86.11%, and Tasks 87.5% remain below the 90% family ship floor.
+- Compact-v5 hints improved Email, Search/HF quant, and Shell on development;
+  a Calendar hint regressed and was rejected rather than shipped.
+
+- Ten-family focused baseline: 19/20 functional and 20/20 routing/execution.
+- Typo and cross-family read flows: 26/26 passed.
+- Real use exposed untested write correction and search-to-fetch follow-ups.
+- Email production access, browser interaction, search quality, and broader
+  multi-turn mutations are not yet proven.
+- Nine enabled chat-capable regular API models pass the ten-family read-only
+  legacy-RAG baseline (90/90 combined). Their stricter typo/follow-up profile is
+  178/180 turns: eight models are 20/20 and Luna is 18/20 due only to its
+  misspelled Shell request. One pinned image-generation model is explicitly
+  unsupported and two visible local models are currently offline.
+- Native VL object/spatial recognition and reload follow-up pass. Exact OCR
+  fails equally on the fine-tune and untouched 9B base and remains unresolved.
+  PNG, JPEG, and WebP transport all pass.
+- Reversible create/correct/API-verify/cleanup flows pass 6/6 across every
+  stateful family.
+- Search Web-toggle combinations pass 8/8 and the focused quality suite passes
+  3/3. Production-path email account/inbox/referential reads pass 3/3.
+- The latest regular-model regression is 90/90 across the nine enabled
+  chat-capable API models, with zero failed model turns; two local endpoints
+  remain offline and the image-only model is unsupported.
+- The Epictetus OMLX endpoint was recovered after an unsupported
+  `qwen3_5_mtp` model load wedged the server. Its supported Qwen 27B 4-bit
+  model passes the ten-family real-7011 legacy-RAG smoke 10/10; the unsupported
+  MTP artifact is recorded as a runtime limitation rather than a timeout.
+- Fresh compact-v5 UI regressions pass stateful 6/6, Email 3/3, Search 3/3,
+  private-browser 3/3, and VL workflow 3/3.
+- The exact-model, family-scoped compact runtime now passes 20/20 direct and
+  same-family turns across all ten families on the real 7011 Agent UI. A
+  separate 36/36 robustness run passes misspellings, bounded repeats, browser
+  and news continuation, ambiguous follow-ups, family switchbacks, and a
+  greeting before a tool request.
+- The mobile active-email editor path passes 1/1: `Write reply this email`
+  offers and executes only `update_document`, mutates the open draft, and
+  preserves its reply headers and quoted thread.
+- The active-editor classifier now also covers short mobile wording without a
+  pronoun (`Write reply` / `Draft a reply`) while explicit note, code, file, and
+  new-object requests retain their own families. Whole-draft requests are bound
+  to the sole offered `update_document` writer until one successful write, then
+  tools are removed for the confirmation round. The deployed real-route email
+  regression passes 3/3—including the exact unspecified `Write reply to this
+  email` form—with one write, verified mutation, and preserved reply headers.
+  Clean-v3 now also emits the established `doc_update` event and flattened
+  document metadata on `tool_output`, so a successful database write updates
+  the already-open editor instead of leaving stale UI beside a success message.
+- The client now reuses the existing assistant bubble for `agent_step` round 1
+  instead of replacing it before the first token. A real-7011 sampled
+  greeting-to-Notes conversation passes 2/2 with stable first-round DOM
+  identity; round 2+ remains the only continuation-bubble path.
+- Clean-runtime metrics now expose provider-counted initial injected tokens,
+  all-round input/output, TTFT, tok/s, schema count, agent rounds, and tool-call
+  count. A real 7011 browser run passes 2/2 and visibly renders compact footers
+  plus the full details popup; the sampled Notes turns streamed progressively.
+- The deployed startup bottleneck was an unindexed quadratic transcript-FTS
+  reconciliation. Live-database import fell from about 36 seconds to 0.54
+  seconds; 7011 now answers in about 3 seconds after a controlled restart.
+- A controlled identical-compact comparison already proves the fine-tune's
+  accuracy benefit: 94.48% (325/344) versus the untouched base's 77.91%
+  (268/344). Raw serving speed is effectively tied, so product speed comes
+  from the compact contract and fewer failed/redundant rounds.
+- A fully merged 10,000-row category-repair candidate reached 97.32% scorable
+  development but only 92.26% scorable sealed blind. Calendar (87.5%), Tasks
+  (87.5%), and Shell/files (86.11%) remained below the family floor, so it was
+  rejected and not deployed. Compact-v4/full development A/Bs did not improve
+  Calendar or Tasks over compact-v5; full-schema Shell also fell from 97.22%
+  to 94.44%. This rules out compactness as the primary cause of the remaining
+  blind gaps and supports keeping the compact contract.
+
+## Non-negotiable architecture rules
+
+1. Runtime selection follows exact model identity. The trained Odysseus model
+   uses the clean compact runtime across endpoint aliases; all other models use
+   legacy RAG. Add a regression test for both sides.
+2. Resolve permissions, toggles, and available backends once per turn. Produce
+   one immutable contract satisfying `required ⊆ offered ⊆ executable`.
+3. Never offer a tool that the preview policy will categorically reject. Add a
+   contract self-check covering every offered action/effect combination.
+4. Follow-ups consume typed prior evidence: native call, result, success state,
+   family, and object identifiers. Do not infer continuity from keyword RAG.
+5. Contextual write authority may revise only a recently proven object in the
+   same family. It may not authorize a new object, another family, a destructive
+   action, or an external side effect.
+6. The model chooses tools and valid arguments. The harness validates and
+   executes; it does not silently substitute another family, rewrite arguments,
+   fabricate success, or replace a failed tool with prose claiming completion.
+7. One owner renders each turn: streamed prose or canonical structured output.
+   Never both, and never expose hidden prompts or raw untrusted wrappers.
+8. No exact-prompt production patches. A fix must name the failed layer, add a
+   generic failing invariant test, and cover neighboring cases.
+
+## Failure layers
+
+Every failure is assigned to exactly one primary layer before code changes:
+
+1. **Route:** wrong model runtime or endpoint identity.
+2. **Contract:** required tool absent, forbidden tool present, or toggle drift.
+3. **Model:** wrong/no tool or semantically wrong required arguments despite a
+   correct contract.
+4. **Policy:** valid proposed operation incorrectly allowed or denied.
+5. **Execution:** canonical arguments, backend dispatch, timeout, or result
+   envelope is wrong.
+6. **Evidence:** result is empty, irrelevant, truncated badly, or insufficient.
+7. **Answer:** model misstates or ignores valid tool evidence.
+8. **Rendering:** duplicate, dump-at-end, missing structured output, or stopped
+   stream.
+9. **Performance:** startup, TTFT, tool latency, or oversized context.
+
+Reports store aggregate category, relevant contract/tool metadata, timings, and
+sanitized outputs. Do not copy private hidden benchmark prompts or create a log
+dump that nobody can audit.
+
+## Test matrix
+
+Use the real authenticated 7011 Agent UI and the normal `preheret` picker alias.
+Use `sft_alex_creator` for reversible writes. Never mutate the personal account
+from an automated test.
+
+### A. Every one of the ten families
+
+For calendar, notes, email, tasks, documents, memory, skills, Cookbook/admin,
+search/browser, and shell/files, test:
+
+- direct request;
+- natural misspelling;
+- ambiguous same-family follow-up;
+- switch to another family and back;
+- no-tool greeting before the tool request;
+- requested count/field limit;
+- backend failure rendered truthfully;
+- reload the permalink before a follow-up.
+
+### B. Stateful mutation families
+
+For notes, calendar, tasks, documents, memory, and skills:
+
+- create → verify by API → referential correction → verify;
+- create → list/read → correction → verify;
+- typo correction such as name/date/title without repeating the family noun;
+- correction after one unrelated conversational turn;
+- destructive request is denied atomically;
+- failed write never produces a success claim;
+- cleanup deletes only the UUID-owned test artifact and verifies absence.
+
+### C. Search and browser conversations
+
+- search → summarize existing results without a new call;
+- search → inspect one result with `web_fetch`;
+- poor results → refine query once;
+- insufficient evidence → say so without fabrication;
+- Web toggle combinations `00`, `01`, `10`, and `11` across two turns;
+- private browser open/snapshot/click only after its permission boundary is
+  deliberately enabled and specified; do not smuggle it in via web search.
+
+Grade source relevance, freshness, authority, and whether claims are supported,
+not merely whether `web_search` was called.
+
+### D. Email and shell
+
+- Separate fixture accuracy from production connectivity. A fixture pass cannot
+  promote production email health.
+- Test account listing, inbox listing, reading, and referential follow-up against
+  the configured production-like backend before enabling email actions.
+- Shell remains toggle-gated. Test off/on transitions, canonical raw command
+  dispatch, read-only output, and denial of network/destructive commands.
+
+### E. Rendering and performance
+
+- Assert first visible streamed token, monotonic DOM growth, one final answer,
+  persistence/reload equality, stop behavior, and structured list rendering.
+- Record request preparation, TTFT, tool duration, post-tool TTFT, total time,
+  input/output tokens, and tool-result bytes.
+- Diagnose the 30–40 second 7011 restart separately from inference latency.
+- Bound large calendar/search results before replaying them into later rounds,
+  while preserving IDs and fields needed for follow-ups.
+
+### F. Image/VL recognition
+
+- Attach real PNG, JPEG, and WebP images through the 7011 UI and verify the
+  trained model receives native multimodal message content on its clean route.
+- Test object recognition, visible text/OCR, spatial relationships, charts, and
+  screenshots. Score required facts instead of stylistic wording.
+- Test image → ambiguous follow-up, image → tool request, and tool result → image
+  comparison without requiring the user to attach the same image again.
+- Verify image references survive persistence and permalink reload without raw
+  base64, local paths, or hidden wrappers appearing in chat output.
+- Separate direct model vision from `inspect_media`, browser screenshots, and
+  image generation. The harness must not silently substitute one for another.
+- Compare the fine-tune with its base VL model on the same images to detect
+  whether tool training regressed visual understanding.
+
+### G. Regular-model legacy RAG and tool coverage
+
+- Inventory every enabled non-Odysseus endpoint/model visible in 7011, including
+  its provider, schema mode, native-tool support, context limit, and configured
+  permissions. Do not assume every provider supports the same wire format.
+- Assert that no non-Odysseus model enters the clean-v3 runtime. These models
+  retain the regular RAG/tool loop and are repaired only in that owning path.
+- For each model, test every tool family the effective user policy offers:
+  direct request, misspelling, ambiguous follow-up, family switch, backend
+  failure, and Web/Bash toggle transitions. Record unsupported families as an
+  explicit capability limitation, not a silent pass.
+- Test full schemas versus compact schemas only where both are valid for that
+  model. Store the selected schema mode in every report.
+- Verify provider-native tool calls, textual fallback parsing where required,
+  canonical argument conversion, execution, evidence replay, and rendering.
+- Group fixes by shared legacy-runtime or provider-adapter defect. Do not add
+  model-name prompt exceptions when a transport, schema, or RAG ranking issue is
+  responsible.
+- Maintain a per-model compatibility matrix so adding or changing an endpoint
+  cannot silently regress previously working tools.
+
+## Fix protocol
+
+For each failure:
+
+1. Preserve the raw report and reproduce once on a fresh test session.
+2. Identify the primary failure layer from the taxonomy above.
+3. Add the smallest generic red test at that layer.
+4. Fix the owning module or invariant—not the literal prompt.
+5. Run the focused unit tests, the original scenario, two adjacent scenarios,
+   and the affected family suite.
+6. After a batch of category fixes, rerun the ten-family matrix and legacy-RAG
+   isolation test. Do not rerun training unless the contract and harness are
+   proven correct and failures remain model-owned.
+
+If three failures share a layer, pause case-by-case patching and refactor that
+layer before continuing.
+
+## Execution phases
+
+### Phase 1 — Make the runtime auditable
+
+- Add a sanitized per-turn decision record: model runtime, contract, proposed
+  calls, policy decisions with reason codes, executions, render owner, timings.
+- Add startup/runtime provenance to the UI so a linked chat proves which harness
+  handled it.
+- Add the offered-versus-policy compatibility self-test.
+- Correct stale preview documentation.
+
+### Phase 2 — Build the conversation suite
+
+- Extend the current Playwright verifier with reusable multi-turn scenarios and
+  reversible artifact fixtures.
+- Implement the matrix above, prioritizing search continuations and all
+  stateful corrections because real usage already exposed those gaps.
+- Run independent family groups in parallel, but serialize writes that share a
+  backend or fixture account.
+- Add a small versioned VL fixture set with locally generated, non-private
+  images and deterministic answer keys.
+
+### Phase 3 — Repair by architecture category
+
+- Consolidate model-specific runtime selection in one function.
+- Represent prior successful objects explicitly for referential follow-ups.
+- Align tool capability classification, contract offering, and policy decisions.
+- Standardize tool results into bounded envelopes with source/object IDs.
+- Keep search refinement and evidence sufficiency generic.
+
+### Phase 4 — Accuracy and speed comparison
+
+- Compare the clean fine-tune with the base model using identical compact tools,
+  prompts, toggles, backend state, and semantic scoring.
+- Report functional accuracy, argument accuracy, unsupported success claims,
+  TTFT, total latency, and tokens. Do not compare one model on full schemas and
+  another on compact schemas.
+- Only consider more SFT/RL for failures classified as model-owned after the
+  harness audit.
+
+### Phase 4B — Regular-model repair and verification
+
+- Snapshot the enabled non-Odysseus model inventory.
+- Run the legacy-RAG compatibility matrix in bounded parallel groups, respecting
+  endpoint rate limits and shared backend write serialization.
+- Fix shared harness/provider defects first, then rerun all affected models.
+- Publish separate per-model scores and limitations; do not blend them into the
+  Odysseus fine-tune score.
+
+### Phase 5 — Ship gate
+
+Ship only when:
+
+- every family is at least 90% on sealed functional holdout;
+- overall functional accuracy is at least 95%;
+- realistic follow-up suite is at least 95%, with no repeated failure category;
+- image/VL fixture accuracy does not regress materially from the base model and
+  all attachment/follow-up/persistence flows pass;
+- routing/execution and safety invariants are 100%;
+- all reversible writes are API-verified and cleaned up;
+- search quality and production email are reported separately and honestly;
+- non-Odysseus models demonstrably retain legacy RAG;
+- every enabled regular model has a complete tested-tool compatibility record,
+  and every tool advertised as supported passes its functional checks;
+- no hidden prompt leakage, duplicate rendering, or false success remains;
+- pre-heretic passing weights and merged adapter backups remain recoverable.
+
+## Immediate next batch
+
+1. Expand VL fixtures to charts, screenshots, and image-to-tool turns;
+   investigate the shared base-model OCR limitation without hiding it behind a
+   silent external fallback.
+2. Add deliberately permissioned private-browser open/snapshot/click checks;
+   keep browser interaction unavailable when its boundary is not enabled.
+3. Bring the two configured local regular models online and run their matrix.
+4. Compare fine-tune versus untouched base with identical compact contracts,
+   backend state, prompts, and timing instrumentation.
+5. Run the sealed all-action holdout and prioritize failures by shared
+   layer rather than by prompt.

plans/photo-editor-professional-roadmap.md

@@ -0,0 +1,445 @@
+# Plan: Odysseus Professional Photo Editor
+
+> Source PRD: Conversation goal, "a Photoshop/Photopea clone with Odysseus style"
+
+## Product boundary
+
+Odysseus should provide the editing loop people expect from a professional
+layer-based photo editor without copying Photoshop's visual design or trying to
+match every specialist feature. The target is a dependable browser editor for
+real photo work: direct manipulation, non-destructive layers, precise masking,
+retouching, typography, export, recovery, and optional AI assistance.
+
+The existing quiet Odysseus interface remains the visual language. Dense tools
+are acceptable, but controls should stay restrained, compact, predictable, and
+usable on both desktop and touch devices.
+
+## Existing foundation
+
+The current editor already provides meaningful parts of this product:
+
+- Raster and editable text layers
+- Multi-layer selection, nested groups, clipping, visibility, opacity, and locks
+- Layer, group, and selection masks
+- Marquee, lasso, wand, SAM, Quick Mask, and saved selections
+- Brush, eraser, clone, crop, transform, and text tools
+- Blend modes, adjustment stacks, blur, and several image corrections
+- Rulers, guides, grid, snapping, zooming, and panning
+- Undo/redo history with a memory budget
+- Versioned layered-project serialization, autosave drafts, recovery, and export
+- Optional endpoint-backed inpaint and image-processing tools
+- Desktop and mobile editor layouts with Playwright release-gate coverage
+
+## Architectural decisions
+
+Durable decisions that apply across every phase:
+
+- **Editor ownership**: The editor remains an Odysseus feature. Do not embed a
+  third-party editor or imitate another product's chrome.
+- **Document format**: Continue the versioned Odysseus editor document. Every
+  new persistent capability requires a migration, validation, round-trip test,
+  and corrupt-input recovery behavior.
+- **Layer model**: Grow the document into explicit layer kinds rather than
+  hiding more behavior in raster canvases. The intended kinds are raster, text,
+  shape, adjustment, and placed/smart content.
+- **Non-destructive default**: Preserve source pixels and editable parameters
+  whenever practical. Destructive actions remain available as explicit Apply,
+  Rasterize, or Merge commands.
+- **Interaction engine**: Transform, crop, selections, text frames, masks, and
+  shapes share one pointer-session model for hit testing, pointer capture,
+  modifiers, snapping, cancellation, and undo transactions.
+- **Rendering**: Keep Canvas 2D as the compatibility renderer initially. Move
+  expensive compositing and pixel operations behind renderer/worker boundaries
+  before considering WebGL or WebGPU acceleration.
+- **History**: One continuous gesture creates one undo entry. Preview frames are
+  never separate history entries, and Cancel restores the exact starting state.
+- **Persistence routes**: Continue using `/api/editor-drafts` for layered draft
+  persistence and `/api/gallery` for media-library save/replace operations.
+- **AI boundary**: AI features consume capability-based image endpoints. Core
+  editing never requires a particular model, repository, or provider.
+- **Responsive behavior**: Desktop favors precision; touch targets gain larger
+  invisible hit areas without visually enlarging the whole interface.
+- **Testing**: Every phase adds deterministic geometry/unit tests and at least
+  one complete Playwright workflow covering persistence and undo where relevant.
+- **Incremental architecture**: New behavior leaves the main editor orchestrator
+  through small domain modules. Avoid broad refactors that do not deliver a
+  visible editing improvement in the same phase.
+
+---
+
+## Phase 1: Accurate Transform Frame
+
+**User stories**: I can clearly see and grab the transform frame at any zoom. I
+can resize from corners or sides without grabbing invisible or incorrect areas.
+
+### What to build
+
+Replace the four-corner-only frame with a shared frame geometry model. Render
+four corners, four edge handles, a rotation control, and an optional center
+pivot from the same geometry used for hit testing. Keep handles visually compact
+while providing touch-sized invisible targets. Make the frame stay aligned
+during zoom, pan, viewport resize, and when handles extend outside the image.
+
+### Acceptance criteria
+
+- [x] Eight resize handles, rotation control, and center pivot derive from one geometry result.
+- [x] Drawn handles and hit targets cannot disagree.
+- [x] Handles remain a stable visual size from minimum to maximum zoom.
+- [x] Touch hit targets are at least 40 CSS pixels without oversized visuals.
+- [x] Outside-canvas handles remain interactive and visible when space permits.
+- [x] Hover and active cursors match each handle's current screen direction.
+- [x] Desktop and mobile Playwright tests grab every handle successfully.
+
+---
+
+## Phase 2: Correct Rotated Resize
+
+**User stories**: I can resize a rotated layer naturally. The opposite side or
+corner stays fixed, and the frame follows my pointer rather than drifting.
+
+### What to build
+
+Calculate drag movement in the frame's rotated local coordinate system. Anchor
+the opposite handle in document space and derive the new center from that
+anchor. Support crossing an axis as a deliberate flip instead of clamping to a
+one-pixel box. Apply the same geometry to one layer, multiple layers, and a
+selection transform.
+
+### Acceptance criteria
+
+- [x] Rotated corner and edge drags follow the pointer on the frame's local axes.
+- [x] The opposite anchor remains fixed within a sub-pixel tolerance.
+- [x] Crossing width or height zero produces a predictable horizontal or vertical flip.
+- [x] Shift locks the starting aspect ratio.
+- [x] Alt/Option scales around the transform center.
+- [x] Combined Shift+Alt/Option behavior is deterministic.
+- [x] Rotation snaps to 15-degree increments with Shift and remains smooth otherwise.
+- [x] Geometry tests cover 0, 45, 90, 135, and arbitrary-degree rotations.
+
+---
+
+## Phase 3: Transform Interaction Polish
+
+**User stories**: Transform behaves like a professional tool on mouse, pen, and
+touch. I can see exact values, snap precisely, and never lose a drag at the edge.
+
+### What to build
+
+Use a unified pointer session with pointer capture, live modifiers, and a small
+contextual transform readout. Add accurate rotated-frame interior hit testing,
+keyboard nudging, frame snapping, and clear Apply/Cancel behavior. Keep the
+existing compact Odysseus styling and make the numeric popup a precision surface
+rather than a competing transform implementation.
+
+### Acceptance criteria
+
+- [x] Pointer capture keeps a drag alive outside the canvas and browser viewport.
+- [x] Clicking inside a rotated frame moves it; clicking its empty bounding-box corner does not.
+- [x] Live X, Y, W, H, and angle values stay synchronized with direct manipulation.
+- [x] Arrow keys nudge, Shift+Arrow performs a larger nudge, Enter applies, and Escape cancels.
+- [x] Layer edges, document center/edges, guides, and grid participate in transform snapping.
+- [x] Snap guides clearly identify the active alignment without obscuring the photo.
+- [x] A complete gesture creates exactly one undo step.
+- [x] Touch gestures do not conflict with viewport pinch/pan behavior.
+
+---
+
+## Phase 4: Transform Content Correctness
+
+**User stories**: Transforming layers never unexpectedly damages masks, text,
+group layout, clipping, or image quality. Saving and reopening preserves it.
+
+### What to build
+
+Route raster layers, text layers, linked and unlinked masks, selections, clipped
+layers, and grouped multi-selection through the same transform contract. Keep
+immutable source data during previews and validate the final result through
+undo, cancel, autosave, project download, and reopen.
+
+### Acceptance criteria
+
+- [x] Raster previews are always derived from the session source, never a prior preview.
+- [x] Editable text remains editable after scaling, rotation, and flipping.
+- [x] Linked masks follow the layer while unlinked masks remain in document space.
+- [x] Multi-layer transforms preserve relative centers, order, clipping, and group membership.
+- [x] Transforming a selection changes only the selection mask unless content transform is explicitly chosen.
+- [x] Apply, Cancel, Undo, Redo, autosave reopen, and project-file reopen produce matching pixels and metadata.
+- [x] Large transforms cannot allocate beyond the editor's documented surface budget.
+
+---
+
+## Phase 5: Shared Direct-Manipulation Sessions
+
+**User stories**: Crop, selections, masks, text boxes, and shapes feel consistent
+with Transform instead of each behaving like a separate mini application.
+
+### What to build
+
+Generalize the proven transform pointer session into a reusable interaction
+contract. Migrate crop and selection movement first as a visible tracer bullet,
+including modifiers, snapping, pointer capture, cancel, and one-step history.
+
+### Acceptance criteria
+
+- [x] Transform, crop, and selection movement use the same gesture lifecycle.
+- [x] Tool switching safely commits, cancels, or prompts according to one policy.
+- [x] No stale pointer session can modify a newly selected tool or document.
+- [x] Mouse, pen, and touch event behavior is covered by shared tests.
+- [x] Adding a future frame-based tool does not require another global event stack.
+
+---
+
+## Phase 6: Non-Destructive Placed Layers
+
+**User stories**: I can import an image, resize it repeatedly without cumulative
+quality loss, replace its source, and choose when to rasterize it.
+
+### What to build
+
+Introduce a placed/smart layer kind containing source pixels and persistent
+transform metadata. Import-as-layer uses this kind by default. Rendering applies
+the transform at composite time, while Rasterize produces a normal raster layer.
+
+### Acceptance criteria
+
+- [x] Repeated transforms render from the original source rather than resampling the last result.
+- [x] A placed layer can be replaced while preserving its transform and masks.
+- [x] Rasterize produces a visually matching editable raster layer.
+- [x] Masks, clipping, groups, blend modes, and opacity work with placed layers.
+- [x] Version migration and recovery handle missing or corrupt placed sources.
+- [x] Existing raster projects open without changed output.
+
+---
+
+## Phase 7: Professional Selections And Masks
+
+**User stories**: I can build, inspect, refine, save, transform, and reuse precise
+selections without manually repainting every edge.
+
+### What to build
+
+Unify marquee, lasso, wand, SAM, Quick Mask, and saved selections around one
+selection-mask model. Add explicit replace/add/subtract/intersect modes, feather,
+expand, contract, smooth, border, and a focused refine-edge workflow.
+
+### Acceptance criteria
+
+- [x] Every selection tool supports replace, add, subtract, and intersect modes.
+- [x] Feather, expand, contract, smooth, and border preview before applying.
+- [x] Quick Mask edits the same canonical selection shown by marching ants.
+- [x] Selection-to-layer-mask and layer-mask-to-selection round-trip accurately.
+- [x] Saved selections retain names and pixels across reopen.
+- [x] Edge refinement works without requiring an AI dependency.
+
+---
+
+## Phase 8: Paint And Retouch Workflow
+
+**User stories**: I can paint and retouch photographs with predictable strokes,
+reusable presets, and the controls expected for a mouse, pen, or touch device.
+
+### What to build
+
+Promote brush behavior into a reusable brush engine. Add spacing, smoothing,
+pressure mapping, blend mode, sampled color, presets, and stroke preview. Build
+healing, dodge, and burn as complete retouching paths using that engine.
+
+### Acceptance criteria
+
+- [x] Brush, eraser, clone, masks, and inpaint share spacing and smoothing behavior.
+- [x] Pressure can independently affect size, opacity, or flow when supported.
+- [x] Eyedropper samples composite or active-layer color.
+- [x] Brush presets can be created, named, selected, and deleted.
+- [x] Healing, dodge, and burn create one undo entry per stroke.
+- [x] Long strokes remain smooth without blocking the main interface.
+
+---
+
+## Phase 9: Editable Text And Shapes
+
+**User stories**: I can design labels, cards, and overlays with text and vector
+shapes that remain editable after saving and reopening.
+
+### What to build
+
+Add on-canvas text-frame editing, selection, caret behavior, typography, and
+alignment. Introduce shape layers for rectangle, ellipse, line, and path-backed
+polygons with editable fill, stroke, corners, and transform metadata.
+
+### Acceptance criteria
+
+- [x] Text is edited directly on canvas without immediately rasterizing.
+- [x] Font, size, weight, line height, letter spacing, alignment, and color persist.
+- [x] Rectangle, ellipse, line, and polygon shapes remain editable.
+- [x] Shape fill, stroke, width, and corner radius can be changed after creation.
+- [x] Text and shape layers support masks, clipping, groups, blend modes, and transform.
+- [x] Missing fonts fall back predictably without corrupting the project.
+
+---
+
+## Phase 10: Adjustment Layers And Color
+
+**User stories**: I can correct a photograph non-destructively and return later
+to modify the correction without reconstructing the edit.
+
+### What to build
+
+Promote adjustments into first-class layers with masks and clipping. Deliver
+Levels and Curves first, then exposure, white balance, hue/saturation, color
+balance, selective color, gradients, and channel-aware controls.
+
+### Acceptance criteria
+
+- [ ] Adjustment layers affect content below them and can be clipped or grouped.
+- [ ] Every adjustment has live preview, reset, visibility, opacity, mask, Apply, and Cancel behavior.
+- [ ] Levels includes histogram, input range, gamma, and output range.
+- [ ] Curves supports RGB and channel curves with editable points.
+- [ ] Color results match flattened export and project reopen.
+- [ ] Large previews are throttled or worker-backed and remain cancellable.
+
+---
+
+## Phase 11: Layer Effects And Filters
+
+**User stories**: I can add common visual effects without permanently altering
+the layer and can reorder or disable those effects later.
+
+### What to build
+
+Create an ordered non-destructive filter/effect stack. Begin with Gaussian blur,
+sharpen, shadow, stroke, and color overlay; then add filter masks and reusable
+effect presets.
+
+### Acceptance criteria
+
+- [ ] Effects can be added, reordered, toggled, edited, masked, and removed.
+- [ ] Drop shadow, stroke, color overlay, blur, and sharpen survive project reopen.
+- [ ] Effects render correctly inside groups and clipping stacks.
+- [ ] Apply/rasterize produces a pixel-equivalent raster result.
+- [ ] Expensive filters expose progress and cancellation.
+
+---
+
+## Phase 12: Odysseus Professional Workspace
+
+**User stories**: I can work quickly without fighting floating windows or losing
+the active tool, layer, selection, or document context.
+
+### What to build
+
+Refine the existing shell into a consistent professional workspace: contextual
+tool options, properties inspector, panel persistence, command search, status
+information, multi-document switching, and compact touch sheets. Preserve the
+current Odysseus palette, typography, restrained borders, and frosted surfaces.
+
+### Acceptance criteria
+
+- [ ] Tool options appear in one predictable location and never duplicate popup state.
+- [ ] Panels remember size, collapsed state, and position per device class.
+- [ ] The properties inspector follows the active layer, mask, selection, or tool.
+- [ ] Command search exposes actions and shortcuts without adding toolbar clutter.
+- [ ] Switching documents preserves independent history, zoom, pan, and selection.
+- [ ] Mobile prioritizes canvas area while keeping all commands reachable.
+
+---
+
+## Phase 13: File Interchange And Export
+
+**User stories**: I can bring common assets into Odysseus and export predictable
+results without losing transparency, dimensions, or color intent.
+
+### What to build
+
+Strengthen image import/export first, then add layered interchange where a
+maintained parser makes it safe. Keep Odysseus project files as the lossless
+source of truth and clearly report what an external format cannot preserve.
+
+### Acceptance criteria
+
+- [ ] PNG, JPEG, WebP, and supported modern image imports honor orientation and transparency.
+- [ ] Export exposes format, dimensions, quality, metadata, and transparency choices.
+- [ ] Copy/paste and drag/drop preserve alpha and use placed layers when appropriate.
+- [ ] Layered imports report unsupported features instead of silently flattening them.
+- [ ] Exported pixels are covered by deterministic visual comparisons.
+
+---
+
+## Phase 14: Large-Document Performance And Recovery
+
+**User stories**: Large photos and layered projects remain responsive, autosave
+reliably, and recover after a crash or interrupted network connection.
+
+### What to build
+
+Move serialization, thumbnails, filters, and suitable pixel operations into
+workers. Add dirty-region rendering, reusable surfaces, measurable memory
+budgets, operation cancellation, autosave generations, and recovery diagnostics.
+
+### Acceptance criteria
+
+- [ ] Normal interactions remain responsive on the agreed 4K multi-layer benchmark.
+- [ ] Compositing avoids rebuilding unaffected layers and thumbnails.
+- [ ] History and document surfaces stay within explicit memory limits.
+- [ ] Closing or switching documents cancels stale work safely.
+- [ ] Autosave never lets an older request overwrite newer state.
+- [ ] Recovery can identify the last complete generation and explain skipped data.
+
+---
+
+## Phase 15: Odysseus-Native Assisted Editing
+
+**User stories**: I can use an available local or remote image capability as an
+editing assistant while retaining masks, layers, undo, privacy choices, and
+normal manual controls.
+
+### What to build
+
+Standardize image capability discovery and requests for generation, editing,
+inpainting, segmentation, restoration, and upscaling. Results enter the document
+as named layers with provenance and reusable masks. Add orchestration only after
+the manual operation it assists is dependable.
+
+### Acceptance criteria
+
+- [ ] The UI describes required capabilities rather than model or provider names.
+- [ ] Memory and unrelated chat context are not sent to image endpoints.
+- [ ] Requests show progress, support cancellation, and cannot update a closed document.
+- [ ] Generated results arrive as reversible layers with prompt/settings metadata.
+- [ ] A failed endpoint leaves the source document unchanged and offers a useful retry path.
+- [ ] Manual selection and masking remain available when assisted tools are absent.
+
+---
+
+## Phase 16: Professional Release Gate
+
+**User stories**: I can trust the editor for real work and understand what is
+unsupported before committing an edit.
+
+### What to build
+
+Create a release gate around complete user journeys rather than isolated button
+tests. Cover accessibility, keyboard-only operation, touch, browser differences,
+pixel correctness, persistence, failure recovery, and large-document behavior.
+
+### Acceptance criteria
+
+- [ ] Core workflows pass on current Chromium and Firefox desktop builds.
+- [ ] Mobile workflows pass at representative phone and tablet viewports.
+- [ ] Keyboard-only users can reach every command and escape every modal state.
+- [ ] Transform, masks, text, adjustments, export, and reopen have pixel/metadata regression tests.
+- [ ] No supported action silently flattens or discards editable document data.
+- [ ] The ALPHA badge can be removed based on explicit reliability metrics.
+
+---
+
+## Recommended delivery order
+
+The first four phases are one focused Transform 2.0 program and should ship in
+order. Phases 5 and 6 establish the interaction and document foundations needed
+for the remaining professional tools. After that, phases 7 through 13 can be
+prioritized by user value, while performance and release-gate work continue as
+part of every phase rather than being deferred entirely to the end.
+
+The recommended first milestone is complete when Phases 1 through 4 are live:
+transforming one layer, multiple layers, text, masks, and selections feels
+precise on desktop and mobile and remains correct through undo and reopen.

plans/photo-editor-remaining-scope.md

@@ -0,0 +1,159 @@
+# Photo Editor Remaining Scope
+
+Date: 2026-08-29
+
+## Current verdict
+
+Odysseus is now a credible layered everyday editor, not an editor mockup. The
+first nine roadmap phases are implemented: professional transform geometry,
+shared direct-manipulation sessions, retained placed content, unified
+selections and masks, a reusable brush/retouch engine, and retained text and
+shape layers.
+
+Phase 10 is functionally advanced but not closed. First-class adjustment layers
+now support Levels, Curves, Exposure, White Balance, Brightness/Contrast,
+Hue/Saturation/Lightness, Color Balance, Selective Color, and Gradient Map.
+They participate in clipping, groups, masks, visibility, opacity, history, the
+v14 document format, and flattening. Retained effects have since been added as
+a separate ordered stack with Gaussian Blur, Color Overlay, Drop Shadow, and
+Stroke, including editable colors, visibility, opacity, reorder, rasterize,
+history, persistence, and migration.
+
+Practical readiness estimate:
+
+- Everyday layered photo editing: **about 88%**
+- Dependable professional v1 described by the roadmap: **about 62%**
+- Broad Photoshop/Photopea feature parity: **about 50%**
+
+The remaining gap is dominated by large-document rendering outside the live
+composite path, workspace consolidation, interchange/color policy, and release
+proof rather than basic canvas tools.
+
+## Verification snapshot
+
+- The focused editor unit suite currently passes **31 tests** in Docker.
+- The full photo-editor browser suite currently has **41 passing workflows**;
+  the nested-group selection workflow initially exposed a row-hit regression,
+  which now passes on isolated rerun after the slider-selection fix. The new
+  group-effects workflow also passes.
+- The new adjustment tests exercise deterministic pixel math, nested parameter
+  normalization, retained metadata, undo/redo, clipping, masks, and draft
+  reopen.
+- The latest editor changes have not yet been rebuilt into the live `7011`
+  container.
+
+## Close Phase 10
+
+This is the immediate release slice.
+
+1. Finish the bounded preview path for large documents. Downsampled previews
+   now keep control movement responsive and full resolution is restored for
+   commit/export. Live worker composites now use generation checks, latest-only
+   coalescing, and close/reopen invalidation; extend the same guarantees to
+   remaining preview paths.
+2. Add flattened-export versus reopened-project pixel comparisons for every
+   adjustment family, including groups, clipping, masks, blend mode, and
+   partial opacity.
+3. Validate the color algorithms visually. White Balance and Selective Color
+   are currently deterministic approximations, not color-managed photographic
+   transforms.
+4. Test every adjustment popup on phone and desktop viewports, including tall
+   popups, color inputs, drag, Reset, Apply, Cancel, and Escape.
+5. Decide the migration path for the older per-raster `adjLayers` stack. It can
+   remain readable for compatibility, but new UI should converge on first-class
+   adjustment layers instead of maintaining two competing concepts.
+6. Bump static cache versions, rebuild the live container, and run a short
+   visual smoke test on `7011`.
+
+## Phase 11: Retained effects and filters
+
+The retained-effects slice is implemented for raster/placed/text/shape-compatible
+layer output: Gaussian Blur, Sharpen, Color Overlay, Drop Shadow, and Stroke
+have editable colors/parameters, visibility, opacity, reorder, rasterize,
+history, migration, and reopen support. Effect-specific masks, presets, and
+group-level effects are also implemented and covered by focused browser tests.
+Remaining work is:
+
+1. Extend worker coverage to serialization and remaining preview paths.
+   Thumbnail encoding, retained-effect rasterization, and live composite
+   rendering now use a worker where OffscreenCanvas is available, with
+   synchronous compatibility fallbacks. Generation invalidation, latest-only
+   coalescing, and CPU loop cancellation protect live rendering.
+2. Add explicit group-effect blend/ordering tests for nested groups and
+   non-default blend modes, plus visual comparisons for effect stacks.
+
+Introduce the renderer/worker cancellation boundary here rather than adding
+more synchronous full-canvas filters that Phase 14 must immediately replace.
+
+## Phase 12: Professional workspace
+
+Consolidate fragmented popups into one contextual properties surface. Persist
+panel layout by device class, add command search, expose stable document status,
+and support multiple open documents with independent history, zoom, pan, and
+selection. Mobile should use canvas-first sheets rather than compressed desktop
+panels.
+
+## Phase 13: Interchange and export
+
+Harden orientation, transparency, metadata, and color behavior for PNG, JPEG,
+and WebP first. Add copy/paste and drag/drop through placed layers. Treat
+layered formats as explicit compatibility projects: unsupported PSD/TIFF/HEIC
+features must be reported, never silently discarded. Odysseus project files
+remain the lossless source of truth.
+
+## Phase 14: Performance and recovery
+
+Move remaining preview/pixel paths into workers. Thumbnail encoding,
+autosave serialization, adjustment rendering, and retained-effect rendering
+now have worker-backed paths with compatibility fallbacks. Add
+dirty-region compositing, reusable render surfaces, cancellation tokens,
+operation telemetry, a documented surface/history budget, autosave generations,
+and a checked-in 4K multi-layer benchmark.
+
+This phase is the main architectural risk. Canvas 2D remains a valid
+compatibility renderer, but full-document synchronous passes will not scale to
+professional documents.
+
+## Phase 15: Assisted editing
+
+Normalize generation, editing, inpainting, segmentation, restoration, and
+upscaling behind capability-based endpoints. Keep model/provider names out of
+editor logic. Requests must exclude chat memory, show progress, cancel safely,
+and return named reversible layers with provenance. Manual tools remain fully
+usable without an endpoint.
+
+Much of the endpoint plumbing already exists; the remaining work is consistent
+capability discovery, lifecycle safety, and editor-native result handling.
+
+## Phase 16: Release gate
+
+Run complete user journeys on Chromium and Firefox desktop plus representative
+phone/tablet viewports. Add keyboard-only and accessibility coverage, mixed
+20-edit persistence/export tests, failure recovery, and large-document stress
+tests. No supported operation may silently flatten or discard retained state.
+
+## Architecture debt to control
+
+- `galleryEditor.js` is still a large orchestrator. Continue extracting domain
+  modules as visible features move, without a broad rewrite.
+- Legacy raster adjustment sublayers and first-class adjustment layers overlap.
+  Converge on the first-class model.
+- Pixel effects still rely heavily on synchronous full-canvas work.
+- `static/style.css` carries substantial editor-specific surface area and needs
+  clearer component boundaries before workspace customization expands.
+- The repository worktree contains many unrelated changes. Editor release and
+  merge decisions require a scoped diff or clean integration branch.
+
+## Recommended execution order
+
+1. Close and deploy Phase 10.
+2. Build Phase 11 through a cancellable render boundary.
+3. Consolidate the workspace in Phase 12.
+4. Define color/metadata policy and complete Phase 13.
+5. Finish worker rendering, stress, and recovery in Phase 14.
+6. Normalize assisted editing in Phase 15.
+7. Run the cross-browser professional release gate in Phase 16.
+
+Do not expand into full PSD fidelity, CMYK production, RAW development, 3D, or
+complete Photoshop parity before this critical path passes. Those are separate
+product decisions, not prerequisites for a strong Odysseus editor.

requirements-optional.txt

@@ -1,4 +1,7 @@
 # Optional dependencies — install only if you use the corresponding feature.
+# Local OCR for screenshots, scans, labels, and coordinate-grounded text extraction.
+rapidocr==3.9.2
+onnxruntime>=1.20,<2
 # The app handles their absence gracefully (clear error message on first use).
 #
 # Note: chromadb-client + fastembed moved to requirements.txt — RAG, semantic
@@ -12,6 +15,16 @@
 # GPU-accelerated transcription — it's auto-detected, CPU is used otherwise.
 faster-whisper
 
+# Local text-to-speech via Kokoro-82M for the "local" TTS provider.
+# Kokoro 0.9.4 declares Python >=3.10,<3.13; Odysseus itself requires 3.11+,
+# so pip installs these extras on 3.11-3.12 and deliberately skips them on
+# Python 3.13+ (including the Python 3.14 container image). Kokoro declares
+# torch; the local provider still
+# requires a CUDA-enabled torch build and GPU at runtime. SoundFile is separate
+# in Kokoro's official install instructions and is not a transitive dependency.
+kokoro==0.9.4; python_version >= "3.11" and python_version < "3.13"
+soundfile; python_version >= "3.11" and python_version < "3.13"
+
 # DuckDuckGo as a search provider option.
 # Install if you want DDG in the search-provider dropdown.
 # Alternatives: SearXNG, Brave, Tavily, Serper, Google PSE.
@@ -34,3 +47,6 @@ PyMuPDF
 # [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
 # the dependency-age discussion in issue #485.
 markitdown[docx,pptx,xlsx,xls]==0.1.6
+
+# Photoshop PSD opening / flattened previews / layer inspection.
+psd-tools

requirements.txt

@@ -8,6 +8,11 @@ pydantic>=2.13.4
 pydantic-settings>=2.14.1
 SQLAlchemy
 pypdf
+pypdfium2
+Pillow
+faster-whisper
+PyPDF2
+pdfplumber
 beautifulsoup4
 charset-normalizer
 numpy
@@ -19,6 +24,7 @@ numpy
 chromadb-client
 fastembed
 youtube-transcript-api
+yt-dlp
 # Markdown rendering for research reports (src/visual_report.py).
 # Imported at module-top so it's a hard core dep, not optional.
 markdown
@@ -38,7 +44,10 @@ python-dateutil
 caldav
 cryptography
 bcrypt
-mcp
+# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
+# breaking rewrite, so keep fresh installs on the maintained v1 line until the
+# servers are migrated together.
+mcp<2
 pyotp
 qrcode[pil]
 croniter

resources/skills/agent/artifact-completion/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: artifact-completion
+description: Create requested artifacts early, iterate from concrete output, and verify final deliverables
+version: 1.0.0
+category: agent
+tags: [artifacts, files, verification, workflow]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the task requires a file, patch, report, document, image, archive, configuration, or other persistent deliverable rather than only a text answer.
+
+## Procedure
+
+1. Extract the required deliverable path, format, content constraints, and acceptance criteria.
+2. Inspect the source material and existing target without delaying the first valid artifact.
+3. Create a minimal complete version at the required location, then iterate from that concrete output.
+4. Use the format's native parser, renderer, compiler, or test tool to inspect the artifact.
+5. Repair specific validation, content, or presentation failures while preserving correct portions.
+6. Confirm the final path, file type, required content, and usability before reporting completion.
+
+## Pitfalls
+
+- Do not spend the full task budget inspecting without creating the requested output.
+- Do not place the artifact at a convenient path when the task specifies another location.
+- Do not use a filename extension as proof that the file is valid in that format.
+- Do not report completion while placeholders, missing sections, parse errors, or failed checks remain.
+
+## Verification
+
+- The artifact exists at the required path and opens or parses successfully.
+- Required sections, fields, labels, or visual elements are present.
+- Relevant tests, render checks, or validators pass.

resources/skills/agent/terminal-recovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: terminal-recovery
+description: Recover from failed terminal commands using evidence-driven diagnosis and bounded retries
+version: 1.0.0
+category: agent
+tags: [terminal, shell, debugging, recovery]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a command fails, times out, produces incomplete output, or behaves differently from what the task requires.
+
+## Procedure
+
+1. Read the command, exit status, standard output, and standard error before choosing a response.
+2. Confirm the working directory, relevant files, executable availability, permissions, and environment assumptions with minimal read-only probes.
+3. Classify the failure as syntax, missing dependency, wrong path, permissions, resource pressure, timeout, service state, or task logic.
+4. Change one relevant condition and retry the narrowest command that can test the diagnosis.
+5. For a long-running command, use the returned session identifier to poll or provide input instead of launching duplicates.
+6. After recovery, run the original acceptance check and inspect the resulting files or service state.
+
+## Pitfalls
+
+- Do not rerun an unchanged failing command repeatedly.
+- Do not install packages or change global configuration before confirming they are missing and necessary.
+- Do not launch a second server or training job before checking for an existing process and port or device conflicts.
+- Do not treat partial output or a zero exit status as proof that the requested state was produced.
+
+## Verification
+
+- The diagnosed cause is supported by command output or environment state.
+- The corrected command exits as expected.
+- The requested artifact, process, or state passes an independent acceptance check.

resources/skills/agent/tool-discovery/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: tool-discovery
+description: Discover the smallest capable tool set and confirm argument schemas before acting
+version: 1.0.0
+category: agent
+tags: [tools, discovery, routing, schemas]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires tools whose names, capabilities, or argument shapes are not already clear. This is especially useful when many tools are available or a previous call failed because the wrong tool or parameters were selected.
+
+## Procedure
+
+1. Translate the request into required capabilities such as reading, searching, editing, executing, browsing, or verifying.
+2. Search the tool index for those capabilities and inspect the returned tool descriptions and schemas.
+3. Prefer one direct tool over a chain of indirect tools when it can complete the operation and provide evidence.
+4. Check required parameters, identifiers, path rules, side effects, and approval requirements before calling the tool.
+5. Make a small read-only probe when the environment or target is uncertain.
+6. Execute the selected action, inspect the result, and only broaden the tool search if the result shows a concrete capability gap.
+
+## Pitfalls
+
+- Do not guess tool names or argument keys from memory when the index or schema is available.
+- Do not load unrelated tool groups into context.
+- Do not repeat the same failed call without changing the arguments or strategy.
+- Do not use a broad shell or browser workaround when a scoped native tool already owns the operation.
+
+## Verification
+
+- The chosen tool directly matches the required capability.
+- Required arguments follow the exposed schema.
+- The result contains evidence of the requested effect or a specific error that guides the next step.

resources/skills/agent/verified-state-change/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: verified-state-change
+description: Make scoped state changes with target confirmation, minimal mutation, and read-back verification
+version: 1.0.0
+category: agent
+tags: [state, mutation, verification, safety]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when creating, editing, deleting, moving, sending, scheduling, or otherwise changing persistent state through an application, API, filesystem, or service.
+
+## Procedure
+
+1. Read the current state and identify the target using stable identifiers plus enough content to disambiguate it.
+2. Preserve fields the user did not ask to change and choose the narrowest supported mutation.
+3. For destructive or externally visible actions, confirm that the user's instruction authorizes the exact target and effect.
+4. Perform the mutation once and capture the returned identifier, status, or revision.
+5. Read the target again through an independent list, fetch, status, or content operation.
+6. Compare the observed state with the requested outcome and repair only the specific mismatch.
+
+## Pitfalls
+
+- Do not infer the target from a stale active item when a stable identifier can be fetched.
+- Do not report success from an accepted request alone; asynchronous or partial operations may not have completed.
+- Do not replace an entire object when a field-level update is supported and safer.
+- Do not silently broaden a mutation to adjacent files, records, accounts, or services.
+
+## Verification
+
+- The target identity was confirmed before mutation.
+- A read-back shows the intended values and preserves unrelated state.
+- Any external effect has a concrete status, identifier, or observable result.

resources/skills/communication/action-evidence-synthesis/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: action-evidence-synthesis
+description: "Turn messages, meeting notes, and documents into sourced decisions, actions, dependencies, and risks"
+version: 1.0.0
+category: communication
+tags: [messages, meetings, actions, status, evidence]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when information is fragmented across messages, meeting notes, transcripts, or documents and the user needs an action list, status summary, feasibility assessment, or executive brief.
+
+Do not use when the source material is unavailable or when the user only wants a verbatim transcript.
+
+## Procedure
+
+1. Identify the requested scope, audience, time window, and decision to support.
+2. Gather the relevant records in full and preserve stable source identifiers, authors, and timestamps.
+3. Extract explicit decisions, commitments, requests, owners, dates, dependencies, blockers, and changed facts.
+4. Reconcile revisions by preferring the newest authoritative record; keep unresolved conflicts visible instead of guessing.
+5. Separate observed facts from inferred owners, dates, urgency, feasibility, or recommendations, and label every inference as tentative.
+6. Produce the requested format with concise source references beside consequential claims and a final list of open questions.
+
+## Pitfalls
+
+- Do not turn discussion or speculation into a confirmed decision.
+- Do not invent owners or deadlines when none were assigned.
+- Do not silently discard older records that explain a changed commitment.
+- Do not send messages, create tasks, or update calendars unless the user separately authorizes those actions.
+
+## Verification
+
+- Every action has a source, status, and explicit or tentative owner and due date.
+- Conflicting values and revisions are resolved or visibly flagged.
+- The output covers decisions, actions, dependencies, risks, and open questions relevant to the request.
+

resources/skills/communication/reviewable-external-draft/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: reviewable-external-draft
+description: "Reconcile source evidence and prepare an accurate external-facing draft without bypassing review"
+version: 1.0.0
+category: communication
+tags: [drafting, email, messages, review, reconciliation]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when preparing a client, customer, partner, leadership, or other external-facing update from internal messages or documents.
+
+Do not use this procedure to send immediately unless the user explicitly authorizes the exact recipient and final content.
+
+## Procedure
+
+1. Confirm the audience, communication channel, requested tone, and whether the user asked for a draft or an immediate send.
+2. Gather the relevant source records and identify the latest values, dates, commitments, and unresolved discrepancies.
+3. Resolve recipient identity through the available contact source and avoid inferring internal versus external status from a display name alone.
+4. Draft only claims supported by the collected evidence; qualify uncertainty and omit internal-only detail that the audience should not receive.
+5. Save or present a reviewable draft through the native draft or document capability.
+6. Report the draft identifier or location plus any reconciliation notes that require human review.
+
+## Pitfalls
+
+- Do not send a draft merely because a send-capable tool is available.
+- Do not copy stale figures when a later correction exists.
+- Do not conceal unresolved discrepancies behind polished prose.
+- Do not expose private internal discussion, credentials, or unrelated personal data.
+
+## Verification
+
+- Recipient identity and communication mode match the request.
+- Dates, figures, status, and commitments map to current source evidence.
+- The result remains reviewable unless an explicit send-now instruction authorized delivery.
+

resources/skills/communication/scheduling-coordination/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: scheduling-coordination
+description: "Coordinate availability, confirmations, calendar changes, and participant notifications with read-back verification"
+version: 1.0.0
+category: communication
+tags: [calendar, scheduling, coordination, availability]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when arranging or changing a meeting across multiple participants, calendars, time zones, or communication channels.
+
+Do not create or modify an event when the user asked only for available options or a draft invitation.
+
+## Procedure
+
+1. Extract participants, duration, date range, time zones, location constraints, and required attendees.
+2. Resolve participant identities and inspect the relevant availability using declared calendar and contact capabilities.
+3. Compute candidate intervals in one explicit reference time zone and reject conflicts or insufficient travel buffers.
+4. Present or draft a small set of viable options when confirmation is still required.
+5. After authorization or recorded participant confirmation, create or update the event once with stable attendee identifiers.
+6. Read the event back and verify title, start, end, time zone, attendees, location, and conferencing details before drafting notifications.
+
+## Pitfalls
+
+- Do not overwrite or cancel unrelated events to manufacture availability.
+- Do not mix local times without naming the time zone.
+- Do not treat a proposed time as confirmed.
+- Do not create duplicates when an existing event can be updated safely.
+
+## Verification
+
+- The selected interval satisfies duration, availability, and time-zone constraints.
+- The calendar read-back matches the authorized event details.
+- Notifications describe the same confirmed event and remain drafts unless sending was explicitly authorized.
+

resources/skills/communication/support-triage-and-routing/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: support-triage-and-routing
+description: "Prioritize support requests, identify owners, route internally, and prepare safe customer drafts"
+version: 1.0.0
+category: communication
+tags: [support, triage, urgency, routing, drafts]
+status: published
+confidence: 1.0
+source: builtin
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when reviewing a support backlog, identifying urgent incidents, assigning internal ownership, or drafting customer responses.
+
+Do not use when the request is merely to summarize an unrelated inbox or when sender identity cannot be established safely.
+
+## Procedure
+
+1. Read each in-scope request in full and retain its stable message or ticket identifier.
+2. Resolve whether the sender is internal or external and identify the responsible internal team from available contacts and service ownership data.
+3. Classify urgency from impact and time sensitivity: critical for outage, data loss, security exposure, or imminent contractual breach; high for a blocked user without a workaround; medium for degraded service with a workaround; low for non-blocking inquiries.
+4. Record a concise problem statement, evidence, affected scope, workaround, owner, next action, and response deadline.
+5. Route internally only when the user has authorized operational messaging; prepare external responses as reviewable drafts by default.
+6. Re-read created assignments or drafts and produce an escalation summary grouped by urgency.
+
+## Pitfalls
+
+- Do not infer severity from emotional language alone.
+- Do not expose one customer's data in another customer's response.
+- Do not send externally when the task calls for triage or drafting.
+- Do not mark an issue routed without a stable owner or observable routing result.
+
+## Verification
+
+- Every issue has a stable source identifier, urgency rationale, owner, and next action.
+- Critical and high items have explicit response targets and escalation state.
+- External communication is a draft unless the user explicitly authorized sending.

resources/skills/dev/developer-docs/SKILL.md

@@ -0,0 +1,37 @@
+---
+name: developer-docs
+description: Find, read, and apply authoritative developer documentation during implementation
+version: 1.0.0
+category: dev
+tags: [docs, documentation, api, software-development]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when the user asks how a library, framework, API, protocol, CLI, or SDK works, or when implementation depends on version-specific behavior. Prefer this skill over guessing from memory.
+
+## Procedure
+
+1. Identify the exact product, package, version, and task. Ask one focused clarification only when the target is genuinely ambiguous.
+2. Prefer the vendor's or project's primary documentation, source repository, release notes, and API reference. Use a general search only to locate those sources.
+3. Read the relevant page or reference section, then apply the documented behavior to the user's codebase and active workspace.
+4. Separate documented facts from inference, and call out version or environment assumptions.
+5. For code changes, add a focused regression test for the documented contract and run it before reporting completion.
+
+## Pitfalls
+
+- Do not present search snippets, stale cached knowledge, or a third-party tutorial as authoritative when primary documentation is available.
+- Do not silently mix instructions from different major versions.
+- Do not claim an API or option exists without confirming it in the relevant reference.
+- Do not use web search for a local project task when the active workspace and local tools can answer it.
+
+## Verification
+
+- The cited or retrieved documentation matches the target version.
+- The implementation or answer distinguishes source-backed facts from inference.
+- Any code change has a focused test or a concrete verification command.

resources/skills/general/test-driven-development/SKILL.md

@@ -0,0 +1,40 @@
+---
+name: test-driven-development
+description: Build or fix software with a focused red-green-refactor loop
+version: 1.0.0
+category: general
+tags: [tdd, testing, debugging, red-green-refactor]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-18T00:00:00Z"
+---
+
+## When to Use
+
+Use when implementing a feature, fixing a bug, or changing behavior where a regression test can define the expected result. Prefer this workflow for parser, routing, agent-loop, and UI behavior changes.
+
+## Procedure
+
+1. Inspect the relevant code, existing tests, and local conventions before editing.
+2. Write the smallest regression test that demonstrates the requested behavior or reproduces the bug.
+3. Run that test and confirm it fails for the expected reason, not because the test setup is broken.
+4. Make the smallest production change that makes the test pass.
+5. Run the focused test again, then run the surrounding module suite.
+6. Review the diff for unrelated changes, brittle assertions, hidden state, and missing error paths.
+7. Report the tests run and any remaining coverage or environment limits.
+
+## Pitfalls
+
+- Do not write a test that only mirrors the implementation; assert the user-visible contract.
+- Do not weaken an assertion just to make a failing test pass.
+- Do not skip the focused failing-test step when the behavior is observable in a local test.
+- Keep network, filesystem, and model calls deterministic with fakes or fixtures unless the integration itself is under test.
+
+## Verification
+
+- The new regression test fails before the fix and passes after it.
+- The relevant focused suite passes.
+- The broader suite passes or its failure is explained with evidence.
+- The final diff contains the test and the production change needed for the same behavior.

resources/skills/media/multimodal-evidence/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: multimodal-evidence
+description: Extract and verify evidence from images, documents, and video without redundant inspection
+version: 1.0.1
+category: media
+tags: [image, video, document, evidence, ocr]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when the answer or requested artifact depends on visual, temporal, tabular, or textual evidence contained in images, documents, or video.
+
+## Procedure
+
+1. Identify the evidence required: objects, text, values, ordering, timestamps, labels, or visual relationships.
+2. Inspect the whole input or a broad representative sample first to establish structure and likely evidence locations.
+3. Narrow to relevant pages, frames, regions, or time intervals and record observations with their locations.
+4. Use the format's native parser for exact text and numbers: for example `python-docx` or ZIP/XML inspection for DOCX, `pdftotext` or a PDF library for PDF, spreadsheet readers for XLSX, and OCR only when the source is image-based. Do not search binary office files with plain `grep` or `cat`.
+5. Resolve conflicts with one targeted reinspection at better scale or a nearby frame rather than repeating the same crop.
+6. Build the answer or artifact from the evidence ledger and perform a final coverage check against every requested item.
+
+## Pitfalls
+
+- Do not infer unseen content from filenames, surrounding text, or a single thumbnail.
+- Do not repeatedly inspect nearly identical regions without a new hypothesis.
+- Do not trust OCR blindly for small labels, punctuation, or numeric values.
+- Do not finalize before checking that every requested item has supporting evidence.
+
+## Verification
+
+- Each factual output can be traced to a page, frame, region, or timestamp.
+- Exact labels and numbers were visually checked after extraction.
+- The final response or artifact covers all requested evidence categories.

resources/skills/research/web-research-fallback/SKILL.md

@@ -0,0 +1,38 @@
+---
+name: web-research-fallback
+description: Research current web information with source-first search and controlled browser fallback
+version: 1.0.0
+category: research
+tags: [web, search, browser, sources, research]
+status: published
+confidence: 1.0
+source: builtin
+owner: ""
+created: "2026-08-30T00:00:00Z"
+---
+
+## When to Use
+
+Use when a task requires current public information, primary sources, multiple pages, or a site that cannot be reliably read from search results alone.
+
+## Procedure
+
+1. Define the facts needed and the preferred primary source for each fact.
+2. Search with a focused query and use result metadata to select likely authoritative pages.
+3. Open the source directly and extract the relevant passage, date, and URL rather than relying on a search snippet.
+4. Use the private browser when the page requires interaction, client-side rendering, navigation, or visual inspection.
+5. If a page fails, try a primary-source alternative or a narrower route before broadening to secondary sources.
+6. Cross-check unstable or consequential claims and distinguish source-backed facts from inference.
+
+## Pitfalls
+
+- Do not treat snippets as evidence for claims not visible on the source page.
+- Do not browse repeatedly without recording what each page established.
+- Do not use a secondary summary when an accessible primary source answers the question.
+- Do not claim freshness without checking publication or update dates.
+
+## Verification
+
+- Each important claim maps to a source that directly supports it.
+- Time-sensitive facts include an observed date or version.
+- Browser interaction produced the needed page state or a documented fallback was used.

routes/assistant_routes.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 
 from core.database import SessionLocal, CrewMember, ScheduledTask
 from src.auth_helpers import get_current_user
-from core.auth import RESERVED_USERNAMES
+from src.owner_identity import REQUEST_SENTINEL_OWNERS
 from src.task_scheduler import compute_next_run
 
 
@@ -90,11 +90,12 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
     # check-in tasks seeded. Hitting any /assistant route under one of these
     # used to seed a full CrewMember + Morning/Midday/Evening tasks under that
     # owner, which then double-fired alongside the real user's check-ins.
-    # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "".
+    # REQUEST_SENTINEL_OWNERS covers request-only identities; Default/Local is a
+    # reserved login name but remains a valid storage owner.
 
     async def _get_or_create(owner: str) -> CrewMember:
         """Return the per-owner assistant CrewMember, creating it on demand."""
-        if not owner or owner in RESERVED_USERNAMES:
+        if not owner or owner in REQUEST_SENTINEL_OWNERS:
             raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
         db = SessionLocal()
         try:

routes/auth_routes.py

@@ -22,6 +22,8 @@ from src.settings import (
     load_features as _load_features,
     save_features as _save_features,
     DEFAULT_SETTINGS,
+    RETIRED_SETTING_KEYS,
+    without_retired_settings,
 )
 from src.integrations import (
     load_integrations,
@@ -84,6 +86,33 @@ class SetOpenRegistrationRequest(BaseModel):
 SESSION_COOKIE = "odysseus_session"
 
 
+def _secure_cookie(request: Request) -> bool:
+    """Decide the ``Secure`` attribute of the session cookie.
+
+    ``SECURE_COOKIES`` stays authoritative when it holds an explicit value:
+    ``true`` always marks the cookie Secure (the documented knob for a TLS
+    proxy), ``false`` never does, which is the escape hatch for an install
+    that still answers on plain HTTP alongside HTTPS. Anything else —
+    unset, or the present-but-empty value docker-compose injects for a
+    variable the host has not defined — derives it from the request, so an
+    HTTPS login gets a Secure cookie without any configuration.
+
+    Either the connection scheme or ``X-Forwarded-Proto`` saying https is
+    enough, which is the same test ``core/middleware.py`` applies before it
+    sends HSTS. Uvicorn's proxy-headers middleware already folds that header
+    into the scheme for the proxies it trusts, so reading it here only adds
+    the case of a terminator that is not on a trusted address; the cost is
+    that a client talking to the app directly can set the header and lock
+    its own session out over plain HTTP.
+    """
+    configured = os.getenv("SECURE_COOKIES", "").strip().lower()
+    if configured in ("true", "false"):
+        return configured == "true"
+    # A chained proxy sends a list — the client-facing hop comes first.
+    forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0]
+    return request.url.scheme == "https" or forwarded_proto.strip().lower() == "https"
+
+
 def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
     router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -157,7 +186,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             value=token,
             httponly=True,
             samesite="lax",
-            secure=os.getenv("SECURE_COOKIES", "false").lower() == "true",
+            secure=_secure_cookie(request),
             path="/",
         )
         if body.remember:
@@ -345,9 +374,61 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         # docs, email accounts, tasks, etc.
         try:
             from sqlalchemy import func
-            from core.database import Base, SessionLocal
+            from core.database import (
+                Base,
+                EmailAccount,
+                SessionLocal,
+                lock_email_account_owner_mutations,
+            )
             db = SessionLocal()
             try:
+                # Email-account defaults are protected by per-owner mutex rows.
+                # A rename crosses two owner partitions, so lock both in the
+                # shared helper's canonical order before inspecting either.
+                lock_email_account_owner_mutations(
+                    db, old_username, new_username
+                )
+
+                source_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == old_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                destination_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == new_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                if destination_default_ids:
+                    clear_default_ids = (
+                        destination_default_ids[1:] + source_default_ids
+                    )
+                else:
+                    clear_default_ids = source_default_ids[1:]
+                if clear_default_ids:
+                    (
+                        db.query(EmailAccount)
+                        .filter(EmailAccount.id.in_(clear_default_ids))
+                        .update(
+                            {EmailAccount.is_default: False},
+                            synchronize_session=False,
+                        )
+                    )
+
                 for mapper in Base.registry.mappers:
                     model = mapper.class_
                     if not hasattr(model, "owner"):
@@ -637,7 +718,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         a scrubbed copy with secret keys blanked. The frontend uses this
         for keybinds + TTS prefs, so it stays callable without admin."""
         user = _get_current_user(request)
-        settings = _load_settings()
+        settings = without_retired_settings(_load_settings())
         if user and auth_manager.is_admin(user):
             return settings
         return scrub_settings(settings)
@@ -655,8 +736,11 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         _INT_RANGES = {
             "agent_max_rounds": (1, 200),
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
+            "auto_compact_threshold_percent": (50, 95),
         }
         for key in DEFAULT_SETTINGS:
+            if key in RETIRED_SETTING_KEYS:
+                continue
             if key not in body:
                 continue
             val = body[key]
@@ -669,7 +753,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                 val = max(lo, min(val, hi))
             current[key] = val
         _save_settings(current)
-        return current
+        return without_retired_settings(current)
 
     # ---- Integrations CRUD ----
 

routes/backup_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime
 
 from fastapi import APIRouter, HTTPException, Request, Response
 from core.middleware import require_admin
+from services.memory import MemoryStoreUnreadable
 from src.auth_helpers import get_current_user
 from src.settings import load_settings, save_settings, load_features, save_features
 
@@ -76,7 +77,15 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
 
         # ── Memories ──
         if "memories" in body and isinstance(body["memories"], list):
-            existing = memory_manager.load_all()
+            # Strict load: importing on top of an unreadable store would write
+            # only the incoming rows back and drop everything already saved.
+            try:
+                existing = memory_manager.load_all_for_update()
+            except MemoryStoreUnreadable as e:
+                logger.error("Refusing to import memories: %s", e)
+                raise HTTPException(
+                    503, "Memory store is temporarily unreadable — nothing was imported."
+                )
             # Dedup against THIS user's own memories only. Using every tenant's
             # rows (load_all) meant a memory whose text matched any other
             # user's was silently skipped, so the importing user lost their own

routes/calendar_routes.py

@@ -4,15 +4,16 @@ import logging
 import json
 import re
 import uuid
-from datetime import datetime, date, timedelta
+from datetime import datetime, date, timedelta, timezone
 from typing import Optional, List
 
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
 from pydantic import BaseModel
 from sqlalchemy import or_, and_
+from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
-from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
+from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent, Note
 from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
 from src.upload_handler import reserve_upload_references
@@ -206,6 +207,7 @@ class EventCreate(BaseModel):
     calendar_href: Optional[str] = None  # calendar id
     rrule: Optional[str] = None
     color: Optional[str] = None  # per-event color override
+    reminder_minutes: Optional[int] = None
 
 
 class EventUpdate(BaseModel):
@@ -217,26 +219,130 @@ class EventUpdate(BaseModel):
     location: Optional[str] = None
     rrule: Optional[str] = None
     color: Optional[str] = None
+    reminder_minutes: Optional[int] = None
 
 
 # ── Helpers ──
 
+_DEFAULT_CALENDAR_NAMESPACE = uuid.UUID("4840613a-9847-4a3b-bd75-19e6bc5fc3ce")
+
+
+def _default_calendar_id(owner: str, collision_index: int = 0) -> str:
+    """Return one stable primary-key candidate for an owner's lazy default.
+
+    Slot zero preserves the original owner-derived identifier.  Later slots
+    let a username be reused after its prior calendar was migrated to another
+    owner during a rename, without making concurrent first use choose random
+    and therefore divergent identifiers.
+    """
+    if collision_index == 0:
+        candidate_name = owner
+    else:
+        candidate_name = json.dumps(
+            [owner, collision_index],
+            ensure_ascii=False,
+            separators=(",", ":"),
+        )
+    return str(uuid.uuid5(_DEFAULT_CALENDAR_NAMESPACE, candidate_name))
+
+
+def _begin_sqlite_default_write(db) -> None:
+    """Serialize an absent-default check with other SQLite writers.
+
+    SQLite's default deferred transactions allow two workers to both read an
+    empty calendar set before either writes.  ``BEGIN IMMEDIATE`` acquires the
+    writer reservation before the second, authoritative lookup.  We issue it
+    only when the driver has not already opened a write transaction; a caller
+    with a pending write already owns the required reservation.
+    """
+    connection = db.connection()
+    dbapi_connection = connection.connection
+    driver_connection = getattr(
+        dbapi_connection,
+        "driver_connection",
+        dbapi_connection,
+    )
+    if not getattr(driver_connection, "in_transaction", False):
+        connection.exec_driver_sql("BEGIN IMMEDIATE")
+
+
 def _ensure_default_calendar(db, owner: str = None) -> CalendarCal:
-    """Create default calendar if none exist for this owner."""
+    """Return the owner's calendar, staging a default in the caller's transaction.
+
+    A stable owner-derived primary key makes concurrent first-use inserts
+    converge on one row on every SQL backend.  SQLite additionally serializes
+    the absent-row check because its deferred transactions otherwise permit
+    both workers to read the gap before either writes.  Other backends recover
+    a lost insert race inside a savepoint so the caller's event transaction
+    remains usable and atomic.
+    """
     owner = owner or FALLBACK_OWNER
     cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
-    if not cal:
+    if cal:
+        return cal
+
+    dialect = db.get_bind().dialect.name
+    if dialect == "sqlite":
+        _begin_sqlite_default_write(db)
+        # Another worker may have committed while BEGIN IMMEDIATE waited.
+        cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
+        if cal:
+            return cal
+
+    collision_index = 0
+    while True:
+        default_id = _default_calendar_id(owner, collision_index)
+
+        if dialect == "sqlite":
+            # BEGIN IMMEDIATE above makes this occupancy check authoritative:
+            # another SQLite writer cannot rename, delete, or claim this slot
+            # until the caller commits or rolls back.
+            occupant = db.query(CalendarCal).filter(
+                CalendarCal.id == default_id,
+            ).first()
+            if occupant is not None:
+                if occupant.owner == owner:
+                    return occupant
+                collision_index += 1
+                continue
+
         cal = CalendarCal(
-            id=str(uuid.uuid4()),
+            id=default_id,
             owner=owner,
             name="Personal",
             color="#5b8abf",
             source="local",
         )
-        db.add(cal)
-        db.commit()
-        db.refresh(cal)
-    return cal
+
+        if dialect == "sqlite":
+            db.add(cal)
+            db.flush()
+            return cal
+
+        try:
+            # A uniqueness failure rolls back only this savepoint, not an event
+            # or reminder already staged by the caller's outer transaction.
+            with db.begin_nested():
+                db.add(cal)
+                db.flush()
+            return cal
+        except IntegrityError:
+            # Use a locking/current read so repeatable-read backends can observe
+            # the row that won after our transaction's original empty snapshot.
+            occupant = db.query(CalendarCal).filter(
+                CalendarCal.id == default_id,
+            ).with_for_update().first()
+            if occupant is None:
+                # Do not misclassify an unrelated integrity failure as an ID
+                # collision and loop forever. A concurrently deleted winner is
+                # safe for the caller to retry as a fresh transaction.
+                raise
+            if occupant.owner == owner:
+                return occupant
+            # A renamed calendar owns this deterministic slot. Advance to the
+            # next stable slot; concurrent callers for this owner will still
+            # converge there.
+            collision_index += 1
 
 
 # Per-request user time context. chat_routes sets this from browser timezone
@@ -517,7 +623,133 @@ def _parse_dt(s: str) -> datetime:
         raise ValueError(f"could not parse datetime: {s!r}")
 
 
-def _event_to_dict(ev: CalendarEvent) -> dict:
+def _note_due_datetime(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        text = str(value).strip()
+        if text.endswith("Z"):
+            text = text[:-1] + "+00:00"
+        due = datetime.fromisoformat(text)
+        if due.tzinfo is not None:
+            return due.astimezone(timezone.utc).replace(tzinfo=None)
+        return due
+    except Exception:
+        return None
+
+
+def _calendar_reminder_for_event(db, owner: str, ev: CalendarEvent) -> dict | None:
+    """Return the closest Notes reminder that belongs to this calendar event.
+
+    Calendar alarms are currently stored as Notes rows. Older rows do not carry
+    an event UID, so match conservatively by the generated title plus due_date
+    before the event start. This keeps existing reminder notes visible on the
+    calendar without a schema migration.
+    """
+    if not db or not owner or not ev or not ev.dtstart:
+        return None
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return None
+
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    if not notes:
+        return None
+
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    best = None
+    best_minutes = None
+    for note in notes:
+        due = _note_due_datetime(note.due_date)
+        if due is None:
+            continue
+        minutes = round((start - due).total_seconds() / 60)
+        if minutes < 0 or minutes > 7 * 24 * 60:
+            continue
+        if best is None or minutes < best_minutes:
+            best = note
+            best_minutes = minutes
+    if best is None:
+        return None
+    return {
+        "note_id": best.id,
+        "due_date": best.due_date,
+        "minutes": best_minutes,
+    }
+
+
+def _delete_calendar_reminders_for_event(db, owner: str, ev: CalendarEvent) -> int:
+    if not db or not owner or not ev:
+        return 0
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return 0
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    for note in notes:
+        db.delete(note)
+    return len(notes)
+
+
+def _create_calendar_reminder_for_event(db, owner: str, ev: CalendarEvent, minutes_before: int) -> dict:
+    if not owner or not ev or not ev.dtstart:
+        return {"note_id": None, "skipped_reason": "missing event"}
+    minutes_before = max(0, int(minutes_before))
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    remind_at = start - timedelta(minutes=minutes_before)
+    now = datetime.utcnow() if getattr(ev, "is_utc", False) else datetime.now()
+    if start <= now:
+        return {"note_id": None, "skipped_reason": "event already passed"}
+    if remind_at <= now:
+        remind_at = now
+
+    summary = (ev.summary or "(no title)").strip() or "(no title)"
+    location = (ev.location or "").strip()
+    start_fmt = start.strftime("%a %b %d") if ev.all_day else start.strftime("%a %b %d %H:%M")
+    loc = f" @ {location}" if location else ""
+    due_date = remind_at.isoformat() + ("Z" if getattr(ev, "is_utc", False) and not ev.all_day else "")
+    note = Note(
+        id=str(uuid.uuid4()),
+        owner=owner,
+        title=f"Calendar reminder: {summary}",
+        items=json.dumps([{"text": f"{summary}{loc} — {start_fmt}", "done": False, "checked": False}]),
+        note_type="todo",
+        label="calendar",
+        due_date=due_date,
+        source="calendar",
+    )
+    db.add(note)
+    return {"note_id": note.id, "due_date": due_date, "minutes": minutes_before, "skipped_reason": None}
+
+
+def _event_to_dict(ev: CalendarEvent, db=None, owner: str | None = None) -> dict:
     """Convert a CalendarEvent model to the API dict format.
 
     Timed events whose stored datetimes represent UTC (is_utc=True) are
@@ -533,6 +765,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         suffix = "Z" if getattr(ev, "is_utc", False) else ""
         start_str = ev.dtstart.isoformat() + suffix
         end_str = ev.dtend.isoformat() + suffix
+    reminder = _calendar_reminder_for_event(db, owner, ev) if db and owner else None
     return {
         "uid": ev.uid,
         "summary": ev.summary or "",
@@ -549,6 +782,10 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
         "event_type": getattr(ev, "event_type", None),
         "importance": getattr(ev, "importance", None) or "normal",
+        "has_reminder": bool(reminder),
+        "reminder_note_id": reminder["note_id"] if reminder else None,
+        "reminder_due_date": reminder["due_date"] if reminder else None,
+        "reminder_minutes": reminder["minutes"] if reminder else None,
     }
 
 
@@ -580,7 +817,7 @@ def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
 
 
 def _expand_rrule(
-    ev: CalendarEvent, start: datetime, end: datetime
+    ev: CalendarEvent, start: datetime, end: datetime, db=None, owner: str | None = None
 ) -> List[dict]:
     """Expand a single recurring CalendarEvent into occurrence dicts.
 
@@ -598,7 +835,7 @@ def _expand_rrule(
         # Non-recurring — return the base event as-is. list_events
         # already filters non-recurring rows with the overlap check
         # in SQL, so we don't re-check here.
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -624,7 +861,7 @@ def _expand_rrule(
         logger.warning(
             "Failed to parse rrule=%r for event %s: %s", ev.rrule, ev.uid, ex
         )
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -642,7 +879,7 @@ def _expand_rrule(
     expand_start = start - duration
     results = []
     truncated = False
-    base = _event_to_dict(ev)
+    base = _event_to_dict(ev, db=db, owner=owner)
     exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
@@ -1015,6 +1252,9 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         db = SessionLocal()
         try:
             _ensure_default_calendar(db, owner)
+            # Listing calendars intentionally lazily creates a durable default.
+            # Other callers commit it with the event they are creating.
+            db.commit()
             cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all()
             return {"calendars": [
                 {"name": c.name, "href": c.id, "color": c.color, "source": c.source}
@@ -1023,6 +1263,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         except HTTPException:
             raise
         except Exception as e:
+            db.rollback()
             logger.error("Failed to list calendars: %s", e)
             raise HTTPException(500, "Failed to list calendars")
         finally:
@@ -1077,7 +1318,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             # Expand recurring events into individual occurrences.
             expanded = []
             for e in events:
-                expanded.extend(_expand_rrule(e, start_dt, end_dt))
+                expanded.extend(_expand_rrule(e, start_dt, end_dt, db=db, owner=owner))
 
             # Sort by occurrence start time for consistent frontend ordering.
             truncated = any(e.get("truncated") for e in expanded)
@@ -1143,10 +1384,19 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 caldav_sync_pending="create" if cal.source == "caldav" else None,
             )
             db.add(ev)
+            reminder = None
+            if data.reminder_minutes is not None:
+                reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             db.commit()
+            db.refresh(ev)
             if cal.source == "caldav":
                 await _push_caldav_event_after_commit(owner, uid, "create")
-            return {"ok": True, "uid": uid}
+            return {
+                "ok": True,
+                "uid": uid,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1156,6 +1406,17 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    @router.get("/events/{uid}")
+    async def get_event(request: Request, uid: str):
+        owner = _require_user(request)
+        db = SessionLocal()
+        try:
+            base_uid = _resolve_base_uid(uid)
+            ev = _get_or_404_event(db, base_uid, owner)
+            return {"event": _event_to_dict(ev, db=db, owner=owner)}
+        finally:
+            db.close()
+
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
@@ -1192,13 +1453,24 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 ev.rrule = data.rrule
             if data.color is not None:
                 ev.color = data.color if data.color else None
+            reminder = None
+            reminder_fields = getattr(data, "model_fields_set", getattr(data, "__fields_set__", set()))
+            if "reminder_minutes" in reminder_fields:
+                _delete_calendar_reminders_for_event(db, owner, ev)
+                if data.reminder_minutes is not None:
+                    reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"
             db.commit()
+            db.refresh(ev)
             if is_caldav:
                 await _push_caldav_event_after_commit(owner, base_uid, "update")
-            return {"ok": True}
+            return {
+                "ok": True,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1220,6 +1492,8 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             ev = _get_or_404_event(db, base_uid, owner)
             is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if scope in {"occurrence", "instance"} and not is_occurrence_delete:
+                raise HTTPException(400, "Occurrence delete requires a recurring occurrence uid")
             if is_occurrence_delete:
                 key = _occurrence_exdate_key(uid, ev)
                 if not key:
@@ -1236,6 +1510,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
+            _delete_calendar_reminders_for_event(db, owner, ev)
             db.delete(ev)
             db.commit()
             if is_caldav:
@@ -1315,7 +1590,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 raise HTTPException(400, f"Invalid ICS file: {e}")
 
             # Sanitize display name — length cap + strip control chars
-            raw_name = calendar_name.strip() or (file.filename or "").replace(".ics", "").replace("_", " ").strip() or "Imported"
+            raw_name = calendar_name.strip() or re.sub(r"\.(?:calendar|ics|ical)$", "", file.filename or "", flags=re.IGNORECASE).replace("_", " ").strip() or "Imported"
             cal_display = "".join(c for c in raw_name if c.isprintable())[:120] or "Imported"
 
             target_cal = db.query(CalendarCal).filter(

routes/chat_helpers.py

@@ -3,6 +3,7 @@
 import asyncio
 import json
 import logging
+import math
 import os
 import re
 import time
@@ -15,7 +16,7 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
-from src.model_context import estimate_tokens
+from src.model_context import estimate_tokens, get_context_length
 from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
 from src.attachment_refs import attachment_ref
@@ -25,6 +26,56 @@ from fastapi import HTTPException
 
 logger = logging.getLogger(__name__)
 
+_INVISIBLE_RESPONSE_CHARS = "\u2063\u200b\u200c\u200d\ufeff"
+
+
+def _skill_run_is_complex(agent_rounds: int, agent_tool_calls: int) -> bool:
+    """Keep one-off TUI edit loops out of automatic skill extraction."""
+    return agent_tool_calls >= 4 or (agent_rounds >= 5 and agent_tool_calls >= 3)
+
+
+def clean_repeated_assistant_content(text: object) -> str:
+    """Collapse repeated terminal assistant prose before history/SFT storage."""
+    value = str(text or "")
+    for char in _INVISIBLE_RESPONSE_CHARS:
+        value = value.replace(char, "")
+    value = value.strip()
+    if not value:
+        return ""
+
+    # Stream rejoin/finalization races can concatenate the same complete
+    # answer without separators. Collapse only exact 2-4x repetitions.
+    for copies in range(4, 1, -1):
+        if len(value) % copies == 0:
+            width = len(value) // copies
+            unit = value[:width]
+            if unit and unit * copies == value:
+                value = unit.strip()
+                break
+
+    # Interrupted/rejoined streams can leave a short suffix before a closing
+    # think tag at the edge of visible prose, e.g. "ls.\n</think>\n\nHere's...".
+    edge_close_re = re.compile(r"(?is)^\s*(?!<\s*think\b)[^<\n]{0,120}\s*</\s*think\s*>\s*")
+    while True:
+        cleaned = edge_close_re.sub("", value, count=1).strip()
+        if cleaned == value:
+            break
+        value = cleaned
+
+    first_line = next((line.strip() for line in value.splitlines() if line.strip()), "")
+    if 8 <= len(first_line) <= 180:
+        matches = list(re.finditer(r"(?m)^" + re.escape(first_line) + r"\s*$", value))
+        if len(matches) >= 2:
+            value = value[matches[0].start():matches[1].start()].strip()
+
+    value = re.sub(
+        r"(?is)(?<=[.!?])(?:[a-z]{1,12}\.)\s*</\s*think\s*>\s*$",
+        "",
+        value,
+    ).strip()
+    value = re.sub(r"(?is)\s*</\s*think\s*>\s*$", "", value).strip()
+    return value
+
 _CASUAL_OPENING_RE = re.compile(
     r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
     r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
@@ -36,6 +87,14 @@ _CASUAL_BLOCKLIST_RE = re.compile(
     r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
     re.IGNORECASE,
 )
+_PERSONAL_TOOL_CONTEXT_RE = re.compile(
+    r"\b(?:"
+    r"email|emails|mail|inbox|gmail|"
+    r"calendar|events?|meetings?|appointments?|schedule|"
+    r"notes?|todo|checklist|reminders?|tasks?"
+    r")\b",
+    re.IGNORECASE,
+)
 
 
 def _is_casual_low_signal(text: str) -> bool:
@@ -51,6 +110,14 @@ def _is_casual_low_signal(text: str) -> bool:
     return len(tail_words) <= 2
 
 
+def _truthy_request_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
 # Strong references to in-flight fire-and-forget tasks scheduled from this
 # module. asyncio only keeps weak references to tasks created via
 # create_task, so without this the GC can collect a task mid-execution and
@@ -60,6 +127,197 @@ _BG_TASKS: set[asyncio.Task] = set()
 _INCOGNITO_CONTEXTS: dict[str, dict[str, Any]] = {}
 _INCOGNITO_CONTEXT_TTL_SECONDS = 6 * 60 * 60
 _INCOGNITO_CONTEXT_MAX_MESSAGES = 80
+_SFT_TRACE_CAPTURE_ENV = "ODYSSEUS_SFT_TRACE_CAPTURE"
+_SFT_TRACE_DIR_ENV = "ODYSSEUS_SFT_TRACE_DIR"
+_RUNTIME_REVISION_ENV = "ODYSSEUS_RUNTIME_REVISION"
+
+
+def _sft_trace_capture_enabled(owner: str | None) -> bool:
+    flag = os.getenv(_SFT_TRACE_CAPTURE_ENV, "1").strip().lower()
+    return flag not in {"0", "false", "no", "off"} and str(owner or "").startswith("sft_")
+
+
+def _json_safe(value: Any) -> Any:
+    try:
+        json.dumps(value)
+        return value
+    except TypeError:
+        return str(value)
+
+
+def _last_user_message_for_trace(sess) -> str:
+    for msg in reversed(getattr(sess, "history", []) or []):
+        if getattr(msg, "role", None) == "user":
+            return str(getattr(msg, "content", "") or "").strip()
+    return ""
+
+
+def _append_sft_trace_record(
+    *,
+    owner: str | None,
+    session_id: str,
+    sess,
+    assistant_content: str,
+    metadata: dict,
+    message_id: Any = None,
+) -> None:
+    """Append one training-ready trace record for synthetic SFT users."""
+    if not _sft_trace_capture_enabled(owner):
+        return
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        os.makedirs(trace_dir, exist_ok=True)
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        runtime_revision = os.getenv(_RUNTIME_REVISION_ENV, "").strip()
+        record = {
+            "format": "odysseus_sft_trace_turn_v1",
+            "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "owner": owner,
+            "session_id": session_id,
+            "session_name": getattr(sess, "name", "") or "",
+            "message_id": message_id,
+            "user": _last_user_message_for_trace(sess),
+            "assistant": str(assistant_content or "").strip(),
+            "thinking": str((metadata or {}).get("thinking") or "").strip(),
+            "tool_events": _json_safe((metadata or {}).get("tool_events") or []),
+            "round_texts": _json_safe((metadata or {}).get("round_texts") or []),
+            "runtime_revision": runtime_revision,
+            "metadata": {
+                "model": (metadata or {}).get("model"),
+                "requested_model": (metadata or {}).get("requested_model"),
+                "endpoint_label": (metadata or {}).get("endpoint_label"),
+                "endpoint_id": (metadata or {}).get("endpoint_id"),
+                "response_time": (metadata or {}).get("response_time"),
+                "input_tokens": (metadata or {}).get("input_tokens"),
+                "output_tokens": (metadata or {}).get("output_tokens"),
+                "usage_buckets": _json_safe((metadata or {}).get("usage_buckets") or []),
+                "runtime_revision": runtime_revision,
+            },
+        }
+        _prune_sft_retry_rows_before_append(path, record)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception as exc:
+        logger.warning("Failed to append SFT trace record for %s/%s: %s", owner, session_id, exc)
+
+
+def remove_session_sft_trace_rows(owner: str | None, session_id: str) -> int:
+    """Remove every captured training row for a deleted synthetic session."""
+    if not _sft_trace_capture_enabled(owner) or not str(session_id or "").strip():
+        return 0
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        if not os.path.exists(path):
+            return 0
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as source:
+            for line in source:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if str(row.get("session_id") or "") != session_id:
+                    kept.append(raw)
+                    continue
+                row["deleted_from_training"] = True
+                removed.append(json.dumps(row, ensure_ascii=False))
+        if not removed:
+            return 0
+        tmp_path = f"{path}.{os.getpid()}.{time.time_ns()}.tmp"
+        with open(tmp_path, "w", encoding="utf-8") as target:
+            for raw in kept:
+                target.write(raw + "\n")
+        os.replace(tmp_path, path)
+        with open(path + ".trash", "a", encoding="utf-8") as trash:
+            for raw in removed:
+                trash.write(raw + "\n")
+        logger.info("Removed %d SFT trace row(s) for deleted session %s", len(removed), session_id)
+        return len(removed)
+    except Exception as exc:
+        logger.warning("Failed to remove SFT trace rows for session %s: %s", session_id, exc)
+        return 0
+
+
+def _prune_sft_retry_rows_before_append(path: str, record: dict[str, Any]) -> None:
+    """For SFT traces, keep only the latest retry for a repeated user send.
+
+    The browser resend flow can append a second identical user turn without
+    first calling the delete endpoint. Training wants the final attempt, not
+    both sends, so remove prior trailing rows in the same session with the same
+    user prompt before appending the replacement.
+    """
+    current_session = str(record.get("session_id") or "")
+    current_user = str(record.get("user") or "").strip()
+    if not current_session or not current_user or not os.path.exists(path):
+        return
+    kept: list[str] = []
+    parsed: list[tuple[str, dict | None]] = []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    parsed.append((raw, json.loads(raw)))
+                except json.JSONDecodeError:
+                    parsed.append((raw, None))
+
+        last_different_same_session = -1
+        for idx, (_raw, row) in enumerate(parsed):
+            if not isinstance(row, dict) or row.get("session_id") != current_session:
+                continue
+            if str(row.get("user") or "").strip() != current_user:
+                last_different_same_session = idx
+
+        removed: list[str] = []
+        for idx, (raw, row) in enumerate(parsed):
+            should_remove = (
+                idx > last_different_same_session
+                and isinstance(row, dict)
+                and row.get("session_id") == current_session
+                and str(row.get("user") or "").strip() == current_user
+            )
+            if should_remove:
+                tombstone = dict(row)
+                tombstone["deleted_from_training"] = True
+                tombstone["delete_reason"] = "sft_retry_replaced"
+                removed.append(json.dumps(tombstone, ensure_ascii=False))
+            else:
+                kept.append(raw)
+
+        if not removed:
+            return
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        with open(path + ".trash", "a", encoding="utf-8") as f:
+            for raw in removed:
+                f.write(raw + "\n")
+        logger.info(
+            "Removed %d prior SFT retry row(s) before appending replacement for session %s",
+            len(removed),
+            current_session,
+        )
+    except Exception as exc:
+        logger.warning("Failed to prune prior SFT retry rows for %s: %s", current_session, exc)
+
+
+def strip_tui_local_context(content: Any) -> Any:
+    """Remove client-only workspace metadata before persistence/display."""
+    if not isinstance(content, str):
+        return content
+    return re.sub(r"\s*<local_context\b[^>]*>.*?</local_context>\s*", "", content, flags=re.IGNORECASE | re.DOTALL).strip()
 
 
 def _spawn_bg(coro) -> asyncio.Task:
@@ -113,6 +371,8 @@ class PresetInfo:
     max_tokens: Optional[int]
     system_prompt: Optional[str]
     character_name: Optional[str]
+    persona_memory: Optional[str] = None
+    persona_memory_schema: str = "general"
 
 
 @dataclass
@@ -152,10 +412,38 @@ class ChatContext:
     # Uploads attached to this user turn, resolved and owner-checked for the
     # agent's private context. This is not emitted to the browser.
     uploaded_files: list = field(default_factory=list)
+    # Route-neutral prompt before any model-window compaction/trimming. This is
+    # retained only when explicit foreground fallbacks are enabled so each
+    # concrete candidate can apply its own context budget independently.
+    route_messages: list = field(default_factory=list)
 
 
 # ── Helpers ────────────────────────────────────────────────────────────── #
 
+def _allowed_models_from_privileges(privs: dict) -> Optional[frozenset[str]]:
+    if privs.get("block_all_models"):
+        return frozenset()
+    allowed_raw = privs.get("allowed_models")
+    allowed = allowed_raw if isinstance(allowed_raw, list) else []
+    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
+    return frozenset(model for model in allowed if isinstance(model, str)) if restricted else None
+
+
+def _allowed_models_for_request(request) -> Optional[frozenset[str]]:
+    """Return the caller's model allowlist, or ``None`` when unrestricted."""
+
+    try:
+        user = effective_user(request)
+    except Exception:
+        user = None
+    if not user:
+        return None
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not auth_manager:
+        return None
+    privs = auth_manager.get_privileges(user) or {}
+    return _allowed_models_from_privileges(privs)
+
 def _enforce_chat_privileges(request, sess) -> None:
     """Apply the per-user privilege gates (allowed_models + max_messages_per_day)
     that both /api/chat and /api/chat_stream must enforce BEFORE any LLM work.
@@ -185,10 +473,8 @@ def _enforce_chat_privileges(request, sess) -> None:
     if privs.get("block_all_models"):
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
-    allowed_raw = privs.get("allowed_models")
-    allowed = allowed_raw if isinstance(allowed_raw, list) else []
-    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
-    if restricted and sess.model and sess.model not in allowed:
+    allowed_models = _allowed_models_from_privileges(privs)
+    if allowed_models is not None and sess.model and sess.model not in allowed_models:
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
     cap = int(privs.get("max_messages_per_day") or 0)
@@ -225,6 +511,14 @@ def needs_auto_name(name: str) -> bool:
     return False
 
 
+def fallback_session_title(text: str, *, max_words: int = 6) -> str:
+    words = re.findall(r"[A-Za-z0-9@._'-]+", text)
+    if not words:
+        return "New chat"
+    title = " ".join(words[:max_words]).strip()
+    return title[:60] or "New chat"
+
+
 async def auto_name_session(session_manager, sess):
     """Generate a short title for a session from its first user message."""
     try:
@@ -247,6 +541,17 @@ async def auto_name_session(session_manager, sess):
         if not first_msg:
             return
 
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model_name = str(getattr(sess, "model", "") or "")
+        if (
+            "ttft" in model_name.lower()
+            or re.search(r":18\d{3}\b", endpoint_url)
+        ):
+            title = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, title)
+            logger.info(f"Auto-named session {sess.id} deterministically: {title}")
+            return
+
         owner = getattr(sess, "owner", None)
         t_url, t_model, t_headers = resolve_task_endpoint(
             sess.endpoint_url, sess.model, sess.headers, owner=owner
@@ -268,9 +573,9 @@ async def auto_name_session(session_manager, sess):
                 {"role": "user", "content": first_msg},
             ],
             temperature=0.3,
-            max_tokens=4096,
+            max_tokens=64,
             headers=t_headers,
-            timeout=60,
+            timeout=15,
         )
 
         title = title.strip().strip('"\'').strip()
@@ -278,108 +583,47 @@ async def auto_name_session(session_manager, sess):
         # via the central helper.
         from src.text_helpers import strip_think
         title = strip_think(title, prose=False, prompt_echo=False)
-        if title and len(title) < 80:
-            session_manager.update_session_name(sess.id, title)
-            logger.info(f"Auto-named session {sess.id}: {title}")
+        if not title or len(title) >= 80 or "\n" in title:
+            fallback = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, fallback)
+            logger.info(
+                "Auto-named session %s with fallback title after unusable model title: %s",
+                sess.id,
+                fallback,
+            )
+            return
+
+        session_manager.update_session_name(sess.id, title)
+        logger.info(f"Auto-named session {sess.id}: {title}")
 
     except Exception as e:
         import traceback
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
-def try_fallback_endpoint(sess, session_id: str) -> dict | None:
-    """Find an alternative working endpoint when the current one fails.
-
-    Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None.
-    """
-    import requests as _req
-    from src.endpoint_resolver import (
-        build_chat_url,
-        build_headers,
-        build_models_url,
-        normalize_base,
-        resolve_endpoint_runtime,
-    )
-    from src.chatgpt_subscription import is_chatgpt_subscription_base
-
-    current_url = sess.endpoint_url or ""
-    owner = getattr(sess, "owner", None)
-    db = SessionLocal()
+async def auto_name_session_after_stream(session_id: str, session_manager, sess):
+    """Delay chat title generation until the first response stream is settled."""
     try:
-        q = db.query(ModelEndpoint).filter(
-            ModelEndpoint.is_enabled == True
-        )
-        if owner:
-            from src.auth_helpers import owner_filter
-            q = owner_filter(q, ModelEndpoint, owner)
-        endpoints = q.all()
-    finally:
-        db.close()
-
-    for ep in endpoints:
-        base = normalize_base(ep.base_url)
-        # Skip current endpoint
-        if current_url and base in current_url:
-            continue
+        waited = 0.0
+        while _is_session_stream_active(session_id) and waited < 30.0:
+            await asyncio.sleep(0.25)
+            waited += 0.25
+        # Let the final SSE chunk/message_saved bookkeeping clear before any
+        # title model call can contend with the user's visible response.
+        await asyncio.sleep(0.5)
         try:
-            base, api_key = resolve_endpoint_runtime(ep, owner=owner)
-        except Exception:
-            continue
-        ping_url = build_models_url(base)
-        headers = build_headers(api_key, base)
-        try:
-            if ping_url:
-                r = _req.get(ping_url, headers=headers, timeout=5)
-                r.raise_for_status()
-                data = r.json()
-                models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
-                if not models:
-                    models = [
-                        m.get("name") or m.get("model")
-                        for m in (data.get("models") or [])
-                        if m.get("name") or m.get("model")
-                    ]
-            else:
-                models = json.loads(ep.cached_models or "[]")
-            if not models:
-                continue
-            # Found a working endpoint — update session
-            new_model = models[0]
-            chat_url = build_chat_url(base)
-            new_headers = build_headers(api_key, base)
-            persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers
-
-            sess.model = new_model
-            sess.endpoint_url = chat_url
-            sess.headers = new_headers
-
-            # Persist
-            _db = SessionLocal()
-            try:
-                _db.query(DBSession).filter(DBSession.id == session_id).update({
-                    "model": new_model,
-                    "endpoint_url": chat_url,
-                    "headers": persisted_headers,
-                })
-                _db.commit()
-            finally:
-                _db.close()
-
-            logger.info(f"Fallback: switched session {session_id} from {current_url} to {ep.name} ({new_model})")
-            return {
-                "model": new_model,
-                "endpoint_url": chat_url,
-                "endpoint_name": ep.name,
-            }
-        except Exception:
-            continue
-
-    return None
+            sess = session_manager.get_session(session_id)
+        except Exception as e:
+            logger.warning("[auto-name] Could not reload session %s before naming: %s", session_id, e)
+        await auto_name_session(session_manager, sess)
+    except Exception as e:
+        import traceback
+        logger.error(f"Deferred auto-name failed for {session_id}: {e}\n{traceback.format_exc()}")
 
 
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
-    temperature, max_tokens, system_prompt, char_name = (
+    temperature, max_tokens, system_prompt, char_name, persona_memory, persona_memory_schema = (
         chat_handler.validate_and_extract_preset(preset_id)
     )
     return PresetInfo(
@@ -387,6 +631,8 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo:
         max_tokens=max_tokens,
         system_prompt=system_prompt,
         character_name=char_name,
+        persona_memory=persona_memory,
+        persona_memory_schema=persona_memory_schema,
     )
 
 
@@ -470,14 +716,28 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
     return manifest
 
 
-def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
+def add_user_message(
+    sess,
+    chat_handler,
+    preprocessed: PreprocessedMessage,
+    incognito: bool = False,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
+):
     """Add user message to session history and update session name.
     Incognito messages must not mutate persistent session history, even in
     memory, because a later normal turn can persist the same session object."""
     if incognito:
         return
-    user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
-    sess.add_message(ChatMessage("user", preprocessed.user_content, metadata=user_meta))
+    user_meta = {}
+    if preprocessed.attachment_meta:
+        user_meta["attachments"] = preprocessed.attachment_meta
+    if interaction_mode in {"chat", "agent", "research"}:
+        user_meta["interaction_mode"] = interaction_mode
+    if auto_escalated:
+        user_meta["auto_escalated"] = True
+    clean_content = strip_tui_local_context(preprocessed.user_content)
+    sess.add_message(ChatMessage("user", clean_content, metadata=user_meta or None))
     chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
 
 
@@ -687,6 +947,11 @@ async def build_chat_context(
     use_enhanced_message: bool = False,
     agent_mode: bool = False,
     allow_tool_preprocessing: bool = True,
+    defer_context_shaping: bool = False,
+    continuation_context_message: str | None = None,
+    persist_user_message: bool = True,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -710,14 +975,27 @@ async def build_chat_context(
     # Add user message to history. Nobody/incognito uses a request-local
     # transcript store instead of session history so stale saved chats cannot
     # bleed into context and the turn is not persisted.
-    if incognito:
-        user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
+    if persist_user_message and incognito:
+        user_meta = {}
+        if preprocessed.attachment_meta:
+            user_meta["attachments"] = preprocessed.attachment_meta
+        if interaction_mode in {"chat", "agent", "research"}:
+            user_meta["interaction_mode"] = interaction_mode
+        if auto_escalated:
+            user_meta["auto_escalated"] = True
         _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
-    else:
-        add_user_message(sess, chat_handler, preprocessed, incognito=False)
+    elif persist_user_message:
+        add_user_message(
+            sess,
+            chat_handler,
+            preprocessed,
+            incognito=False,
+            interaction_mode=interaction_mode,
+            auto_escalated=auto_escalated,
+        )
 
     # Fire events
-    if not incognito:
+    if persist_user_message and not incognito:
         fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
 
     # Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
@@ -729,13 +1007,22 @@ async def build_chat_context(
         getattr(chat_handler, "upload_handler", None),
         getattr(sess, "owner", None),
     )
-    casual_low_signal = _is_casual_low_signal(message)
+    context_message = (
+        str(continuation_context_message).strip()
+        if continuation_context_message
+        else message
+    )
+    casual_low_signal = _is_casual_low_signal(context_message)
 
     # Memory enabled?
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
     # Skills injection respects its own enable toggle (mirrors memory_enabled).
     # When off, the "Available skills" index is not added to the prompt.
-    skills_enabled = not incognito and uprefs.get("skills_enabled", True)
+    skills_enabled = (
+        not incognito
+        and uprefs.get("skills_enabled", True)
+        and getattr(sess, "skill_injection_enabled", True) is not False
+    )
     if not allow_tool_preprocessing:
         mem_enabled = False
         skills_enabled = False
@@ -760,22 +1047,40 @@ async def build_chat_context(
     if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
         use_rag_val = False
 
-    # If pre-fetched search context was provided (compare mode), skip live web search
-    skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
+    use_web_val = _truthy_request_flag(use_web)
+    # If pre-fetched search context was provided (compare mode), skip live web
+    # search. Personal app requests should be served by their tools; pre-search
+    # here caused calendar/email turns with use_web="false" to run irrelevant
+    # web searches before the agent even saw the tool surface.
+    skip_web = (
+        bool(search_context)
+        or not allow_tool_preprocessing
+        or casual_low_signal
+        or bool(agent_mode and _PERSONAL_TOOL_CONTEXT_RE.search(context_message or ""))
+    )
 
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
     # the sync path uses text_for_context.
-    _ctx_msg = preprocessed.enhanced_message if use_enhanced_message else preprocessed.text_for_context
+    _ctx_msg = (
+        context_message
+        if continuation_context_message
+        else (
+            preprocessed.enhanced_message
+            if use_enhanced_message
+            else preprocessed.text_for_context
+        )
+    )
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
-        use_web=use_web and not skip_web,
+        use_web=use_web_val and not skip_web,
         use_memory=mem_enabled,
         time_filter=time_filter,
         preset_system_prompt=preset.system_prompt,
         owner=user,
         character_name=preset.character_name,
+        persona_memory=preset.persona_memory,
         agent_mode=agent_mode,
         incognito=incognito,
         use_skills=skills_enabled,
@@ -830,13 +1135,22 @@ async def build_chat_context(
         except Exception:
             logger.debug("Failed to add current date/time context", exc_info=True)
 
-    # Auto-compact
-    messages, context_length, was_compacted = await maybe_compact(
-        sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
-    )
+    route_messages = list(messages)
+    # Explicit fallback routing must shape from the same route-neutral prompt
+    # for every candidate. Running selected-model compaction here would mutate
+    # session history before we know which route can answer and would make a
+    # later larger-context candidate unable to recover discarded history.
+    if defer_context_shaping:
+        context_length = get_context_length(sess.endpoint_url, sess.model)
+        was_compacted = False
+    else:
+        messages, context_length, was_compacted = await maybe_compact(
+            sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
+        )
     _before_trim_messages = len(messages)
     _before_trim_tokens = estimate_tokens(messages)
-    messages = trim_for_context(messages, context_length)
+    if not defer_context_shaping:
+        messages = trim_for_context(messages, context_length)
     _after_trim_messages = len(messages)
     _after_trim_tokens = estimate_tokens(messages)
     _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
@@ -860,14 +1174,22 @@ async def build_chat_context(
         context_tokens_after_trim=_after_trim_tokens,
         auto_opened_docs=auto_opened_docs,
         uploaded_files=uploaded_files,
+        route_messages=route_messages,
     )
 
 
 def accumulate_token_usage(session_id: str, metrics: dict):
-    """Add input/output token counts to the session's running totals."""
+    """Add input/output token counts (and USD cost) to the session's totals."""
     in_t = metrics.get("input_tokens", 0)
     out_t = metrics.get("output_tokens", 0)
-    if not (in_t or out_t):
+    cost = metrics.get("cost_usd")
+    try:
+        cost = float(cost) if cost is not None else 0.0
+        if not math.isfinite(cost) or cost < 0:
+            cost = 0.0
+    except (TypeError, ValueError):
+        cost = 0.0
+    if not (in_t or out_t or cost):
         return
     db = SessionLocal()
     try:
@@ -875,6 +1197,8 @@ def accumulate_token_usage(session_id: str, metrics: dict):
         if db_s:
             db_s.total_input_tokens = (db_s.total_input_tokens or 0) + in_t
             db_s.total_output_tokens = (db_s.total_output_tokens or 0) + out_t
+            if cost:
+                db_s.total_cost_usd = (db_s.total_cost_usd or 0.0) + cost
             db.commit()
     except Exception:
         db.rollback()
@@ -927,6 +1251,21 @@ def _normalize_thinking(text: str) -> str:
 
     # Qwen3.5: "Thinking Process:" or "Thinking:" prefix
     if thinking_prefix_re.match(text.lstrip()):
+        # Tool-router checkpoints sometimes narrate several drafts and then
+        # emit an explicit final marker near the end. Prefer the last marker;
+        # the first ordinary-looking paragraph can still be internal review.
+        final_markers = list(re.finditer(
+            r"(?im)^\s*Final\s+(?:decision|answer|output(?:\s+generation)?)\s*:\s*",
+            text,
+        ))
+        if final_markers:
+            marker = final_markers[-1]
+            think = thinking_prefix_re.sub('', text[:marker.start()]).strip()
+            reply = text[marker.end():].strip()
+            if len(reply) >= 2 and reply[0] in {'\"', '\u201c'} and reply[-1] in {'\"', '\u201d'}:
+                reply = reply[1:-1].strip()
+            if reply:
+                return '<think>' + think + '</think>\n\n' + reply
         # Try clean boundary first
         m = re.match(
             r'^(Thinking(?:\s+Process)?:[\s\S]*?)(\n\n(?=[A-Z]|Hey|Yo|Hi|Sure|I |What|Here|Let|The |This |OK|Ok|Yes|No |So |Well |Thank|Alright|Of course|Absolutely|Great|Hello|As ))',
@@ -1055,6 +1394,23 @@ def clean_thinking_for_save(content: str, metadata: dict | None = None) -> tuple
         if info.get("time"):
             md["thinking_time"] = info["time"]
         return info["reply"], md
+    # A stopped stream can end before producing any answer prose. Preserve its
+    # partial reasoning as structured metadata so history rendering and the
+    # next Resume request can both recover it. Normal reasoning-only completed
+    # turns retain the legacy raw-content behavior.
+    if md.get("stopped"):
+        raw = str(content or "")
+        partial = re.match(
+            r'^\s*<think(?:ing)?(?:\s+time="([\d.]+)")?>([\s\S]*?)(?:</think(?:ing)?>\s*)?$',
+            raw,
+            re.IGNORECASE,
+        )
+        if partial and partial.group(2).strip():
+            md["thinking"] = partial.group(2).strip()
+            md["thinking_interrupted"] = True
+            if partial.group(1):
+                md["thinking_time"] = partial.group(1)
+            return "", md
     return content, md
 
 
@@ -1109,6 +1465,16 @@ def save_assistant_response(
     if tool_events:
         md["tool_events"] = tool_events
 
+    # The streaming route may have forwarded textual DSML/XML tool calls as
+    # deltas before the agent loop parsed them. Strip them again at the
+    # persistence boundary so raw tool markup cannot survive in history.
+    try:
+        from src.tool_parsing import strip_tool_blocks
+        full_response = strip_tool_blocks(str(full_response or "")).strip()
+    except Exception:
+        full_response = str(full_response or "")
+    full_response = clean_repeated_assistant_content(full_response)
+
     # Extract thinking into metadata (don't pollute message content with <think> tags)
     _think_info = _extract_thinking_meta(full_response)
     if _think_info:
@@ -1134,10 +1500,25 @@ def save_assistant_response(
     try:
         _last = sess.history[-1]
         _meta = getattr(_last, "metadata", None)
+        _message_id = _meta.get("_db_id") if isinstance(_meta, dict) else None
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+            message_id=_message_id,
+        )
         if isinstance(_meta, dict):
-            return _meta.get("_db_id")
+            return _message_id
     except (IndexError, AttributeError):
-        pass
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+        )
     return None
 
 
@@ -1210,6 +1591,8 @@ def run_post_response_tasks(
     owner: str = None,
     extract_skills: bool = True,
     allow_background_extraction: bool = True,
+    preset_manager=None,
+    persona_memory_schema: str = "general",
 ):
     """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.
 
@@ -1230,7 +1613,8 @@ def run_post_response_tasks(
     # Memory extraction — only every 4th message pair to avoid excess LLM calls
     _msg_count = len(sess.history) if hasattr(sess, 'history') else 0
     _should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0)
-    if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
+    _chat_memory_extract = getattr(sess, "memory_extraction_enabled", True) is not False
+    if allow_background_extraction and not incognito and not compare_mode and _chat_memory_extract and _should_extract and uprefs.get("auto_memory", True):
         from services.memory.memory_extractor import extract_and_store
         from src.task_endpoint import resolve_task_endpoint
         t_url, t_model, t_headers = resolve_task_endpoint(
@@ -1241,6 +1625,27 @@ def run_post_response_tasks(
             t_url, t_model, t_headers,
         )))
 
+    if (
+        allow_background_extraction
+        and not incognito
+        and not compare_mode
+        and _chat_memory_extract
+        and _should_extract
+        and uprefs.get("auto_memory", True)
+        and character_name
+    ):
+        if preset_manager is not None:
+            from services.memory.memory_extractor import update_persona_memory
+            from src.task_endpoint import resolve_task_endpoint
+            p_url, p_model, p_headers = resolve_task_endpoint(
+                sess.endpoint_url, sess.model, sess.headers, owner=owner,
+            )
+            _extraction_jobs.append(("persona-memory", update_persona_memory(
+                sess, preset_manager, character_name,
+                p_url, p_model, p_headers,
+                schema=persona_memory_schema,
+            )))
+
     # Skill extraction from complex agent runs. Only when the user actually
     # chose agent mode — not a chat we auto-escalated for a notes/calendar
     # intent, and never in incognito/compare.
@@ -1255,13 +1660,17 @@ def run_post_response_tasks(
         extract_skills, auto_skills_enabled, incognito, compare_mode,
         agent_rounds, agent_tool_calls, "set" if skills_manager else "MISSING",
     )
+    # A normal inspect/edit/verify turn is commonly three calls. Treating that
+    # as a reusable skill creates one-off titles and makes the skill library
+    # noisy. Automatic extraction is reserved for runs that demonstrate a
+    # genuinely longer procedure; explicit skill tools remain unaffected.
     if (
         extract_skills
         and allow_background_extraction
         and auto_skills_enabled
         and not incognito
         and not compare_mode
-        and (agent_rounds >= 2 or agent_tool_calls >= 2)
+        and _skill_run_is_complex(agent_rounds, agent_tool_calls)
     ):
         if skills_manager is None:
             logger.warning(
@@ -1298,4 +1707,4 @@ def run_post_response_tasks(
 
     # Auto-name
     if needs_auto_name(sess.name):
-        _spawn_bg(auto_name_session(session_manager, sess))
+        _spawn_bg(auto_name_session_after_stream(session_id, session_manager, sess))

routes/contacts/contacts_routes.py

@@ -5,8 +5,10 @@ CardDAV contacts integration. Reads from local Radicale, supports
 search and adding new contacts.
 """
 
+import asyncio
 import re
 import logging
+import threading
 import uuid
 import json
 import csv
@@ -19,10 +21,11 @@ from datetime import datetime
 from urllib.parse import urljoin, urlparse, urlunparse
 
 from core.log_safety import redact_url
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from fastapi import APIRouter, Query, Depends, Request, Response, HTTPException
 from typing import List, Dict, Optional
 
 from core.middleware import require_admin
+from src.auth_helpers import effective_user
 from src.url_safety import check_outbound_url
 
 logger = logging.getLogger(__name__)
@@ -93,22 +96,37 @@ def _normalize_contact(contact: Dict) -> Dict:
     if not name and emails:
         name = emails[0].split("@")[0]
     address = str(contact.get("address") or "").strip()
-    return {
+    out = {
         "uid": str(contact.get("uid") or uuid.uuid4()),
         "name": name,
         "emails": emails,
         "phones": phones,
         "address": address,
     }
+    owner = str(contact.get("owner") or "").strip()
+    if owner:
+        out["owner"] = owner
+    return out
 
 
-def _load_local_contacts() -> List[Dict]:
+def _contact_visible_to_owner(contact: Dict, owner: Optional[str]) -> bool:
+    owner = str(owner or "").strip()
+    row_owner = str(contact.get("owner") or "").strip()
+    if owner:
+        if row_owner:
+            return row_owner == owner
+        return not owner.startswith("sft_")
+    return True
+
+
+def _load_local_contacts(owner: Optional[str] = None) -> List[Dict]:
     try:
         if not LOCAL_CONTACTS_FILE.exists():
             return []
         data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
         rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        contacts = [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        return [c for c in contacts if _contact_visible_to_owner(c, owner)]
     except Exception as e:
         logger.error(f"Failed to load local contacts: {e}")
         return []
@@ -119,7 +137,9 @@ def _save_local_contacts(contacts: List[Dict]) -> None:
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
     _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["by_owner"] = {}
     _contact_cache["fetched_at"] = datetime.utcnow()
+    _contact_cache["failed_at"] = None
 
 
 # ── vCard parsing ──
@@ -264,7 +284,58 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None,
 
 # ── In-memory cache ──
 
-_contact_cache = {"contacts": [], "fetched_at": None}
+_CONTACT_CACHE_TTL_SECONDS = 60
+_CONTACT_FAILURE_BACKOFF_SECONDS = 120
+_CARDDAV_TIMEOUT = httpx.Timeout(5.0, connect=2.0)
+
+# CardDAV can be unavailable for a while. Keep the UI responsive by serving
+# the last known result (or an empty list on first use) while a single worker
+# attempts a refresh in the background.
+_contact_cache = {
+    "contacts": [],
+    "fetched_at": None,
+    "failed_at": None,
+    "by_owner": {},
+}
+_contact_fetch_lock = threading.Lock()
+
+
+def _cached_contacts(owner_key: str) -> List[Dict]:
+    cached = (_contact_cache.get("by_owner") or {}).get(owner_key) or {}
+    if owner_key and cached:
+        return cached.get("contacts") or []
+    return _contact_cache.get("contacts") or []
+
+
+def _mark_contact_fetch_failure(owner_key: str) -> List[Dict]:
+    now = datetime.utcnow()
+    stale_contacts = _cached_contacts(owner_key)
+    _contact_cache["failed_at"] = now
+    if owner_key:
+        _contact_cache.setdefault("by_owner", {})[owner_key] = {
+            "contacts": stale_contacts,
+            "fetched_at": now,
+        }
+    else:
+        _contact_cache["fetched_at"] = now
+    return stale_contacts
+
+
+def _contact_sync_status() -> Dict[str, str]:
+    """Return a safe, user-facing summary for contact autocomplete clients."""
+    if not _carddav_configured():
+        return {"state": "local", "message": "No contact sync is configured."}
+    if _contact_fetch_lock.locked():
+        return {"state": "syncing", "message": "Syncing contacts..."}
+    failed_at = _contact_cache.get("failed_at")
+    if failed_at:
+        age = (datetime.utcnow() - failed_at).total_seconds()
+        if age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return {
+                "state": "unavailable",
+                "message": "Contacts sync is unavailable. Try again later.",
+            }
+    return {"state": "ready", "message": ""}
 
 
 def _abs_url(href: str) -> str:
@@ -306,7 +377,7 @@ def _fetch_via_report(cfg, auth):
             "REPORT", cfg["url"],
             content=_ADDRESSBOOK_QUERY.encode("utf-8"),
             headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
+            auth=auth, timeout=_CARDDAV_TIMEOUT,
         )
         if r.status_code not in (207, 200):
             return None
@@ -337,20 +408,51 @@ def _fetch_via_report(cfg, auth):
         return None
 
 
-def _fetch_contacts(force=False):
+def _fetch_contacts(force=False, owner: Optional[str] = None):
     """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
+    owner_key = str(owner or "").strip()
+    by_owner = _contact_cache.setdefault("by_owner", {})
+    if owner_key and not force and owner_key in by_owner:
+        cached = by_owner.get(owner_key) or {}
+        fetched_at = cached.get("fetched_at")
+        if fetched_at:
+            age = (datetime.utcnow() - fetched_at).total_seconds()
+            if age < _CONTACT_CACHE_TTL_SECONDS:
+                return cached.get("contacts") or []
+
+    if not owner_key and not force and _contact_cache["fetched_at"]:
         age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
+        if age < _CONTACT_CACHE_TTL_SECONDS:
             return _contact_cache["contacts"]
 
+    failed_at = _contact_cache.get("failed_at")
+    if not force and failed_at:
+        failure_age = (datetime.utcnow() - failed_at).total_seconds()
+        if failure_age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return _cached_contacts(owner_key)
+
+    # SFT users must not see the operator's personal/CardDAV contact book.
+    # Their training contacts are seeded as owner-scoped local rows.
+    if owner_key.startswith("sft_"):
+        contacts = _load_local_contacts(owner_key)
+        by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        return contacts
+
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        contacts = _load_local_contacts(owner_key or None)
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        else:
+            _contact_cache["contacts"] = contacts
+            _contact_cache["fetched_at"] = datetime.utcnow()
         return contacts
 
+    # Do not let a burst of typeahead requests start parallel CardDAV timeouts.
+    # A caller that arrives during a refresh gets the most recent cache instead.
+    if not _contact_fetch_lock.acquire(blocking=False):
+        return _cached_contacts(owner_key)
+
     try:
         cfg["url"] = _carddav_base_url(cfg)
         auth = None
@@ -360,17 +462,23 @@ def _fetch_contacts(force=False):
         contacts = _fetch_via_report(cfg, auth)
         if contacts is None:
             # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            r = httpx.get(cfg["url"], auth=auth, timeout=_CARDDAV_TIMEOUT)
             if r.status_code != 200:
                 logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
+                return _mark_contact_fetch_failure(owner_key)
             contacts = _parse_vcards(r.text)
+        fetched_at = datetime.utcnow()
         _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        _contact_cache["fetched_at"] = fetched_at
+        _contact_cache["failed_at"] = None
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": fetched_at}
         return contacts
     except Exception as e:
         logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
+        return _mark_contact_fetch_failure(owner_key)
+    finally:
+        _contact_fetch_lock.release()
 
 
 def _resolve_resource_url(uid: str) -> str:
@@ -394,25 +502,31 @@ def _resolve_resource_url(uid: str) -> str:
     return _lookup() or _vcard_url(uid)
 
 
-def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
+def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None, owner: Optional[str] = None) -> bool:
     """Add a new contact via CardDAV or local contacts."""
     email = (email or "").strip()
     phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         email_l = email.lower()
         for c in contacts:
+            if owner_key and not _contact_visible_to_owner(c, owner_key):
+                continue
             if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
                 return True
             if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
                 return True
-        contacts.append(_normalize_contact({
+        row = {
             "name": name,
             "emails": [email] if email else [],
             "phones": phone_list,
             "address": address,
-        }))
+        }
+        if owner_key:
+            row["owner"] = owner_key
+        contacts.append(_normalize_contact(row))
         _save_local_contacts(contacts)
         return True
 
@@ -650,24 +764,34 @@ def _contacts_to_csv(contacts: List[Dict]) -> str:
     return out.getvalue()
 
 
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "", owner: Optional[str] = None) -> bool:
     """Rewrite an existing contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         found = False
         out = []
         for c in contacts:
             if c.get("uid") == uid:
+                if owner_key and not _contact_visible_to_owner(c, owner_key):
+                    out.append(c)
+                    continue
                 # Preserve existing address when caller passes "" (only
                 # updating name/emails/phones, not touching address).
                 addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}
+                if owner_key:
+                    row["owner"] = owner_key
+                out.append(_normalize_contact(row))
                 found = True
             else:
                 out.append(c)
         if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+            row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}
+            if owner_key:
+                row["owner"] = owner_key
+            out.append(_normalize_contact(row))
         _save_local_contacts(out)
         return True
 
@@ -694,12 +818,16 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], a
         return False
 
 
-def _delete_contact(uid: str) -> bool:
+def _delete_contact(uid: str, owner: Optional[str] = None) -> bool:
     """Delete a contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
+        remaining = [
+            c for c in contacts
+            if c.get("uid") != uid or (owner_key and not _contact_visible_to_owner(c, owner_key))
+        ]
         _save_local_contacts(remaining)
         return True
 
@@ -739,17 +867,17 @@ def setup_contacts_routes():
     router = APIRouter(prefix="/api/contacts", tags=["contacts"])
 
     @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
+    async def list_contacts(request: Request, _admin: str = Depends(require_admin)):
         """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
+        return {"contacts": contacts, "count": len(contacts), "sync": _contact_sync_status()}
 
     @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+    async def search_contacts(request: Request, q: str = Query(""), _admin: str = Depends(require_admin)):
         """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
         if not q:
-            return {"results": []}
+            return {"results": [], "sync": _contact_sync_status()}
         q_lower = q.lower()
         results = []
         for c in contacts:
@@ -760,11 +888,12 @@ def setup_contacts_routes():
                 if q_lower in em.lower():
                     results.append(c)
                     break
-        return {"results": results[:10]}
+        return {"results": results[:10], "sync": _contact_sync_status()}
 
     @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+    async def add_contact(data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Add a new contact."""
+        owner = effective_user(request)
         name = (data.get("name") or "").strip()
         email = (data.get("email") or "").strip()
         phone = (data.get("phone") or "").strip()
@@ -778,17 +907,20 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, phone, or address required"}
         if not name:
             name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-        contacts = _fetch_contacts()
+        contacts = _fetch_contacts(owner=owner)
         for c in contacts:
             if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
                 return {"success": True, "message": "Already exists", "contact": c}
             if phones and any(p in (c.get("phones") or []) for p in phones):
                 return {"success": True, "message": "Already exists", "contact": c}
         create_params = inspect.signature(_create_contact).parameters
-        if "phones" in create_params:
-            ok = _create_contact(name, email, address, phones=phones)
-        elif len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
+        if len(create_params) >= 3:
+            create_kwargs = {}
+            if "phones" in create_params:
+                create_kwargs["phones"] = phones
+            if "owner" in create_params:
+                create_kwargs["owner"] = owner
+            ok = _create_contact(name, email, address, **create_kwargs)
         else:
             ok = _create_contact(name, email)
         # If a phone was provided, do an immediate update to thread it
@@ -796,7 +928,7 @@ def setup_contacts_routes():
         # email + address; phones happen via update).
         if ok and phones and "phones" not in create_params:
             try:
-                fresh = _fetch_contacts(force=True)
+                fresh = _fetch_contacts(force=True, owner=owner)
                 created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
                 if created:
                     _update_contact(
@@ -804,6 +936,7 @@ def setup_contacts_routes():
                         created.get("emails", []),
                         phones,
                         address,
+                        owner=owner,
                     )
             except Exception:
                 pass
@@ -830,11 +963,16 @@ def setup_contacts_routes():
 
     @router.get("/export")
     async def export_contacts(
+        request: Request,
         format: str = Query("vcf", pattern="^(vcf|csv)$"),
         _admin: str = Depends(require_admin),
     ):
         """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
+        contacts = await asyncio.to_thread(
+            _fetch_contacts,
+            force=True,
+            owner=effective_user(request),
+        )
         if format == "csv":
             content = _contacts_to_csv(contacts)
             media_type = "text/csv; charset=utf-8"
@@ -876,19 +1014,28 @@ def setup_contacts_routes():
         _save_settings(settings)
         # Force re-fetch
         _contact_cache["fetched_at"] = None
+        _contact_cache["failed_at"] = None
         return {"success": True}
 
     @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
+    async def clear_contacts(request: Request, _admin: str = Depends(require_admin)):
         """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
+        owner = effective_user(request)
+        if owner:
+            remaining = [
+                c for c in _load_local_contacts()
+                if not _contact_visible_to_owner(c, owner)
+            ]
+            _save_local_contacts(remaining)
+        else:
+            _save_local_contacts([])
         return {"success": True}
 
     # NOTE: the /{uid} routes are declared LAST so the literal paths above
     # (/list, /search, /add, /config) win — otherwise PUT /config would
     # match PUT /{uid} with uid="config".
     @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+    async def edit_contact(uid: str, data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Edit an existing contact — name / emails / phones / address."""
         name = (data.get("name") or "").strip()
         emails = data.get("emails")
@@ -902,15 +1049,15 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, or address required"}
         if not name and emails:
             name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
+        ok = _update_contact(uid, name, emails, phones, address, owner=effective_user(request))
         return {"success": ok}
 
     @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+    async def delete_contact(uid: str, request: Request, _admin: str = Depends(require_admin)):
         """Delete a contact by UID."""
         if not uid:
             return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
+        ok = _delete_contact(uid, owner=effective_user(request))
         return {"success": ok}
 
     return router

routes/cookbook_helpers.py

@@ -1085,6 +1085,10 @@ class ServeRequest(BaseModel):
     hf_token: str | None = None
     gpus: str | None = None
     platform: str | None = None    # "linux", "termux", or "windows"
+    # Optional explicit image runtime adapter. "auto" preserves compatibility
+    # with older callers; catalog-backed launches can set this without relying
+    # on model-name heuristics in the generated runner.
+    runtime_adapter: str | None = None
 
 
 def _parse_serve_phase(snapshot: str, task_type: str = "serve") -> dict:
@@ -1204,6 +1208,41 @@ def _safe_env_prefix(ep: str | None) -> str | None:
     return f'[ -f "{path}" ] && source "{path}" || true'
 
 
+def _local_windows_bash_env_prefix(ep: str | None) -> str | None:
+    """Convert a frontend PowerShell venv prefix for the local Git Bash runner."""
+    if not ep:
+        return ep
+
+    prefix = ep.strip()
+    if not prefix.startswith("&"):
+        return ep
+
+    raw_path = prefix[1:].lstrip()
+    if not raw_path:
+        return ep
+    if raw_path.startswith("'"):
+        if len(raw_path) < 2 or not raw_path.endswith("'"):
+            return ep
+        quoted_path = raw_path[1:-1]
+        if "'" in quoted_path.replace("''", ""):
+            return ep
+        path = quoted_path.replace("''", "'")
+    else:
+        path = raw_path.rstrip()
+        if "'" in path or '"' in path:
+            return ep
+    if any(c in path for c in "\r\n;&|`$<>"):
+        return ep
+    if not path.replace("\\", "/").casefold().endswith("/scripts/activate.ps1"):
+        return ep
+
+    bash_path = _git_bash_path(path)
+    if "\\" in bash_path:
+        return ep
+    bash_path = bash_path[: -len("Activate.ps1")] + "activate"
+    return "source " + shlex.quote(bash_path)
+
+
 def _ssh_ps(host, script_path, port=None):
     """Build SSH command to run a PowerShell script on a Windows remote."""
     pf = f"-p {port} " if port and port != "22" else ""

routes/cookbook_routes.py

@@ -50,7 +50,7 @@ from routes.cookbook_helpers import (
     _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
     _validate_local_dir, _validate_gpus, _shell_path,
     _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
-    _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
+    _safe_env_prefix, _local_windows_bash_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
     load_stored_hf_token,
     _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
@@ -73,6 +73,30 @@ _HF_TOKEN_STATUS_SNIPPET = (
 )
 
 
+def _windows_local_pid_record_line(pid_path: Path, ready_path: Path) -> str:
+    """Build the Git Bash prelude that records a Win32-stoppable PID.
+
+    Python publishes the detached outer process's Win32 PID first, then touches
+    ``ready_path``. The inner Git Bash runner waits for that publication before
+    replacing the fallback with its own Win32 PID from /proc/<msys-pid>/winpid.
+
+    Missing, malformed, or late mappings leave the valid outer PID untouched.
+    """
+    pp = shlex.quote(pid_path.as_posix())
+    rp = shlex.quote(ready_path.as_posix())
+    return (
+        "i=0; "
+        f"while [ ! -e {rp} ] && [ \"$i\" -lt 500 ]; do "
+        "i=$((i+1)); sleep 0.01; done; "
+        f"if [ -e {rp} ]; then "
+        "winpid=\"$(cat /proc/$$/winpid 2>/dev/null || true)\"; "
+        "case \"$winpid\" in ''|*[!0-9]*) ;; "
+        f"*) printf '%s\\n' \"$winpid\" > {pp} ;; esac; "
+        "fi; "
+        f"rm -f {rp}"
+    )
+
+
 def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     """Write the MLX image API helper next to the tmux runner on remote hosts."""
     script_path = Path(__file__).resolve().parents[1] / "scripts" / "mlx_image_server.py"
@@ -90,6 +114,16 @@ def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     runner_lines.append('chmod +x scripts/mlx_image_server.py 2>/dev/null || true')
 
 
+def _normalize_runtime_adapter(value: str | None) -> str:
+    """Return a shell-safe explicit image adapter name."""
+    value = (value or "auto").strip().lower()
+    if not value:
+        return "auto"
+    if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,39}", value):
+        raise HTTPException(400, "Invalid runtime adapter")
+    return value
+
+
 def _venv_root_from_serve_cmd(cmd: str) -> str:
     """Best-effort venv root from an absolute venv python in a serve command."""
     try:
@@ -978,15 +1012,18 @@ def setup_cookbook_routes() -> APIRouter:
         directly (simple commands only). Returns the launched job record."""
         log_path = TMUX_LOG_DIR / f"{session_id}.log"
         pid_path = TMUX_LOG_DIR / f"{session_id}.pid"
+        pid_ready_path: Path | None = None
         bash = find_bash()
         if bash:
             # Run the existing bash wrapper verbatim through Git Bash, redirecting
             # all output to the log the poller reads. Paths handed to bash use
             # POSIX form + shell-quoting so drive paths / spaces survive.
             inner = TMUX_LOG_DIR / f"{session_id}_run.sh"
-            pp = shlex.quote(pid_path.as_posix())
+            pid_ready_path = TMUX_LOG_DIR / f"{session_id}.pid.ready"
+            pid_ready_path.unlink(missing_ok=True)
             inner.write_text(
-                f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n",
+                _windows_local_pid_record_line(pid_path, pid_ready_path) + "\n"
+                + "\n".join(bash_lines) + "\n",
                 encoding="utf-8",
             )
             lp = shlex.quote(log_path.as_posix())
@@ -1020,7 +1057,18 @@ def setup_cookbook_routes() -> APIRouter:
             env=env,
             **detached_popen_kwargs(),
         )
+        # Publish a valid Win32 ancestor first. The Git Bash runner may then
+        # replace it with its own Win32 pid, but never before this fallback exists.
         pid_path.write_text(str(proc.pid), encoding="utf-8")
+        if pid_ready_path is not None:
+            try:
+                pid_ready_path.touch()
+            except OSError as e:
+                logger.warning(
+                    "Could not publish Windows local PID handoff for %s: %s",
+                    session_id,
+                    e,
+                )
         return {"pid": proc.pid, "log_path": str(log_path)}
 
     @router.post("/api/model/download")
@@ -1298,7 +1346,7 @@ def setup_cookbook_routes() -> APIRouter:
             # Local: run hf download in the background (tmux on POSIX, a detached
             # process + logfile on Windows where tmux doesn't exist).
             if req.env_prefix:
-                lines.append(_safe_env_prefix(req.env_prefix))
+                lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 lines.append("deactivate 2>/dev/null; hash -r")
             # Show whether the HF token reached this run (masked) — tells a gated
@@ -1373,7 +1421,6 @@ def setup_cookbook_routes() -> APIRouter:
         # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection.
         host = validate_remote_host(host)
         ssh_port = validate_ssh_port(ssh_port)
-        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
 
         model_dirs = []
         if model_dir:
@@ -1385,20 +1432,17 @@ def setup_cookbook_routes() -> APIRouter:
                     model_dirs.append(d)
         paths_code = _cached_model_scan_script(model_dirs)
 
-        scan_py = TMUX_LOG_DIR / "scan_cache.py"
-        scan_py.write_text(paths_code, encoding="utf-8")
-
         async def _run_cached_scan_once():
+            # Each request owns its script bytes. A shared scan_cache.py races
+            # when the tool scans several hosts/directories concurrently.
             if host:
-                _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
-                _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
-                if platform == "windows":
-                    # Windows: use 'python' and pipe via stdin with double-quote wrapping
-                    cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
-                else:
-                    cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
-                proc = await asyncio.create_subprocess_shell(
-                    cmd,
+                ssh_args = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
+                            '-o', 'ServerAliveInterval=4', '-o', 'ServerAliveCountMax=1']
+                if ssh_port and ssh_port != '22':
+                    ssh_args.extend(['-p', ssh_port])
+                proc = await asyncio.create_subprocess_exec(
+                    *ssh_args, host, 'python -' if platform == 'windows' else 'python3 -',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
@@ -1416,12 +1460,31 @@ def setup_cookbook_routes() -> APIRouter:
                     or which_tool("py") or "python"
                 )
                 proc = await asyncio.create_subprocess_exec(
-                    local_py, str(scan_py),
+                    local_py, '-',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
                 )
-            return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode
+            try:
+                output = await asyncio.wait_for(proc.communicate(paths_code.encode('utf-8')), timeout=60)
+                return output, proc.returncode
+            finally:
+                # A timed-out/cancelled request must not abandon its scanner.
+                # This handle belongs only to this request, never a model job.
+                if proc.returncode is None:
+                    try:
+                        proc.terminate()
+                    except ProcessLookupError:
+                        pass
+                    try:
+                        await asyncio.wait_for(proc.wait(), timeout=2)
+                    except asyncio.TimeoutError:
+                        try:
+                            proc.kill()
+                        except ProcessLookupError:
+                            pass
+                        await asyncio.wait_for(proc.wait(), timeout=2)
 
         (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
         stderr_txt = stderr_b.decode(errors="replace").strip()
@@ -1936,6 +1999,7 @@ def setup_cookbook_routes() -> APIRouter:
         validate_remote_host(req.remote_host)
         req.ssh_port = validate_ssh_port(req.ssh_port)
         req.gpus = _validate_gpus(req.gpus)
+        req.runtime_adapter = _normalize_runtime_adapter(req.runtime_adapter)
         req.hf_token = req.hf_token or _load_stored_hf_token()
         _validate_token(req.hf_token)
         # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
@@ -2128,7 +2192,7 @@ def setup_cookbook_routes() -> APIRouter:
             if req.gpus:
                 runner_lines.append(f"export CUDA_VISIBLE_DEVICES='{req.gpus}'")
             if req.env_prefix:
-                runner_lines.append(_safe_env_prefix(req.env_prefix))
+                runner_lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 runner_lines.append("deactivate 2>/dev/null; hash -r")
             _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)
@@ -2564,19 +2628,20 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('print(model)')
                 runner_lines.append('PY')
                 runner_lines.append(')"')
-                runner_lines.append('if printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; then')
+                runner_lines.append(f"export ODYSSEUS_MLX_IMAGE_ADAPTER='{_bash_squote(req.runtime_adapter or 'auto')}'")
+                runner_lines.append('if [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "hidream" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import mlx, mlx_vlm, transformers, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: HiDream MLX serving needs the model requirements in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart mlx mlx-vlm \'transformers>=4.57.0,<6.0\' huggingface_hub safetensors numpy pillow tqdm sentencepiece hf_transfer"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "boogu" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import boogu_image_mlx, mlx, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: Boogu MLX serving needs boogu-image-mlx in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U git+https://github.com/xocialize/boogu-image-mlx.git fastapi uvicorn python-multipart pillow"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "ddcolor" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: DDColor MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2596,7 +2661,7 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "inpaint" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2616,10 +2681,12 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
-                runner_lines.append('  echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
-                runner_lines.append('  echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "mflux" ] || [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ]; then')
+                runner_lines.append('  if ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
                 runner_lines.append('fi')
             elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
@@ -3478,12 +3545,19 @@ def setup_cookbook_routes() -> APIRouter:
             return {"ok": False, "error": str(e)}
 
     @router.get("/api/cookbook/hf-latest")
-    async def hf_latest(vram_gb: float = 0, limit: int = 10, pipeline: str = "text-generation", owner: str = Depends(require_user)):
+    async def hf_latest(
+        vram_gb: float = 0,
+        limit: int = 10,
+        pipeline: str = "text-generation",
+        official_only: bool = False,
+        owner: str = Depends(require_user),
+    ):
         """Fetch latest HuggingFace models, filtered by what fits in available VRAM.
 
         vram_gb: total available VRAM in GB. 0 = no filter (return everything).
         limit:   how many models to return (default 10).
         pipeline: HF pipeline_tag filter (text-generation, text-to-image, etc.).
+        official_only: restrict results to recognized first-party provider namespaces.
         """
         import re
         import httpx
@@ -3549,6 +3623,20 @@ def setup_cookbook_routes() -> APIRouter:
                     return True
             return False
 
+        # HF does not expose a universal "first-party" flag. Keep this as a
+        # namespace policy rather than a model-name list, so newly published
+        # provider models are included without recommending community forks.
+        OFFICIAL_NAMESPACES = {
+            "apple", "black-forest-labs", "deepseek-ai", "google", "lightricks",
+            "meta-llama", "microsoft", "mistralai", "nvidia", "openai", "qwen",
+            "stabilityai", "tencent", "runwayml",
+        }
+
+        def _is_official(entry: dict, repo_id: str) -> bool:
+            namespace = repo_id.split("/", 1)[0].strip().lower() if "/" in repo_id else ""
+            author = str(entry.get("author") or "").strip().lower()
+            return namespace in OFFICIAL_NAMESPACES and (not author or author == namespace)
+
         out = []
         for entry in raw:
             repo_id = entry.get("modelId") or entry.get("id") or ""
@@ -3563,6 +3651,8 @@ def setup_cookbook_routes() -> APIRouter:
             # Skip adapters, LoRAs, datasets, etc.
             if _is_excluded(repo_id, tags):
                 continue
+            if official_only and not _is_official(entry, repo_id):
+                continue
 
             est_fp16 = _est_vram_fp16(repo_id)
             quant_mult = _quant_factor(repo_id, tags)
@@ -3576,7 +3666,11 @@ def setup_cookbook_routes() -> APIRouter:
                     # if we cannot estimate size from the repo id/tags, do not
                     # present it as runnable on this hardware.
                     continue
-                if needed_vram > vram_gb:
+                # Leave allocator/runtime headroom instead of treating the
+                # reported total as a safe load budget. This keeps the
+                # official-only list honest on tight GPUs as well.
+                usable_vram = vram_gb * 0.90
+                if needed_vram > usable_vram:
                     continue
 
             out.append({
@@ -4374,6 +4468,7 @@ def setup_cookbook_routes() -> APIRouter:
 
             progress_text = ""
             full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""
+            _persisted_terminal = False
 
             if local_win_task:
                 # File-based liveness + output for the detached-process model.
@@ -4407,9 +4502,10 @@ def setup_cookbook_routes() -> APIRouter:
                     and bool(full_snapshot)
                     and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
                 )
-                if _task_status in {"stopped", "done", "completed",
+                _persisted_terminal = _task_status in {"stopped", "done", "completed",
                                     "crashed", "error", "failed",
-                                    "ended", "killed"} and not _persisted_serve_ready:
+                                    "ended", "killed"} and not _persisted_serve_ready
+                if _persisted_terminal:
                     is_alive = False
                     # Keep the persisted output_tail for the UI — it's
                     # what the agent uses to diagnose past failures.
@@ -4448,7 +4544,9 @@ def setup_cookbook_routes() -> APIRouter:
                 and (
                     ".incomplete" in full_snapshot
                     or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
-                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
+                    or (not _persisted_terminal and _download_cache_incomplete(
+                        _payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or ""
+                    ))
                 )
             )
             if is_alive or (local_win_task and full_snapshot):
@@ -4500,6 +4598,7 @@ def setup_cookbook_routes() -> APIRouter:
                         progress_text = "Download complete"
                 elif (
                     task_type == "download"
+                    and not _persisted_terminal
                     and not download_has_incomplete_evidence
                     and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 ):

routes/document/__init__.py

@@ -0,0 +1,6 @@
+"""Document route domain package (slice 2m, #4082/#4071).
+
+Contains document_routes.py and document_helpers.py, migrated from the flat
+routes/ directory. Backward-compat shims at routes/document_routes.py and
+routes/document_helpers.py re-export from here.
+"""

routes/document/document_helpers.py

@@ -0,0 +1,243 @@
+"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+
+"""Document routes — CRUD for living documents with version history."""
+
+import logging
+import os
+import re
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException, Request
+from pydantic import BaseModel
+
+from core.database import Document, DocumentVersion
+from core.database import Session as DbSession
+from src.auth_helpers import _auth_disabled
+from src.upload_handler import UploadHandler
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class DocumentCreate(BaseModel):
+    session_id: Optional[str] = None
+    title: str = "Untitled"
+    language: Optional[str] = None
+    content: str = ""
+
+class DocumentUpdate(BaseModel):
+    content: str
+    summary: Optional[str] = None
+    force_version: bool = False
+
+class DocumentPatch(BaseModel):
+    title: Optional[str] = None
+    language: Optional[str] = None
+    session_id: Optional[str] = None  # link/unlink document to a session
+
+
+# ---- Helpers ----
+
+def _doc_to_dict(doc: Document) -> Dict[str, Any]:
+    return {
+        "id": doc.id,
+        "session_id": doc.session_id,
+        "title": doc.title,
+        "language": doc.language,
+        "current_content": doc.current_content,
+        "version_count": doc.version_count,
+        "is_active": doc.is_active,
+        "archived": bool(getattr(doc, "archived", False)),
+        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
+        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
+        # Source-email provenance (set when doc was created from an email
+        # attachment) — drives the "Send signed reply" menu item.
+        "source_email_uid":        getattr(doc, "source_email_uid", None),
+        "source_email_folder":     getattr(doc, "source_email_folder", None),
+        "source_email_account_id": getattr(doc, "source_email_account_id", None),
+        "source_email_message_id": getattr(doc, "source_email_message_id", None),
+    }
+
+def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
+    return {
+        "id": v.id,
+        "document_id": v.document_id,
+        "version_number": v.version_number,
+        "content": v.content,
+        "summary": v.summary,
+        "source": v.source,
+        "created_at": v.created_at.isoformat() if v.created_at else None,
+    }
+
+
+def _verify_doc_owner(db, doc: Document, user: str):
+    """Verify `user` owns this document. Raise 404 if not.
+
+    Documents now carry their own `owner` column, so a doc whose session
+    was deleted (session_id → NULL) can still prove ownership and stay
+    openable / cloneable. We trust that column first and only fall back to
+    the session join for any not-yet-backfilled legacy row.
+    """
+    if user is None:
+        if _auth_disabled():
+            return  # Single-user / no-auth mode: allow access
+        raise HTTPException(403, "Authentication required")
+    if doc.owner is not None:
+        if doc.owner != user:
+            raise HTTPException(404, "Document not found")
+        return
+    # Legacy fallback: derive ownership from the linked session.
+    if not doc.session_id:
+        raise HTTPException(404, "Document not found")
+    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
+    if not session or session.owner != user:
+        raise HTTPException(404, "Document not found")
+
+
+def _owner_session_filter(q, user):
+    """Restrict a documents query to those owned by `user`.
+
+    Documents now carry their own `owner` column (backfilled at boot from
+    the linked session, or assigned to the admin user for legacy/orphaned
+    docs). We filter on that directly rather than on a session join, so a
+    document whose session was deleted (session_id → NULL) still shows up
+    for its owner instead of silently vanishing from the Library + search.
+
+    The owner backfill runs in init_db before the app serves requests, so
+    by the time this filter is live there are no NULL-owner rows to leak;
+    we therefore match the owner strictly for authenticated callers."""
+    if not user:
+        if user == "" or _auth_disabled():
+            return q
+        return q.filter(False)
+    return q.filter(Document.owner == user)
+
+
+
+def _slug(name: str) -> str:
+    """Filesystem-friendly version of a document title.
+
+    Whitespace becomes underscores; other unsafe punctuation is dropped.
+    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
+    """
+    import re as _re
+    s = (name or "").strip()
+    # Drop the trailing extension if the title happens to include one
+    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
+    s = _re.sub(r'\s+', '_', s)
+    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
+    s = _re.sub(r'_+', '_', s).strip('_')
+    return s or "form"
+
+
+# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
+_PDF_RENDER_SCALE = 2.0
+
+
+def _upload_path_inside(upload_dir: str, path: str) -> bool:
+    base = os.path.realpath(upload_dir)
+    p = os.path.realpath(path)
+    try:
+        return os.path.commonpath([base, p]) == base
+    except Exception:
+        return False
+
+
+def _resolve_user_upload_path(
+    upload_handler: Any,
+    upload_id: str,
+    owner: Optional[str],
+    auth_manager=None,
+) -> Optional[str]:
+    """Resolve an upload id to a filesystem path the caller may read."""
+    if upload_handler is None:
+        return None
+    resolved = upload_handler.resolve_upload(
+        upload_id,
+        owner=owner,
+        auth_manager=auth_manager,
+    )
+    if not isinstance(resolved, dict) or not resolved:
+        return None
+    path = resolved.get("path")
+    upload_dir = getattr(upload_handler, "upload_dir", None)
+    if path and upload_dir and not _upload_path_inside(upload_dir, path):
+        logger.warning("Upload path outside upload directory: %s", path)
+        return None
+    return path
+
+
+def _locate_upload(
+    upload_dir: str,
+    file_id: str,
+    owner: Optional[str] = None,
+    auth_manager=None,
+    upload_handler: Any = None,
+):
+    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
+    if upload_handler is None:
+        from src.upload_handler import UploadHandler
+
+        base_dir = os.path.dirname(os.path.abspath(upload_dir))
+        upload_handler = UploadHandler(base_dir, upload_dir)
+    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
+
+
+def _assert_pdf_marker_upload_owned(
+    request: Request,
+    content: str,
+    user: Optional[str],
+    upload_handler: Any,
+) -> None:
+    """Reject document content whose pdf_source marker points at another user's upload."""
+    if upload_handler is None:
+        return
+    from src.pdf_form_doc import find_source_upload_id
+
+    upload_id = find_source_upload_id(content or "")
+    if not upload_id:
+        return
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
+        raise HTTPException(
+            400,
+            "Document PDF marker references an upload you do not own",
+        )
+
+
+def _derive_title(content: str) -> str:
+    """Derive a title from document content."""
+    import re
+    if not isinstance(content, str):
+        return "Untitled"
+    text = content.strip()
+    if not text:
+        return "Untitled"
+
+    # Markdown header
+    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
+    if md:
+        title = md.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # HTML heading
+    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
+    if html:
+        title = html.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # First non-empty line (if short enough)
+    for line in text.split('\n'):
+        line = line.strip()
+        if line and 2 <= len(line) <= 60:
+            title = re.sub(r'[:#*`]+$', '', line).strip()
+            if title and len(title) > 50:
+                title = title[:48] + "…"
+            return title or "Untitled"
+
+    return "Untitled"

routes/document_helpers.py

@@ -1,243 +1,14 @@
-"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
 
-"""Document routes — CRUD for living documents with version history."""
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.document_helpers``, ``from routes.document_helpers import
+X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
+pattern used by test_security_regressions.py all operate on the *same* object.
+Keeps existing import paths working after slice 2m (#4082/#4071).
+"""
 
-import logging
-import os
-import re
-from typing import Any, Dict, Optional
+import sys as _sys
 
-from fastapi import HTTPException, Request
-from pydantic import BaseModel
+from routes.document import document_helpers as _canonical  # noqa: F401
 
-from core.database import Document, DocumentVersion
-from core.database import Session as DbSession
-from src.auth_helpers import _auth_disabled
-from src.upload_handler import UploadHandler
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class DocumentCreate(BaseModel):
-    session_id: Optional[str] = None
-    title: str = "Untitled"
-    language: Optional[str] = None
-    content: str = ""
-
-class DocumentUpdate(BaseModel):
-    content: str
-    summary: Optional[str] = None
-    force_version: bool = False
-
-class DocumentPatch(BaseModel):
-    title: Optional[str] = None
-    language: Optional[str] = None
-    session_id: Optional[str] = None  # link/unlink document to a session
-
-
-# ---- Helpers ----
-
-def _doc_to_dict(doc: Document) -> Dict[str, Any]:
-    return {
-        "id": doc.id,
-        "session_id": doc.session_id,
-        "title": doc.title,
-        "language": doc.language,
-        "current_content": doc.current_content,
-        "version_count": doc.version_count,
-        "is_active": doc.is_active,
-        "archived": bool(getattr(doc, "archived", False)),
-        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
-        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
-        # Source-email provenance (set when doc was created from an email
-        # attachment) — drives the "Send signed reply" menu item.
-        "source_email_uid":        getattr(doc, "source_email_uid", None),
-        "source_email_folder":     getattr(doc, "source_email_folder", None),
-        "source_email_account_id": getattr(doc, "source_email_account_id", None),
-        "source_email_message_id": getattr(doc, "source_email_message_id", None),
-    }
-
-def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
-    return {
-        "id": v.id,
-        "document_id": v.document_id,
-        "version_number": v.version_number,
-        "content": v.content,
-        "summary": v.summary,
-        "source": v.source,
-        "created_at": v.created_at.isoformat() if v.created_at else None,
-    }
-
-
-def _verify_doc_owner(db, doc: Document, user: str):
-    """Verify `user` owns this document. Raise 404 if not.
-
-    Documents now carry their own `owner` column, so a doc whose session
-    was deleted (session_id → NULL) can still prove ownership and stay
-    openable / cloneable. We trust that column first and only fall back to
-    the session join for any not-yet-backfilled legacy row.
-    """
-    if user is None:
-        if _auth_disabled():
-            return  # Single-user / no-auth mode: allow access
-        raise HTTPException(403, "Authentication required")
-    if doc.owner is not None:
-        if doc.owner != user:
-            raise HTTPException(404, "Document not found")
-        return
-    # Legacy fallback: derive ownership from the linked session.
-    if not doc.session_id:
-        raise HTTPException(404, "Document not found")
-    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
-    if not session or session.owner != user:
-        raise HTTPException(404, "Document not found")
-
-
-def _owner_session_filter(q, user):
-    """Restrict a documents query to those owned by `user`.
-
-    Documents now carry their own `owner` column (backfilled at boot from
-    the linked session, or assigned to the admin user for legacy/orphaned
-    docs). We filter on that directly rather than on a session join, so a
-    document whose session was deleted (session_id → NULL) still shows up
-    for its owner instead of silently vanishing from the Library + search.
-
-    The owner backfill runs in init_db before the app serves requests, so
-    by the time this filter is live there are no NULL-owner rows to leak;
-    we therefore match the owner strictly for authenticated callers."""
-    if not user:
-        if user == "" or _auth_disabled():
-            return q
-        return q.filter(False)
-    return q.filter(Document.owner == user)
-
-
-
-def _slug(name: str) -> str:
-    """Filesystem-friendly version of a document title.
-
-    Whitespace becomes underscores; other unsafe punctuation is dropped.
-    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
-    """
-    import re as _re
-    s = (name or "").strip()
-    # Drop the trailing extension if the title happens to include one
-    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
-    s = _re.sub(r'\s+', '_', s)
-    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
-    s = _re.sub(r'_+', '_', s).strip('_')
-    return s or "form"
-
-
-# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
-_PDF_RENDER_SCALE = 2.0
-
-
-def _upload_path_inside(upload_dir: str, path: str) -> bool:
-    base = os.path.realpath(upload_dir)
-    p = os.path.realpath(path)
-    try:
-        return os.path.commonpath([base, p]) == base
-    except Exception:
-        return False
-
-
-def _resolve_user_upload_path(
-    upload_handler: Any,
-    upload_id: str,
-    owner: Optional[str],
-    auth_manager=None,
-) -> Optional[str]:
-    """Resolve an upload id to a filesystem path the caller may read."""
-    if upload_handler is None:
-        return None
-    resolved = upload_handler.resolve_upload(
-        upload_id,
-        owner=owner,
-        auth_manager=auth_manager,
-    )
-    if not isinstance(resolved, dict) or not resolved:
-        return None
-    path = resolved.get("path")
-    upload_dir = getattr(upload_handler, "upload_dir", None)
-    if path and upload_dir and not _upload_path_inside(upload_dir, path):
-        logger.warning("Upload path outside upload directory: %s", path)
-        return None
-    return path
-
-
-def _locate_upload(
-    upload_dir: str,
-    file_id: str,
-    owner: Optional[str] = None,
-    auth_manager=None,
-    upload_handler: Any = None,
-):
-    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
-    if upload_handler is None:
-        from src.upload_handler import UploadHandler
-
-        base_dir = os.path.dirname(os.path.abspath(upload_dir))
-        upload_handler = UploadHandler(base_dir, upload_dir)
-    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
-
-
-def _assert_pdf_marker_upload_owned(
-    request: Request,
-    content: str,
-    user: Optional[str],
-    upload_handler: Any,
-) -> None:
-    """Reject document content whose pdf_source marker points at another user's upload."""
-    if upload_handler is None:
-        return
-    from src.pdf_form_doc import find_source_upload_id
-
-    upload_id = find_source_upload_id(content or "")
-    if not upload_id:
-        return
-    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
-    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
-        raise HTTPException(
-            400,
-            "Document PDF marker references an upload you do not own",
-        )
-
-
-def _derive_title(content: str) -> str:
-    """Derive a title from document content."""
-    import re
-    if not isinstance(content, str):
-        return "Untitled"
-    text = content.strip()
-    if not text:
-        return "Untitled"
-
-    # Markdown header
-    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
-    if md:
-        title = md.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # HTML heading
-    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
-    if html:
-        title = html.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # First non-empty line (if short enough)
-    for line in text.split('\n'):
-        line = line.strip()
-        if line and 2 <= len(line) <= 60:
-            title = re.sub(r'[:#*`]+$', '', line).strip()
-            if title and len(title) > 50:
-                title = title[:48] + "…"
-            return title or "Untitled"
-
-    return "Untitled"
+_sys.modules[__name__] = _canonical

routes/editor_draft_routes.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel
 
 from core.database import EditorDraft, SessionLocal
 from src.auth_helpers import get_current_user
+from src.upload_limits import EDITOR_DRAFT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +76,16 @@ def _load_payload(raw: Optional[str]) -> Dict[str, Any]:
     return payload if isinstance(payload, dict) else {}
 
 
+def _dump_payload(payload: Dict[str, Any]) -> str:
+    raw = json.dumps(payload or {}, separators=(",", ":"))
+    if len(raw.encode("utf-8")) > EDITOR_DRAFT_MAX_BYTES:
+        raise HTTPException(
+            413,
+            f"Editor draft exceeds the {EDITOR_DRAFT_MAX_BYTES // (1024 * 1024)} MB safety limit",
+        )
+    return raw
+
+
 def setup_editor_draft_routes() -> APIRouter:
     router = APIRouter(tags=["editor-drafts"])
 
@@ -120,13 +131,15 @@ def setup_editor_draft_routes() -> APIRouter:
                 source_image_id=body.source_image_id,
                 width=body.width,
                 height=body.height,
-                payload=json.dumps(body.payload or {}),
+                payload=_dump_payload(body.payload),
                 thumbnail=body.thumbnail,
             )
             db.add(d)
             db.commit()
             db.refresh(d)
             return _summary(d)
+        except HTTPException:
+            raise
         except Exception as e:
             db.rollback()
             logger.warning(f"editor-draft create failed: {e}")
@@ -151,7 +164,7 @@ def setup_editor_draft_routes() -> APIRouter:
             if body.height is not None:
                 d.height = body.height
             if body.payload is not None:
-                d.payload = json.dumps(body.payload)
+                d.payload = _dump_payload(body.payload)
             if body.thumbnail is not None:
                 d.thumbnail = body.thumbnail
             db.commit()