chore(release): align dev version with 1.0.3

Keep dev version metadata aligned with the current hotfix release while the rolling branch continues toward 1.1.0.

Evidence: the canonical APP_VERSION imports as 1.0.3 and the diff check is clean. This commit changes version metadata only; it does not tag or publish a release.

.env.example

@@ -76,12 +76,24 @@ SEARXNG_INSTANCE=http://localhost:8080
 # Change this if another local service already uses 7000 (macOS AirPlay often does).
 # APP_PORT=7000
 
+# Optional HTTP address advertised in companion/mobile pairing codes. Set this
+# when Docker would otherwise advertise a container address or loopback. Use a
+# LAN or Tailscale IPv4 address, a single-label hostname, or an mDNS *.local
+# name that the phone can reach. HTTPS and public hostnames are not supported
+# by the current companion client. Do not include credentials, a path, query,
+# or fragment.
+# COMPANION_BASE_URL=http://192.168.1.50:7000
+
 # Development-only auth bypass for loopback requests.
 # Keep false for Docker, LAN, reverse proxy, and any shared deployment.
 # LOCALHOST_BYPASS=false
 
-# Mark session cookies Secure. Set true when Odysseus is served through HTTPS
-# by a trusted reverse proxy or private access gateway.
+# Mark session cookies Secure. Left unset, this follows the request scheme:
+# an HTTPS login gets a Secure cookie, a plain-HTTP one does not. Set true to
+# force it on, or false to force it off while you still serve plain HTTP.
+# Upgrading: this used to default to false. Drop a leftover SECURE_COOKIES=false
+# from your .env unless you still need that escape hatch — it keeps HTTPS logins
+# on a non-Secure cookie.
 # SECURE_COOKIES=true
 
 # Optional: pre-seed the first admin password during setup.
@@ -130,6 +142,42 @@ SEARXNG_INSTANCE=http://localhost:8080
 # FASTEMBED_MODEL=sentence-transformers/all-MiniLM-L6-v2
 # FASTEMBED_CACHE_PATH=  # defaults to ~/.cache/fastembed
 
+# ============================================================
+# Google OAuth2 (Google Workspace / .edu email accounts)
+# ============================================================
+# Required to use the "Connect with Google" OAuth flow in email account setup.
+# Create credentials at: console.cloud.google.com → APIs & Services → Credentials
+#   1. Enable the Gmail API for your project.
+#   2. Configure the OAuth consent screen (User Type: Internal for Workspace orgs).
+#      Add scopes: https://mail.google.com/ and email.
+#   3. Create an OAuth 2.0 Client ID (type: Web application).
+#      Add your redirect URI: http://localhost:7000/api/email/oauth/google/callback
+#      (replace host/port for hosted installs).
+#   4. Copy the Client ID and Client Secret below.
+#
+# GOOGLE_OAUTH_CLIENT_ID=your-client-id.apps.googleusercontent.com
+# GOOGLE_OAUTH_CLIENT_SECRET=replace-with-client-secret
+#
+# Set this explicitly for HTTPS, reverse-proxy, or hosted deployments. The
+# value must exactly match an authorized redirect URI in the Google client.
+# Local HTTP setups may use the callback URL inferred by the application.
+# GOOGLE_OAUTH_REDIRECT_URI=https://your-domain.com/api/email/oauth/google/callback
+
+# Origin the MCP OAuth callback is sent back to, for remote (Streamable HTTP)
+# MCP servers that register it dynamically. Defaults to http://localhost:$APP_PORT,
+# which is right only when you reach Odysseus directly on that port. Set it for
+# HTTPS, reverse-proxy, hosted, and Docker installs — inside the container the
+# app always listens on 7000 and cannot see the host port map, so the default is
+# wrong there whenever APP_PORT is not 7000.
+#
+# Not for Google MCP servers. Those use Desktop App credentials, and Google only
+# accepts loopback redirect URIs for that client type, so a public origin here is
+# rejected with redirect_uri_mismatch. Leave it unset for a Google-only install:
+# the loopback default is what Google wants, and remote users finish through the
+# paste-back page, which never has to load the redirect.
+# https://developers.google.com/identity/protocols/oauth2/native-app
+# OAUTH_REDIRECT_BASE_URL=https://your-domain.com
+
 # ============================================================
 # Misc
 # ============================================================
@@ -168,6 +216,7 @@ SEARXNG_INSTANCE=http://localhost:8080
 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400       # email compose attachment (25 MB)
 # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400                  # speech-to-text audio (25 MB)
 # ODYSSEUS_ICS_MAX_BYTES=10485760                        # calendar .ics import (10 MB)
+# ODYSSEUS_TTS_CACHE_MAX_BYTES=524288000                 # TTS cache (500 MB)
 
 # ============================================================
 # Host Docker access (explicit opt-in)

.gitattributes

@@ -15,6 +15,13 @@ docker/entrypoint.sh text eol=lf
 *.cmd text eol=crlf
 *.bat text eol=crlf
 
+# Vendored third-party bundles in static/lib/ are published minified artifacts
+# and must stay byte-identical to what npm ships — stripping trailing whitespace
+# to satisfy `git diff --check` would desync them from the upstream release. Turn
+# the whitespace check off for that tree instead, and keep the bundles out of
+# GitHub's language statistics.
+static/lib/** -whitespace linguist-vendored
+
 # Binary assets — never normalize.
 *.png  binary
 *.jpg  binary

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -6,26 +6,38 @@ body:
   - type: markdown
     attributes:
       value: |
-        **Before submitting:** search [open issues](https://github.com/pewdiepie-archdaemon/odysseus/issues)
-        and [discussions](https://github.com/pewdiepie-archdaemon/odysseus/discussions) first.
+        **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues)
+        and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first.
         Duplicate reports slow things down.
 
         For security vulnerabilities, **do not open a public issue** —
-        use [GitHub Security Advisories](https://github.com/pewdiepie-archdaemon/odysseus/security/advisories/new)
-        and read [SECURITY.md](https://github.com/pewdiepie-archdaemon/odysseus/blob/main/SECURITY.md) first.
+        use [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new)
+        and read [SECURITY.md](https://github.com/odysseus-dev/odysseus/blob/main/SECURITY.md) first.
 
   - type: checkboxes
     id: prerequisites
     attributes:
       label: Prerequisites
       options:
-        - label: I searched [open issues](https://github.com/pewdiepie-archdaemon/odysseus/issues?q=is%3Aissue+is%3Aopen) and [discussions](https://github.com/pewdiepie-archdaemon/odysseus/discussions) and did not find an existing report of this bug.
+        - label: I searched [open issues](https://github.com/odysseus-dev/odysseus/issues?q=is%3Aissue+is%3Aopen) and [discussions](https://github.com/odysseus-dev/odysseus/discussions) and did not find an existing report of this bug.
           required: true
-        - label: This is **not** a security vulnerability. (Vulnerabilities go to [GitHub Security Advisories](https://github.com/pewdiepie-archdaemon/odysseus/security/advisories/new) — see [SECURITY.md](https://github.com/pewdiepie-archdaemon/odysseus/blob/main/SECURITY.md).)
+        - label: This is **not** a security vulnerability. (Vulnerabilities go to [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new) — see [SECURITY.md](https://github.com/odysseus-dev/odysseus/blob/main/SECURITY.md).)
           required: true
         - label: I am running the latest code from the `dev` branch (the default branch you get on clone, where fixes land first) and the bug still reproduces there. Please `git pull` the latest `dev` before filing.
           required: true
 
+  - type: input
+    id: revision
+    attributes:
+      label: Odysseus Revision
+      description: |
+        From the repository root (on the host when using Docker), run
+        `git show -s --abbrev=12 --format='%h (%cs)' HEAD`
+        and paste the output exactly.
+      placeholder: "1fef4929cf1d (2026-08-11)"
+    validations:
+      required: true
+
   - type: dropdown
     id: install-method
     attributes:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,13 +1,13 @@
 blank_issues_enabled: false
 contact_links:
   - name: Question / Need Help
-    url: https://github.com/pewdiepie-archdaemon/odysseus/discussions/categories/q-a
+    url: https://github.com/odysseus-dev/odysseus/discussions/categories/q-a
     about: Ask how-to questions, setup help, and model configuration questions here. Issues are for confirmed bugs and concrete proposals only.
 
   - name: Idea or Suggestion
-    url: https://github.com/pewdiepie-archdaemon/odysseus/discussions/categories/ideas
+    url: https://github.com/odysseus-dev/odysseus/discussions/categories/ideas
     about: Discuss ideas and gauge interest before opening a formal feature request. If there is already a discussion, link it in your feature request.
 
   - name: Security Vulnerability
-    url: https://github.com/pewdiepie-archdaemon/odysseus/security/advisories/new
+    url: https://github.com/odysseus-dev/odysseus/security/advisories/new
     about: Report vulnerabilities privately via GitHub Security Advisories — never as a public issue. Read SECURITY.md before reporting.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -6,22 +6,22 @@ body:
   - type: markdown
     attributes:
       value: |
-        **Before submitting:** search [open issues](https://github.com/pewdiepie-archdaemon/odysseus/issues)
-        and [discussions](https://github.com/pewdiepie-archdaemon/odysseus/discussions) first.
-        Feature requests that duplicate [ROADMAP.md](https://github.com/pewdiepie-archdaemon/odysseus/blob/main/ROADMAP.md)
+        **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues)
+        and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first.
+        Feature requests that duplicate [ROADMAP.md](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md)
         or an existing open issue will be closed as duplicates.
 
         If your idea needs community input before it becomes a concrete proposal,
-        start a [discussion](https://github.com/pewdiepie-archdaemon/odysseus/discussions/categories/ideas) instead.
+        start a [discussion](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas) instead.
 
   - type: checkboxes
     id: prerequisites
     attributes:
       label: Prerequisites
       options:
-        - label: I searched [open issues](https://github.com/pewdiepie-archdaemon/odysseus/issues?q=is%3Aissue+is%3Aopen) and this has not already been proposed.
+        - label: I searched [open issues](https://github.com/odysseus-dev/odysseus/issues?q=is%3Aissue+is%3Aopen) and this has not already been proposed.
           required: true
-        - label: I searched [discussions](https://github.com/pewdiepie-archdaemon/odysseus/discussions) and this is not already being debated there.
+        - label: I searched [discussions](https://github.com/odysseus-dev/odysseus/discussions) and this is not already being debated there.
           required: true
         - label: This is a concrete, actionable proposal — not a vague "it would be nice if..." request.
           required: true

.github/pull_request_template.md

@@ -24,10 +24,11 @@ Fixes #
 
 ## Checklist
 
-- [ ] I searched [open issues](https://github.com/pewdiepie-archdaemon/odysseus/issues) and [open PRs](https://github.com/pewdiepie-archdaemon/odysseus/pulls) — this is not a duplicate.
+- [ ] I searched [open issues](https://github.com/odysseus-dev/odysseus/issues) and [open PRs](https://github.com/odysseus-dev/odysseus/pulls) — this is not a duplicate.
 - [ ] This PR targets `dev`
 - [ ] My changes are limited to the scope described above — no unrelated refactors or whitespace changes mixed in.
 - [ ] I actually ran the app (`docker compose up` or `uvicorn app:app`) and verified the change works end-to-end. Type-checks and unit tests are not enough.
+- [ ] I did not run the app/runtime validation and stated that gap in **How to Test**. Leave this unchecked when the app-run box above is checked.
 
 ## How to Test
 

.github/scripts/check-issue-description.js

@@ -41,6 +41,14 @@ module.exports = async ({ github, context, core }) => {
       break;
 
     case 'bug': {
+      const revisionText = section('Odysseus Revision');
+      if (!/^[0-9a-f]{12} \(\d{4}-\d{2}-\d{2}\)$/i.test(revisionText)) {
+        failures.push(
+          '**Odysseus Revision** — paste the 12-character commit SHA and date, ' +
+          'for example `1fef4929cf1d (2026-08-11)`',
+        );
+      }
+
       if (!section('Install Method')) {
         failures.push('**Install Method** — select how you installed Odysseus');
       }
@@ -153,6 +161,16 @@ module.exports = async ({ github, context, core }) => {
     }
   }
 
+  const LABEL_BAD  = 'needs more info';
+  const LABEL_GOOD = 'ready for review';
+
+  // Closed issues are no longer awaiting review.
+  // This also prevents later edits to closed issues from restoring the label.
+  if (issue.state === 'closed') {
+    await dropLabel(LABEL_GOOD);
+    return;
+  }
+
   // ── Find existing bot comment to update in-place ──────────────────────────
   const MARKER = '<!-- issue-description-check -->';
   const { data: comments } = await github.rest.issues.listComments({
@@ -160,9 +178,6 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER));
 
-  const LABEL_BAD  = 'needs more info';
-  const LABEL_GOOD = 'ready for review';
-
   if (failures.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });

.github/scripts/check-pr-description.js

@@ -21,11 +21,11 @@ module.exports = async ({ github, context, core }) => {
     return strip(m?.[0].replace(new RegExp(`#+\\s+${heading}`, 'i'), '') ?? '');
   }
 
-  const problems = [];
+  const descriptionProblems = [];
 
   // 1. Summary must be filled in.
   if (section('Summary').length < 20) {
-    problems.push('**Summary** is empty or too short — describe what changed and why.');
+    descriptionProblems.push('**Summary** is empty or too short — describe what changed and why.');
   }
 
   // 2. Linked Issue must reference a real issue. Accept a bare #NNN, a closing
@@ -34,18 +34,18 @@ module.exports = async ({ github, context, core }) => {
   const linkedSection = section('Linked Issue');
   const hasIssueRef = /#\d+\b/.test(linkedSection) || /\/issues\/\d+/.test(linkedSection);
   if (!linkedSection || !hasIssueRef) {
-    problems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
+    descriptionProblems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.');
   }
 
   // 3. At least one Type of Change box must be checked.
   const typeBlock = body.match(/##\s+Type of Change[\s\S]*?(?=\n##\s|$)/i)?.[0] ?? '';
   if (!/- \[x\]/i.test(typeBlock)) {
-    problems.push('**Type of Change** — check at least one box.');
+    descriptionProblems.push('**Type of Change** — check at least one box.');
   }
 
   // 4. Duplicate-search checklist item must be checked.
   if (!/- \[x\] I searched/i.test(body)) {
-    problems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
+    descriptionProblems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.');
   }
 
   // 5. How to Test must contain enough real detail for a reviewer to act on.
@@ -53,7 +53,83 @@ module.exports = async ({ github, context, core }) => {
   //    code block — so we only require non-trivial content, not a specific shape.
   const howTo = section('How to Test');
   if (howTo.length < 30) {
-    problems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
+    descriptionProblems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").');
+  }
+
+  // Classify paths from GitHub's API. This workflow runs in the privileged base
+  // context, so it must never check out or execute code from the PR branch.
+  const changedFiles = await github.paginate(github.rest.pulls.listFiles, {
+    owner, repo, pull_number: prNum, per_page: 100,
+  });
+  const changedPaths = changedFiles.map(file => file.filename);
+
+  function isUiSensitivePath(filename) {
+    const path = filename.toLowerCase();
+    return path.startsWith('static/')
+      || path.startsWith('templates/')
+      || /\.(?:html?|css|svg)$/.test(path);
+  }
+
+  function isDocsOnlyPath(filename) {
+    const path = filename.toLowerCase();
+    return /\.(?:md|mdx|rst|adoc|txt)$/.test(path)
+      || (path.startsWith('docs/') && !isUiSensitivePath(path));
+  }
+
+  function isRuntimeSensitivePath(filename) {
+    const path = filename.toLowerCase();
+    if (isUiSensitivePath(path)) return false;
+    if (path.startsWith('tests/') || path.startsWith('.github/')) return false;
+    return /^(?:app\.py|routes\/|services\/|src\/|core\/|mcp_servers\/|scripts\/|docker\/)/.test(path)
+      || /^(?:dockerfile|docker-compose.*\.ya?ml|requirements(?:-optional)?\.txt|pyproject\.toml|setup\.py)$/.test(path)
+      || /\.(?:py|sh|ps1|bat)$/.test(path);
+  }
+
+  let classification = 'tooling';
+  if (changedPaths.some(isUiSensitivePath)) {
+    classification = 'UI-sensitive';
+  } else if (changedPaths.some(isRuntimeSensitivePath)) {
+    classification = 'backend/runtime';
+  } else if (changedPaths.length > 0 && changedPaths.every(isDocsOnlyPath)) {
+    classification = 'docs-only';
+  }
+
+  const appRan = /- \[x\]\s+I actually ran the app\b/i.test(body);
+  const appNotRun = /- \[x\]\s+I did not run the app\/runtime validation\b/i.test(body);
+  // Anchor on the wording, not the template's emphasis: a ticked box the author
+  // retyped without the surrounding ** renders identically on the PR page, so
+  // treating it as unchecked is invisible from their side. Matches the two
+  // attestations above, which already ignore formatting.
+  const screenshotChecked = /- \[x\]\s+[*_]{0,2}Screenshot or short clip[*_]{0,2}/i.test(body);
+  const screenshotSection = section('Screenshots / clips');
+  const hasVisualEvidence = /!\[[^\]]*\]\([^)]+\)|<(?:img|video|source)\b[^>]*(?:src|href)=|https?:\/\/[^\s)]+/i.test(screenshotSection);
+  const evidenceGaps = [];
+  let needsRuntimeValidation = false;
+  let needsVisualEvidence = false;
+
+  if (classification === 'backend/runtime' || classification === 'UI-sensitive') {
+    if (appRan && appNotRun) {
+      needsRuntimeValidation = true;
+      evidenceGaps.push('The app-run and explicit not-run boxes are both checked. Select the one state that is true.');
+    } else if (!appRan) {
+      needsRuntimeValidation = true;
+      if (appNotRun) {
+        evidenceGaps.push('The author explicitly reports that app/runtime validation was not performed.');
+      } else {
+        evidenceGaps.push('App/runtime validation is not author-attested. Check the run box only after running it, or check the explicit not-run box and describe the gap.');
+      }
+    }
+  }
+
+  if (classification === 'UI-sensitive') {
+    if (!screenshotChecked) {
+      needsVisualEvidence = true;
+      evidenceGaps.push('The screenshot/clip checkbox is not checked for this UI-sensitive change.');
+    }
+    if (!hasVisualEvidence) {
+      needsVisualEvidence = true;
+      evidenceGaps.push('The Screenshots / clips section does not contain an actual attachment or link.');
+    }
   }
 
   // ── Comment ──────────────────────────────────────────────────────────────
@@ -62,22 +138,43 @@ module.exports = async ({ github, context, core }) => {
   });
   const existing = comments.find(c => (c.body ?? '').includes(MARKER));
 
-  if (problems.length === 0) {
+  if (descriptionProblems.length === 0 && evidenceGaps.length === 0) {
     if (existing) {
       await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id });
     }
   } else {
-    const commentBody = [
-      MARKER,
-      '⚠️ **PR description — action needed**',
-      '',
-      'The following required sections are missing or incomplete. Please update the PR description to address them:',
-      '',
-      problems.map(p => `- ${p}`).join('\n'),
+    const commentLines = [MARKER];
+    if (descriptionProblems.length > 0) {
+      commentLines.push(
+        '⚠️ **PR description — action needed**',
+        '',
+        'The following required sections are missing or incomplete. Please update the PR description to address them:',
+        '',
+        descriptionProblems.map(problem => `- ${problem}`).join('\n'),
+      );
+    } else {
+      commentLines.push(
+        '⚠️ **PR description is complete; validation evidence is still outstanding**',
+        '',
+        `Changed-file classification: **${classification}**.`,
+      );
+    }
+    if (evidenceGaps.length > 0) {
+      commentLines.push(
+        '',
+        '**Author-reported runtime / visual state**',
+        '',
+        evidenceGaps.map(gap => `- ${gap}`).join('\n'),
+        '',
+        'Checkboxes are author attestations. GitHub Actions results remain the execution evidence for CI; this check does not prove that a local command ran.',
+      );
+    }
+    commentLines.push(
       '',
       '---',
-      '_This comment is deleted automatically once all sections are complete._',
-    ].join('\n');
+      '_This comment updates automatically when the description or changed files change._',
+    );
+    const commentBody = commentLines.join('\n');
 
     if (existing) {
       await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody });
@@ -97,34 +194,47 @@ module.exports = async ({ github, context, core }) => {
       return true;
     } catch (e) {
       if (e.status === 404) return false;
+      if (e.status === 403) {
+        core.warning(`Could not inspect label "${name}" — token lacks label read access; skipping.`);
+        return false;
+      }
       throw e;
     }
   }
 
-  async function swapLabel(num, add, remove) {
-    if (await labelExists(add)) {
+  async function setLabel(name, wanted) {
+    if (wanted && await labelExists(name)) {
       try {
-        await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] });
+        await github.rest.issues.addLabels({ owner, repo, issue_number: prNum, labels: [name] });
       } catch (e) {
         // Fail soft on a token that can't write labels so a label permission
         // problem never masks the actual description verdict.
-        if (e.status !== 403) throw e;
-        core.warning(`Could not add "${add}" — token lacks label write here; skipping.`);
+        if (e.status !== 403 && e.status !== 404) throw e;
+        core.warning(`Could not add "${name}" — label is unavailable or the token lacks label write access; skipping.`);
       }
+    } else if (wanted) {
+      core.warning(`Label "${name}" does not exist in the repo — skipping. Create it once to enable labelling.`);
     } else {
-      core.warning(`Label "${add}" does not exist in the repo — skipping. Create it once to enable labelling.`);
-    }
-    try {
-      await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name: remove });
-    } catch (e) {
-      if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
+      try {
+        await github.rest.issues.removeLabel({ owner, repo, issue_number: prNum, name });
+      } catch (e) {
+        if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
+      }
     }
   }
 
-  if (problems.length === 0) {
-    await swapLabel(prNum, 'ready for review', 'needs work');
-  } else {
-    await swapLabel(prNum, 'needs work', 'ready for review');
-    core.setFailed(`PR description has ${problems.length} issue(s) — see bot comment for details.`);
+  const descriptionComplete = descriptionProblems.length === 0;
+  const evidenceComplete = evidenceGaps.length === 0;
+  const isDraft = Boolean(context.payload.pull_request.draft);
+  await setLabel(
+    'ready for review',
+    descriptionComplete && evidenceComplete && !isDraft,
+  );
+  await setLabel('needs work', !descriptionComplete);
+  await setLabel('needs runtime validation', needsRuntimeValidation);
+  await setLabel('needs visual evidence', needsVisualEvidence);
+
+  if (!descriptionComplete) {
+    core.setFailed(`PR description has ${descriptionProblems.length} issue(s) — see bot comment for details.`);
   }
 };

.github/workflows/ci.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
   pull_request:
 
 # Least privilege: none of the jobs write to the repo.
@@ -103,10 +103,7 @@ jobs:
   python-tests:
     name: Python tests (pytest)
     runs-on: ubuntu-latest
-    # Informational for now: the suite has known flaky / environment-dependent
-    # failures (test isolation + embedding-model assertions). Tracked under the
-    # ROADMAP "fresh install smoke tests" item; make this required once green.
-    continue-on-error: true
+    # Make Python test validation authoritative for the configured scope.
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
         with:

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: CodeQL
+
+# Advanced setup so CodeQL also runs on pull requests (including from forks),
+# surfacing findings before merge instead of only after a change lands on dev.
+on:
+  push:
+    branches: [dev, main]
+  pull_request:
+    branches: [dev]
+  schedule:
+    - cron: "17 3 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      actions: read
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [actions, javascript-typescript, python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: none
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e  # v4.36.2
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/issue-description-check.yml

@@ -2,7 +2,7 @@ name: ci / issue description check
 
 on:
   issues:
-    types: [opened, edited, reopened]
+    types: [opened, edited, reopened, closed]
 
 permissions:
   issues: write

.github/workflows/pr-description-check.yml

@@ -5,7 +5,11 @@ on:
   # works on fork PRs. Safe here: the checkout pins to the base branch (no fork
   # code runs) and the scripts only read context.payload and call the GitHub API.
   pull_request_target:  # zizmor: ignore[dangerous-triggers]
-    types: [opened, edited, synchronize, reopened, ready_for_review]
+    types: [opened, edited, synchronize, reopened, ready_for_review, converted_to_draft]
+
+concurrency:
+  group: pr-description-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
 
 # Default-deny at the workflow level; each job opts into only the scopes it needs.
 # Note: modifying a PR's labels/comments needs pull-requests:write even though the
@@ -59,12 +63,14 @@ jobs:
 
   check-mergeable:
     name: Flag unmergeable PRs
+    needs: check-description
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
       issues: write
-    # Skip bots: they open PRs programmatically and have their own process.
-    if: github.event.pull_request.user.type != 'Bot'
+    # Run after description validation failures, but never from an obsolete
+    # workflow run canceled by a newer PR event.
+    if: ${{ !cancelled() && github.event.pull_request.user.type != 'Bot' }}
     steps:
       - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
         with:

ACKNOWLEDGMENTS.md

@@ -65,6 +65,16 @@ Vendored in `static/lib/` and served directly:
 | [jsPDF](https://github.com/parallax/jsPDF) (bundled in html2pdf) | PDF generation | MIT |
 | [html2canvas](https://github.com/niklasvh/html2canvas) (bundled in html2pdf) | DOM → canvas rasterization | MIT |
 | [node-qrcode](https://github.com/soldair/node-qrcode) (`qrcode.min.js`) | QR-code rendering (2FA setup) | MIT |
+| [KaTeX](https://github.com/KaTeX/KaTeX) v0.16.22 (`katex/katex.min.{js,css}` + `katex/fonts/*.woff2`) | Math typesetting | MIT ([`licenses/KaTeX-MIT-LICENSE.txt`](licenses/KaTeX-MIT-LICENSE.txt)) |
+| [Mermaid](https://github.com/mermaid-js/mermaid) v11.16.1 (`mermaid.min.js`) | Diagrams from text | MIT ([`licenses/Mermaid-MIT-LICENSE.txt`](licenses/Mermaid-MIT-LICENSE.txt)) |
+
+KaTeX and Mermaid are loaded on first use by `static/js/markdown.js` rather than
+from `index.html`, so a session that renders no math and no diagram never fetches
+either. Only the `.woff2` KaTeX fonts are shipped, matching `static/fonts/`; the
+`.woff` and `.ttf` variants its stylesheet also lists are never requested by a
+browser that supports `woff2`. The bundles are the published npm artifacts,
+unmodified — `.gitattributes` turns the whitespace check off for `static/lib/`
+so they can stay byte-identical to upstream.
 
 ## Front-end libraries loaded at runtime (CDN)
 
@@ -72,8 +82,6 @@ Referenced from `cdn.jsdelivr.net` / `cdnjs.cloudflare.com` at runtime — not v
 
 | Library | Purpose | License |
 |---|---|---|
-| [KaTeX](https://github.com/KaTeX/KaTeX) 0.16.22 | Math typesetting | MIT |
-| [Mermaid](https://github.com/mermaid-js/mermaid) 11 | Diagrams from text | MIT |
 | [Pyodide](https://github.com/pyodide/pyodide) 0.27.5 | In-browser Python runtime | MPL-2.0 |
 | [PDFObject](https://github.com/pipwerks/PDFObject) 2.1.1 | Inline PDF embedding | MIT |
 

CONTRIBUTING.md

@@ -25,7 +25,7 @@ End-users cloning the repo will land on `dev` by default. To run the curated/sta
 Docker is the recommended path for normal testing:
 
 ```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 cp .env.example .env
 docker compose up -d --build

Dockerfile

@@ -16,7 +16,8 @@ FROM python:3.14-slim
 # downloads, and serves from Docker installs.
 # git/cmake are required when Cookbook builds llama.cpp on first llama.cpp
 # launch inside Docker.
-# nodejs/npm provide npx for the optional built-in Browser MCP server.
+# nodejs/npm provide npx for the built-in Browser MCP server.
+# chromium provides the actual browser binary used by that MCP server.
 # gosu lets the entrypoint drop privileges cleanly so signals still reach
 # uvicorn directly (no extra shell layer like `su`/`sudo` would add).
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -26,6 +27,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     nodejs \
     npm \
+    chromium \
     tmux \
     openssh-client \
     gosu \
@@ -54,7 +56,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # /var/run/docker.sock mount). The Debian `docker.io` package ships
 # dockerd but not the client binary on slim, so grab the static client
 # tarball from download.docker.com instead.
-ARG DOCKER_CLI_VERSION=27.5.1
+ARG DOCKER_CLI_VERSION=29.6.2
 RUN ARCH="$(dpkg --print-architecture)" \
     && case "$ARCH" in \
          amd64) DARCH=x86_64 ;; \

README.md

@@ -25,10 +25,10 @@
 
 ## Quick Start
 
-> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main) if you want the more curated branch.
+> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/odysseus-dev/odysseus/tree/main) if you want the more curated branch.
 
 ```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 cp .env.example .env
 docker compose up -d --build
@@ -59,15 +59,20 @@ Help is welcome. The best entry points are fresh-install testing, provider setup
 
 ## Security
 
-Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes).
+Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly.
+
+- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
+- Keep `LOCALHOST_BYPASS=false` outside local development.
+
+Deployment details are in the [setup guide](docs/setup.md#security-notes).
 
 ## Star History
 
-<a href="https://www.star-history.com/?repos=pewdiepie-archdaemon%2Fodysseus&type=date&legend=top-left">
+<a href="https://star-history.dera.page/#odysseus-dev/odysseus&type=date&legend=top-left">
  <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=pewdiepie-archdaemon/odysseus&type=date&theme=dark&legend=top-left" />
-   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=pewdiepie-archdaemon/odysseus&type=date&legend=top-left" />
-   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=pewdiepie-archdaemon/odysseus&type=date&legend=top-left" />
+   <source media="(prefers-color-scheme: dark)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&theme=dark&legend=top-left" />
+   <source media="(prefers-color-scheme: light)" srcset="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
+   <img alt="Star History Chart" src="https://star-history.dera.page/svg?repos=odysseus-dev/odysseus&type=date&legend=top-left" />
  </picture>
 </a>
 

ROADMAP.md

@@ -12,7 +12,6 @@ the codebase, you are probably right to stay away.
   and WSL all need coverage.
 
 - Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. 
-- Self-host troubleshooting cookbook. Document the weird 30-second fixes that otherwise become 30-minute searches: Dovecot cleartext auth for local stacks, ntfy Android Instant Delivery for non-ntfy.sh servers, clipboard limits on plain-HTTP Tailscale URLs, Radicale collection URLs, and similar traps.
 - Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments.
 - Cookbook SGLang support across platforms. Make sure SGLang setup/serve works
   predictably on Linux, Windows/WSL, macOS where possible, Docker, and common
@@ -33,6 +32,14 @@ the codebase, you are probably right to stay away.
   before the user request really starts. We need slimmer prompts, better tool
   selection, smaller default tool sets, and clearer guidance for models with
   4k/8k/16k context windows.
+- Local model speculative decoding support. For Odysseus-tuned local models,
+  plan to ship or recommend a small same-tokenizer draft model when the serving
+  backend supports it. Early vLLM testing showed a generic `Qwen3-0.6B` draft
+  beside `Qwen3-8B` can materially reduce wall time, while an unsupported
+  DSpark conversion performed poorly. Treat this as a supported draft-model lane
+  first; keep MTP-specific packaging as future work only when the architecture
+  and runtime support are real. Judge this by time-to-success, tool correctness,
+  grammar, and unchanged target output, not tokens/sec alone.
 - Skill/tool prompt-injection audit. User-editable skills, notes, documents,
   fetched pages, and memories should be treated as untrusted data. Keep testing
   whether models follow malicious instructions from those surfaces.

SECURITY.md

@@ -10,7 +10,7 @@ Security fixes are handled on the default branch until formal releases are cut.
 
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
-- Set `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
+- Leave `SECURE_COOKIES` unset unless you need to override it: session cookies are marked `Secure` whenever the request arrives over HTTPS. Set `SECURE_COOKIES=true` to force it on (for a proxy Odysseus cannot see the scheme of), or `SECURE_COOKIES=false` to force it off while you still serve plain HTTP alongside HTTPS.
 - Use HTTPS when exposing the app beyond localhost.
 - Put the authenticated Odysseus web/API entrypoint behind a trusted reverse proxy or private access layer such as Cloudflare Access, Tailscale, or a VPN.
 - Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only.

THREAT_MODEL.md

@@ -37,7 +37,7 @@ Non-admin defaults are in `core/auth.py:DEFAULT_PRIVILEGES`. Tool enforcement is
 
 - **Sessions:** bcrypt passwords, 7-day session tokens stored atomically in `data/sessions.json` via `core/atomic_io.py`.
 - **2FA:** TOTP with 8 single-use backup codes. Verified after password check, before session issuance.
-- **Reserved usernames:** `internal-tool`, `api`, `demo`, `system` cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
+- **Reserved usernames:** request sentinels and the Default/Local storage owner cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`.
   - `internal-tool` is security-critical: `core/middleware.py:require_admin` treats any request where `request.state.current_user == "internal-tool"` as the in-process tool loopback and grants admin unconditionally. A real account with that name would silently pass every `require_admin` check.
 - **Orphan sessions:** `validate_token` re-checks that the user record still exists on every call. A deleted user's cookie is dropped on next request rather than continuing to authenticate.
 

app.py

@@ -67,7 +67,13 @@ from core.constants import (
     REQUEST_TIMEOUT, OPENAI_API_KEY, AUTH_FILE,
 )
 from core.database import SessionLocal, ApiToken
-from core.middleware import SecurityHeadersMiddleware, is_cors_preflight
+from core.middleware import (
+    SecurityHeadersMiddleware,
+    get_application_route_path,
+    is_cors_preflight,
+    path_is_route_or_child,
+    with_asgi_root_path,
+)
 from core.auth import AuthManager, normalize_known_username
 from core.exceptions import (
     SessionNotFoundError, InvalidFileUploadError,
@@ -78,6 +84,7 @@ import bcrypt as _bcrypt
 
 from src.app_helpers import abs_join, serve_html_with_nonce
 from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
+from src.owner_identity import auth_disabled
 from starlette.responses import RedirectResponse
 
 # ========= LOGGING =========
@@ -248,7 +255,7 @@ from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
 
 auth_manager = AuthManager()
 app.state.auth_manager = auth_manager
-AUTH_ENABLED = os.getenv("AUTH_ENABLED", "true").lower() != "false"
+AUTH_ENABLED = not auth_disabled()
 LOCALHOST_BYPASS = os.getenv("LOCALHOST_BYPASS", "false").lower() == "true"
 if LOCALHOST_BYPASS:
     logger.warning("LOCALHOST_BYPASS is enabled, loopback requests bypass authentication. Do not expose this instance to a network.")
@@ -284,7 +291,7 @@ if AUTH_ENABLED:
     def _is_auth_exempt(path: str) -> bool:
         if path in AUTH_EXEMPT_EXACT:
             return True
-        if any(path.startswith(p) for p in AUTH_EXEMPT_PREFIXES):
+        if any(path_is_route_or_child(path, p) for p in AUTH_EXEMPT_PREFIXES):
             return True
         return any(p.match(path) for p in AUTH_EXEMPT_PATTERNS)
 
@@ -355,7 +362,7 @@ if AUTH_ENABLED:
 
     class AuthMiddleware(BaseHTTPMiddleware):
         async def dispatch(self, request: Request, call_next):
-            path = request.url.path
+            path = get_application_route_path(request.scope)
             # A genuine CORS preflight (OPTIONS + Access-Control-Request-Method)
             # carries no credentials by design and must reach CORSMiddleware to be
             # answered. AuthMiddleware is the outermost middleware, so gating the
@@ -399,7 +406,10 @@ if AUTH_ENABLED:
             if not auth_manager.is_configured:
                 # No users yet — redirect to login for first-time setup
                 if not path.startswith("/api/"):
-                    return RedirectResponse(url="/login", status_code=302)
+                    return RedirectResponse(
+                        url=with_asgi_root_path(request.scope, "/login"),
+                        status_code=302,
+                    )
                 return JSONResponse(status_code=401, content={"error": "Setup required"})
 
             # --- Bearer token auth (API tokens for external integrations) ---
@@ -461,7 +471,10 @@ if AUTH_ENABLED:
             if not auth_manager.validate_token(token):
                 if path.startswith("/api/"):
                     return JSONResponse(status_code=401, content={"error": "Not authenticated"})
-                return RedirectResponse(url="/login", status_code=302)
+                return RedirectResponse(
+                    url=with_asgi_root_path(request.scope, "/login"),
+                    status_code=302,
+                )
 
             # Attach current username to request state for downstream routes
             request.state.current_user = auth_manager.get_username_for_token(token)
@@ -630,13 +643,24 @@ app.include_router(auth_router)
 
 @app.post("/api/activity/heartbeat")
 async def activity_heartbeat():
-    from src.interactive_gate import mark_browser_activity
+    from src.interactive_gate import (
+        mark_browser_activity,
+        maybe_stop_background_tasks_for_heartbeat,
+    )
+
     await mark_browser_activity()
+
     async def _stop_background():
         try:
-            await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
+            await maybe_stop_background_tasks_for_heartbeat(
+                task_scheduler.stop_background_tasks_for_foreground
+            )
         except Exception:
-            logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
+            logging.getLogger("app.foreground_gate").debug(
+                "heartbeat task stop failed",
+                exc_info=True,
+            )
+
     asyncio.create_task(_stop_background())
     return {"ok": True}
 
@@ -655,10 +679,15 @@ app.include_router(setup_emoji_routes())
 # Sessions
 from routes.session_routes import setup_session_routes
 session_config = {"REQUEST_TIMEOUT": REQUEST_TIMEOUT, "OPENAI_API_KEY": OPENAI_API_KEY, "SESSIONS_FILE": SESSIONS_FILE}
-app.include_router(setup_session_routes(session_manager, session_config, webhook_manager=webhook_manager))
+app.include_router(setup_session_routes(
+    session_manager,
+    session_config,
+    webhook_manager=webhook_manager,
+    upload_handler=upload_handler,
+))
 
 # Admin Danger Zone wipes (Settings → System → Danger Zone)
-from routes.admin_wipe_routes import setup_admin_wipe_routes
+from routes.admin_wipe.admin_wipe_routes import setup_admin_wipe_routes
 app.include_router(setup_admin_wipe_routes(session_manager))
 
 # Memory
@@ -684,10 +713,10 @@ app.include_router(setup_research_routes(research_handler, session_manager=sessi
 
 # History
 from routes.history.history_routes import setup_history_routes
-app.include_router(setup_history_routes(session_manager))
+app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler))
 
 # Search
-from routes.search_routes import setup_search_routes
+from routes.search.search_routes import setup_search_routes
 app.include_router(setup_search_routes(config))
 
 # Presets
@@ -699,7 +728,7 @@ from routes.diagnostics_routes import setup_diagnostics_routes
 app.include_router(setup_diagnostics_routes(rag_manager, rag_available, research_handler, memory_vector))
 
 # Cleanup
-from routes.cleanup_routes import setup_cleanup_routes
+from routes.cleanup.cleanup_routes import setup_cleanup_routes
 app.include_router(setup_cleanup_routes(session_manager))
 
 # Personal docs
@@ -734,7 +763,7 @@ app.include_router(setup_stt_routes(stt_service))
 logger.info("STT service initialized (provider managed via settings)")
 
 # Documents (artifacts/canvas)
-from routes.document_routes import setup_document_routes
+from routes.document.document_routes import setup_document_routes
 document_router = setup_document_routes(session_manager, upload_handler)
 app.include_router(document_router)
 
@@ -755,7 +784,7 @@ from src.task_scheduler import TaskScheduler
 task_scheduler = TaskScheduler(session_manager)
 from src.event_bus import set_task_scheduler
 set_task_scheduler(task_scheduler)
-from routes.task_routes import setup_task_routes
+from routes.task.task_routes import setup_task_routes
 app.include_router(setup_task_routes(task_scheduler))
 
 from routes.assistant_routes import setup_assistant_routes
@@ -763,7 +792,7 @@ app.include_router(setup_assistant_routes(task_scheduler))
 
 # Calendar (CalDAV)
 from routes.calendar_routes import setup_calendar_routes
-calendar_router = setup_calendar_routes()
+calendar_router = setup_calendar_routes(upload_handler=upload_handler)
 app.include_router(calendar_router)
 
 # Shell (user-facing command execution)
@@ -782,7 +811,7 @@ from routes.hwfit_routes import setup_hwfit_routes
 app.include_router(setup_hwfit_routes())
 
 # Model A/B Comparison
-from routes.compare_routes import setup_compare_routes
+from routes.compare.compare_routes import setup_compare_routes
 app.include_router(setup_compare_routes(session_manager))
 
 # User Preferences
@@ -800,7 +829,7 @@ app.include_router(setup_font_routes())
 # MCP (Model Context Protocol)
 from src.mcp_manager import McpManager
 from src.agent_tools import set_mcp_manager
-from routes.mcp_routes import setup_mcp_routes
+from routes.mcp.mcp_routes import setup_mcp_routes
 
 mcp_manager = McpManager()
 set_mcp_manager(mcp_manager)
@@ -815,7 +844,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr)
 logger.info("AI interaction tools initialized (session, memory, RAG, UI control)")
 
 # Webhooks
-from routes.webhook_routes import setup_webhook_routes
+from routes.webhook.webhook_routes import setup_webhook_routes
 app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager))
 
 # API Tokens
@@ -825,8 +854,8 @@ app.include_router(setup_api_token_routes())
 logger.info("Webhook & API token routes initialized")
 
 # Notes (Google Keep-style notes/todos)
-from routes.note_routes import setup_note_routes
-app.include_router(setup_note_routes(task_scheduler))
+from routes.note.note_routes import setup_note_routes
+app.include_router(setup_note_routes(task_scheduler, upload_handler=upload_handler))
 
 # Email
 from routes.email_routes import setup_email_routes
@@ -847,7 +876,7 @@ app.include_router(setup_codex_routes(
 ))
 app.include_router(setup_claude_routes())
 
-from routes.vault_routes import setup_vault_routes
+from routes.vault.vault_routes import setup_vault_routes
 app.include_router(setup_vault_routes())
 
 # Contacts (CardDAV)
@@ -1043,7 +1072,7 @@ async def _startup_event():
         except BaseException as e:
             logger.warning(f"Built-in MCP registration failed (non-critical): {type(e).__name__}: {e}")
         try:
-            await asyncio.wait_for(mcp_manager.connect_all_enabled(), timeout=20)
+            await mcp_manager.connect_all_enabled()
         except asyncio.TimeoutError:
             logger.warning("User MCP startup timed out (non-critical)")
         except BaseException as e:

build-macos-app.sh

@@ -73,6 +73,10 @@ cat > "$APP/Contents/MacOS/$APP_NAME.tmpl" <<'LAUNCHER'
 INSTALL_DIR="__INSTALL_DIR__"
 PORT="__PORT__"
 URL="http://127.0.0.1:${PORT}"
+# uvicorn is started with --port below, but APP_PORT is what the app itself
+# reads when it needs to build a URL for this instance (internal_api_base(),
+# companion pairing, the MCP OAuth callback), so export it as well.
+export APP_PORT="$PORT"
 export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
 
 UVICORN="$INSTALL_DIR/venv/bin/uvicorn"

companion/pairing.py

@@ -6,11 +6,14 @@ units so the route layer stays thin and the logic is directly testable.
 
 from __future__ import annotations
 
+import ipaddress
 import json
 import os
+import re
 import secrets
 import socket
 import uuid
+from urllib.parse import urlsplit
 
 import bcrypt
 
@@ -20,6 +23,102 @@ PAIRING_VERSION = 1
 COMPANION_SCOPE = "chat"
 
 
+_COMPANION_IPV4_NETWORKS = tuple(
+    ipaddress.ip_network(cidr)
+    for cidr in (
+        "10.0.0.0/8",
+        "100.64.0.0/10",
+        "127.0.0.0/8",
+        "169.254.0.0/16",
+        "172.16.0.0/12",
+        "192.168.0.0/16",
+    )
+)
+_DNS_LABEL_RE = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\Z")
+
+
+def _valid_companion_client_host(host: str) -> bool:
+    """Match the host forms supported by the current v1 Expo client."""
+    if not host or len(host) > 253 or not host.isascii() or "%" in host:
+        return False
+
+    try:
+        address = ipaddress.ip_address(host)
+    except ValueError:
+        labels = host.split(".")
+        if any(not _DNS_LABEL_RE.fullmatch(label) for label in labels):
+            return False
+        if any(label.startswith("xn--") for label in labels):
+            return False
+        # WHATWG URL parsers treat a decimal or ``0x`` single-label hostname
+        # as an IPv4 number even though Python's strict ``ipaddress`` parser
+        # rejects that spelling.  The v1 client interpolates this host back
+        # into a URL, so accepting e.g. ``134744072`` would make the phone send
+        # its bearer token to public 8.8.8.8.  Keep DNS labels unambiguous.
+        if len(labels) == 1 and (
+            labels[0].isdigit()
+            or re.fullmatch(r"0x[0-9a-f]*", labels[0]) is not None
+        ):
+            return False
+        return len(labels) == 1 or (len(labels) >= 2 and labels[-1] == "local")
+
+    return isinstance(address, ipaddress.IPv4Address) and any(
+        address in network for network in _COMPANION_IPV4_NETWORKS
+    )
+
+
+def parse_companion_base_url(value: str) -> tuple[str, int]:
+    """Validate a v1 companion address and return its legacy (host, port).
+
+    The deployed client understands only HTTP plus a LAN-style host and port.
+    Reject anything outside that exact contract instead of advertising a URL
+    the client would reject, downgrade, or interpret differently.
+    """
+    if not isinstance(value, str) or not value:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    if not value.isascii():
+        raise ValueError("COMPANION_BASE_URL must contain only ASCII characters")
+    if any(
+        ord(char) <= 32 or ord(char) == 127 or char in {"\\", "%"}
+        for char in value
+    ):
+        raise ValueError(
+            "COMPANION_BASE_URL contains a forbidden character"
+        )
+
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except ValueError as exc:
+        raise ValueError("COMPANION_BASE_URL must be a valid HTTP LAN origin") from exc
+
+    host = parsed.hostname
+    if parsed.scheme.lower() != "http" or not parsed.netloc or not host:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    if parsed.username is not None or parsed.password is not None:
+        raise ValueError("COMPANION_BASE_URL must not contain credentials")
+    if parsed.path or parsed.query or parsed.fragment:
+        raise ValueError("COMPANION_BASE_URL must not contain a path, query, or fragment")
+    if port is not None and not 1 <= port <= 65535:
+        raise ValueError("COMPANION_BASE_URL port must be between 1 and 65535")
+    if not _valid_companion_client_host(host):
+        raise ValueError("COMPANION_BASE_URL host is not supported by companion v1")
+
+    netloc = f"{host}:{port}" if port is not None else host
+    origin = f"http://{netloc}"
+    if value != origin:
+        raise ValueError("COMPANION_BASE_URL must be a canonical HTTP LAN origin")
+    return host, port or 80
+
+
+def configured_companion_origin() -> tuple[str, int] | None:
+    """Return the validated operator-configured v1 address, if any."""
+    value = os.environ.get("COMPANION_BASE_URL")
+    if value is None or value == "":
+        return None
+    return parse_companion_base_url(value)
+
+
 def default_port() -> int:
     """Best guess at the port the server is reachable on. Callers that know the
     real request port should pass it explicitly."""

companion/routes.py

@@ -23,7 +23,7 @@ from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import HTMLResponse
 
 from core.middleware import require_admin
-from src.auth_helpers import get_current_user
+from src.auth_helpers import _auth_disabled, get_current_user
 
 from companion import pairing as _pairing
 
@@ -113,8 +113,9 @@ def setup_companion_routes() -> APIRouter:
         The stock /api/models route scopes to get_current_user, which for a
         bearer token is the sandboxed pseudo-user "api" (owns nothing). Here we
         scope to the token's real owner instead, plus legacy null-owner shared
-        rows -- the same rule as owner_filter. Read-only; never returns api_key
-        material.
+        rows -- the same rule as owner_filter. Explicit auth-disabled mode keeps
+        the stock route's single-user all-endpoints view. Read-only; never
+        returns api_key material.
         """
         require_models_scope(request)
         import json as _json
@@ -123,6 +124,11 @@ def setup_companion_routes() -> APIRouter:
         from src.endpoint_resolver import build_chat_url
 
         owner = token_owner(request)
+        single_user_mode = (
+            owner is None
+            and not getattr(request.state, "api_token", False)
+            and _auth_disabled()
+        )
         out = []
         db = SessionLocal()
         try:
@@ -133,7 +139,7 @@ def setup_companion_routes() -> APIRouter:
             if owner:
                 q = q.filter((ModelEndpoint.owner == owner) | (ModelEndpoint.owner == None))  # noqa: E711
             for ep in q.all():
-                if not owner_can_see(ep.owner, owner):
+                if not single_user_mode and not owner_can_see(ep.owner, owner):
                     continue
                 try:
                     model_ids = _json.loads(ep.cached_models) if ep.cached_models else []
@@ -194,19 +200,27 @@ def setup_companion_routes() -> APIRouter:
         the code works immediately, no restart. `?format=json` returns the
         payload for an in-app pairing screen."""
         require_admin(request)
+        try:
+            configured_origin = _pairing.configured_companion_origin()
+        except ValueError as exc:
+            raise HTTPException(500, str(exc)) from None
         owner = get_current_user(request)
         invalidate = getattr(request.app.state, "invalidate_token_cache", None)
         token_id, raw_token = mint_pairing_token(owner, invalidate)
 
-        hosts = _pairing.lan_ip_candidates()
-        host = hosts[0] if hosts else "127.0.0.1"
-        port = request.url.port or _pairing.default_port()
+        if configured_origin:
+            host, port = configured_origin
+            hosts = [host]
+        else:
+            hosts = _pairing.lan_ip_candidates()
+            host = hosts[0] if hosts else "127.0.0.1"
+            port = request.url.port or _pairing.default_port()
         payload = _pairing.pairing_payload(host, port, raw_token)
         qr = _pairing.pairing_qr_png_data_uri(payload)
         qr_ok = bool(qr and qr.startswith("data:image/png;base64,"))
 
         if (request.query_params.get("format") or "").lower() == "json":
-            return {
+            response = {
                 "host": host,
                 "port": port,
                 "token": raw_token,
@@ -215,6 +229,7 @@ def setup_companion_routes() -> APIRouter:
                 "payload": payload,
                 "qr": qr if qr_ok else None,
             }
+            return response
 
         import json as _json
         payload_json = _json.dumps(payload, separators=(",", ":"))

core/atomic_io.py

@@ -15,31 +15,53 @@ from __future__ import annotations
 
 import json
 import os
+import uuid
 from typing import Any, Optional
 
 
 def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> None:
     """Atomically persist `data` as JSON at `path`.
 
-    The temp file uses the live PID as a suffix so two processes saving the
-    same file (e.g. unit tests) don't collide on the rename target.
+    The temp file uses a random suffix so two concurrent writers saving the
+    same file don't collide on the rename target. A PID suffix does not do
+    this: the PID is constant for the life of a process, so two writers on
+    the same path within one process (or one single-process container, where
+    the PID never changes at all) still race for the same temp file.
     """
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{os.getpid()}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=indent)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=indent)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
 
 
 def atomic_write_text(path: str, text: str) -> None:
     if not isinstance(text, str):
         raise TypeError("atomic_write_text expects a string")
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
-    tmp = f"{path}.tmp.{os.getpid()}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        f.write(text)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, path)
+    tmp = f"{path}.tmp.{uuid.uuid4().hex}"
+
+    try:
+        with open(tmp, "w", encoding="utf-8") as f:
+            f.write(text)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp, path)
+    finally:
+        # Directly unlink to avoid a check-then-act race condition.
+        # Swallows FileNotFoundError (on success path) and other cleanup OSErrors.
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass

core/auth.py

@@ -20,7 +20,6 @@ logger = logging.getLogger(__name__)
 
 
 from core.atomic_io import atomic_write_json as _atomic_write_json  # noqa: E402
-from core.middleware import INTERNAL_TOOL_USER  # noqa: E402
 
 DEFAULT_PRIVILEGES = {
     "can_use_agent": True,
@@ -49,24 +48,18 @@ ADMIN_PRIVILEGES["allowed_models_restricted"] = False
 ADMIN_PRIVILEGES["block_all_models"] = False
 
 from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH
+from src.owner_identity import RESERVED_AUTH_USERNAMES
 DEFAULT_AUTH_PATH = AUTH_FILE
 TOKEN_TTL = 60 * 60 * 24 * 7  # 7 days
 
-# Usernames the auth + middleware layer reserve as internal "synthetic owner"
-# sentinels; they must never belong to a real account. The most dangerous is
-# "internal-tool": `core.middleware.require_admin` treats any request whose
-# `current_user == "internal-tool"` as the in-process tool loopback and grants
-# admin, and because the cookie auth path sets `current_user` to the raw
-# username, an account literally named "internal-tool" would be silently
-# treated as an admin by every `require_admin`-gated route. "api" collides with
-# the bearer-token owner-attribution sentinel. "demo"/"system" round out the
-# synthetic-owner set the rest of the codebase already special-cases (see
-# `_SYNTHETIC_OWNERS` in routes/assistant_routes.py and the matching guards in
-# src/task_scheduler.py / routes/research_routes.py) — a real account with one
-# of those names would be denied an assistant and inconsistently owner-scoped.
-# Refuse to create or rename into any of them so the sentinels can't be
-# impersonated. (Keep this in sync with that synthetic-owner set.)
-RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"})
+# Usernames the auth + middleware layer reserves for request sentinels and
+# internal storage owners; they must never belong to a real login account.
+# "internal-tool" is the most dangerous because `core.middleware.require_admin`
+# treats it as the in-process tool loopback. "api" collides with bearer-token
+# attribution. "demo"/"system" are synthetic owners already special-cased by
+# scheduler/assistant/research paths. The Default/Local owner is a storage
+# bucket for explicit auth-disabled no-login mode, not a login username.
+RESERVED_USERNAMES = frozenset(RESERVED_AUTH_USERNAMES)
 
 
 def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]:

core/database.py

@@ -3,13 +3,16 @@ import logging
 import sqlite3
 from datetime import datetime, timezone
 from pathlib import Path
-from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
-from sqlalchemy.engine import Engine
+from typing import Optional
+from urllib.parse import unquote, urlparse
+from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text
+from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.ext.declarative import declarative_base, declared_attr
 from sqlalchemy.orm import relationship, sessionmaker, backref
 
 from src.runtime_paths import get_app_root
+from core.platform_compat import safe_chmod, IS_WINDOWS
 
 logger = logging.getLogger(__name__)
 
@@ -42,12 +45,28 @@ def _default_database_url() -> str:
 
 
 def _normalize_sqlite_url(url: str) -> str:
-    if not url.startswith("sqlite:///"):
+    """Resolve relative ordinary SQLite paths without rewriting URI filenames."""
+    try:
+        parsed = make_url(url)
+    except Exception:
         return url
-    db_path = url.replace("sqlite:///", "", 1)
-    if db_path == ":memory:" or os.path.isabs(db_path):
+
+    if parsed.get_backend_name() != "sqlite":
         return url
-    return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}"
+
+    db_path = parsed.database
+    if (
+        not db_path
+        or db_path == ":memory:"
+        or str(db_path).lower().startswith("file:")
+        or os.path.isabs(str(db_path))
+    ):
+        return url
+
+    absolute_path = (Path(get_app_root()) / str(db_path)).resolve().as_posix()
+    return parsed.set(database=absolute_path).render_as_string(
+        hide_password=False
+    )
 
 
 # Get database URL from environment, default to SQLite in DATA_DIR
@@ -59,6 +78,59 @@ engine = create_engine(
     connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
 )
 
+
+# Sidecar files SQLite can create next to the main DB. -journal is the default
+# rollback journal; -wal/-shm appear once WAL is enabled. Each can hold copies of
+# secret-bearing pages, so they get the same 0o600 lockdown as the DB itself.
+_SQLITE_SIDECARS = ("-journal", "-wal", "-shm")
+
+
+def _sqlite_db_path(url) -> Optional[str]:
+    """Return the filesystem path for a file-backed SQLite URL.
+
+    SQLite query parameters such as ``mode=memory`` only affect filename
+    semantics when SQLAlchemy enables URI handling with ``uri=true``. Ordinary
+    file URLs must therefore remain file-backed even when they contain a query
+    parameter named ``mode``.
+
+    For SQLite ``file:`` URIs, an empty authority or ``localhost`` identifies a
+    local path. Other authorities are retained as UNC-style paths.
+    """
+    if url.get_backend_name() != "sqlite":
+        return None
+
+    db_path = url.database
+    if not db_path or db_path == ":memory:":
+        return None
+
+    db_path = str(db_path)
+    query = {
+        str(key).lower(): str(value).strip().lower()
+        for key, value in dict(getattr(url, "query", {}) or {}).items()
+    }
+    uri_enabled = query.get("uri") in {"1", "true", "yes", "on"}
+    is_file_uri = db_path.lower().startswith("file:")
+
+    if not uri_enabled or not is_file_uri:
+        return db_path
+
+    if (
+        db_path.lower().startswith("file::memory:")
+        or query.get("mode") == "memory"
+    ):
+        return None
+
+    parsed = urlparse(db_path)
+    fs_path = parsed.path or ""
+    if not fs_path or fs_path == ":memory:":
+        return None
+
+    authority = parsed.netloc
+    if authority and authority.lower() != "localhost":
+        fs_path = f"//{authority}{fs_path}"
+
+    return unquote(fs_path)
+
 # Create session factory
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
@@ -358,6 +430,93 @@ class EmailAccount(TimestampMixin, Base):
     )
 
 
+class EmailAccountOwnerLock(Base):
+    """Durable per-owner mutex for email-account default mutations.
+
+    Row-locking databases serialize mutations by locking this row before they
+    inspect or stage EmailAccount changes.  SQLite uses ``BEGIN IMMEDIATE``
+    instead, because it ignores ``SELECT ... FOR UPDATE``; keeping the table in
+    the shared metadata still makes the non-SQLite path available without a
+    separate migration.  The empty key represents the normalized legacy /
+    unconfigured scope shared by ``owner IS NULL`` and ``owner = ''`` rows.
+    """
+    __tablename__ = "email_account_owner_locks"
+
+    owner_key = Column(String, primary_key=True)
+
+
+_EMAIL_ACCOUNT_DEFAULT_INDEX = "ux_email_accounts_one_default_per_owner"
+_EMAIL_ACCOUNT_DEFAULT_INDEX_DDL = {
+    "sqlite": (
+        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
+        "ON email_accounts (COALESCE(owner, '')) WHERE is_default = 1"
+    ),
+    "postgresql": (
+        f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} "
+        "ON email_accounts ((COALESCE(owner, ''))) WHERE is_default IS TRUE"
+    ),
+}
+
+
+# SQLAlchemy cannot express one portable partial, functional index across the
+# two supported database families.  Register dialect-specific DDL so fresh
+# databases get the invariant as part of create_all(); the startup migration
+# below installs the same index on existing databases after normalizing legacy
+# duplicate rows.
+for _dialect_name, _index_ddl in _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.items():
+    event.listen(
+        EmailAccount.__table__,
+        "after_create",
+        DDL(_index_ddl).execute_if(dialect=_dialect_name),
+    )
+
+
+def lock_email_account_owner_mutations(db, *owners: str) -> None:
+    """Lock normalized email-account owner scopes in canonical order.
+
+    ``NULL`` and the empty string are one legacy/single-user owner partition,
+    matching the unique default-account index.  SQLite has only a database
+    writer reservation, while row-locking databases use durable mutex rows.
+    Sorting all requested owner keys keeps multi-owner operations such as user
+    rename from deadlocking with another mutation that requests the same keys
+    in the opposite order.
+    """
+    from sqlalchemy.exc import IntegrityError
+
+    owner_keys = sorted({owner or "" for owner in owners} or {""})
+    if db.get_bind().dialect.name == "sqlite":
+        db.execute(text("BEGIN IMMEDIATE"))
+        return
+
+    for owner_key in owner_keys:
+        lock_row = db.get(
+            EmailAccountOwnerLock,
+            owner_key,
+            with_for_update=True,
+        )
+        if lock_row is not None:
+            continue
+
+        inserted = False
+        try:
+            with db.begin_nested():
+                db.add(EmailAccountOwnerLock(owner_key=owner_key))
+                db.flush()
+                inserted = True
+        except IntegrityError:
+            # A competing transaction created the mutex row first.  Once its
+            # insert commits, lock that durable row before touching accounts.
+            pass
+
+        if not inserted:
+            (
+                db.query(EmailAccountOwnerLock)
+                .filter(EmailAccountOwnerLock.owner_key == owner_key)
+                .with_for_update()
+                .one()
+            )
+
+
 class ModelEndpoint(TimestampMixin, Base):
     """Admin-configured model endpoints. Models are auto-discovered via /v1/models."""
     __tablename__ = "model_endpoints"
@@ -1332,8 +1491,25 @@ def _migrate_assign_legacy_owner():
             with open(prefs_path, "r", encoding="utf-8") as f:
                 prefs = _json.load(f)
             if "_users" not in prefs and prefs:
-                # Flat format → nest under admin user
-                new_prefs = {"_users": {admin_user: prefs}}
+                # Flat format → nest ordinary preferences under the admin
+                # user. Foreground fallback is an explicit per-owner opt-in,
+                # so auth-disabled consent must remain inert at the flat root
+                # rather than becoming consent for the first named owner.
+                foreground_keys = {
+                    "foreground_fallback_enabled",
+                    "foreground_model_fallbacks",
+                }
+                named_prefs = {
+                    key: value
+                    for key, value in prefs.items()
+                    if key not in foreground_keys
+                }
+                new_prefs = {
+                    key: prefs[key]
+                    for key in foreground_keys
+                    if key in prefs
+                }
+                new_prefs["_users"] = {admin_user: named_prefs}
                 with open(prefs_path, "w", encoding="utf-8") as f:
                     _json.dump(new_prefs, f, indent=2)
                 logger.info(f"Migrated user_prefs.json to per-user format under '{admin_user}'")
@@ -1740,72 +1916,142 @@ class Integration(TimestampMixin, Base):
 
 
 
-def _migrate_seed_email_account():
-    """If email_accounts is empty and settings.json has legacy flat imap_host/smtp_host
-    keys, create a single default account from them so nothing breaks for users who
-    upgraded. Safe to run repeatedly — it short-circuits once any row exists."""
+def _migrate_email_account_default_invariant():
+    """Normalize legacy duplicates and install durable at-most-one enforcement.
+
+    Older databases only had a non-unique ``(owner, is_default)`` lookup index.
+    Keep the oldest default deterministically in each normalized owner scope,
+    then add the same partial functional unique index used for fresh schemas.
+    """
+    dialect_name = engine.dialect.name
+    index_ddl = _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.get(dialect_name)
+    if index_ddl is None:
+        logger.warning(
+            "Email-account default uniqueness is not available for database "
+            "dialect %s; mutations remain serialized but are not protected by "
+            "a database constraint",
+            dialect_name,
+        )
+        return
+
     try:
-        with engine.connect() as conn:
-            tables = [r[0] for r in conn.execute(text(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='email_accounts'"
-            ))]
-            if "email_accounts" not in tables:
-                return
-            existing = conn.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
-            if existing > 0:
+        with engine.begin() as conn:
+            if not inspect(conn).has_table(EmailAccount.__tablename__):
                 return
+            default_rows = conn.execute(text("""
+                SELECT id, owner
+                FROM email_accounts
+                WHERE is_default IS TRUE
+                ORDER BY
+                    COALESCE(owner, ''),
+                    CASE WHEN created_at IS NULL THEN 1 ELSE 0 END,
+                    created_at,
+                    id
+            """)).mappings()
+            seen_owner_keys = set()
+            duplicate_ids = []
+            for row in default_rows:
+                owner_key = row["owner"] or ""
+                if owner_key in seen_owner_keys:
+                    duplicate_ids.append(row["id"])
+                else:
+                    seen_owner_keys.add(owner_key)
 
-        import json as _json
-        import uuid as _uuid
-        from pathlib import Path
-        settings_file = Path(SETTINGS_FILE)
-        if not settings_file.exists():
-            return
-        try:
-            s = _json.loads(settings_file.read_text(encoding="utf-8"))
-        except Exception:
-            return
+            for account_id in duplicate_ids:
+                conn.execute(
+                    text("UPDATE email_accounts SET is_default = :value WHERE id = :id"),
+                    {"value": False, "id": account_id},
+                )
+            conn.execute(text(index_ddl))
 
-        imap_host = (s.get("imap_host") or "").strip()
-        smtp_host = (s.get("smtp_host") or "").strip()
-        if not imap_host and not smtp_host:
-            return  # nothing to migrate
+        if duplicate_ids:
+            logger.warning(
+                "Normalized %d duplicate default email account(s) before "
+                "installing %s",
+                len(duplicate_ids),
+                _EMAIL_ACCOUNT_DEFAULT_INDEX,
+            )
+    except Exception:
+        # Starting without the constraint would silently retain the race this
+        # migration is intended to close.  Fail startup so an operator sees and
+        # can repair an incompatible schema instead of accepting unsafe writes.
+        logger.exception("Failed to enforce the email-account default invariant")
+        raise
+
+
+def _migrate_seed_email_account():
+    """Atomically seed one legacy default account when no account exists.
+
+    Reading settings is intentionally done before taking the owner mutex.  The
+    decisive emptiness check and insert share one locked transaction, so two
+    application workers starting together cannot both seed a default row.
+    """
+    import json as _json
+    import uuid as _uuid
+
+    settings_file = Path(SETTINGS_FILE)
+    if not settings_file.exists():
+        return
+    try:
+        s = _json.loads(settings_file.read_text(encoding="utf-8"))
+    except Exception:
+        return
+
+    imap_host = (s.get("imap_host") or "").strip()
+    smtp_host = (s.get("smtp_host") or "").strip()
+    if not imap_host and not smtp_host:
+        return
+
+    db = None
+    try:
+        if not inspect(engine).has_table(EmailAccount.__tablename__):
+            return
+        db = SessionLocal()
+        lock_email_account_owner_mutations(db, "")
+        existing = db.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0
+        if existing > 0:
+            return
 
         now = utcnow_naive()
-        with engine.begin() as conn:
-            conn.execute(text("""
-                INSERT INTO email_accounts
-                  (id, owner, name, is_default, enabled,
-                   imap_host, imap_port, imap_user, imap_password, imap_starttls,
-                   smtp_host, smtp_port, smtp_user, smtp_password,
-                   from_address, created_at, updated_at)
-                VALUES
-                  (:id, :owner, :name, :is_default, :enabled,
-                   :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
-                   :smtp_host, :smtp_port, :smtp_user, :smtp_password,
-                   :from_address, :created_at, :updated_at)
-            """), {
-                "id": _uuid.uuid4().hex,
-                "owner": None,
-                "name": "Default",
-                "is_default": True,
-                "enabled": True,
-                "imap_host": imap_host,
-                "imap_port": int(s.get("imap_port") or 993),
-                "imap_user": s.get("imap_user") or "",
-                "imap_password": s.get("imap_password") or "",
-                "imap_starttls": bool(s.get("imap_starttls", True)),
-                "smtp_host": smtp_host,
-                "smtp_port": int(s.get("smtp_port") or 465),
-                "smtp_user": s.get("smtp_user") or "",
-                "smtp_password": s.get("smtp_password") or "",
-                "from_address": s.get("email_from") or "",
-                "created_at": now,
-                "updated_at": now,
-            })
-            logging.getLogger(__name__).info("Seeded email_accounts 'Default' from settings.json")
+        db.execute(text("""
+            INSERT INTO email_accounts
+              (id, owner, name, is_default, enabled,
+               imap_host, imap_port, imap_user, imap_password, imap_starttls,
+               smtp_host, smtp_port, smtp_user, smtp_password,
+               from_address, created_at, updated_at)
+            VALUES
+              (:id, :owner, :name, :is_default, :enabled,
+               :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls,
+               :smtp_host, :smtp_port, :smtp_user, :smtp_password,
+               :from_address, :created_at, :updated_at)
+        """), {
+            "id": _uuid.uuid4().hex,
+            "owner": None,
+            "name": "Default",
+            "is_default": True,
+            "enabled": True,
+            "imap_host": imap_host,
+            "imap_port": int(s.get("imap_port") or 993),
+            "imap_user": s.get("imap_user") or "",
+            "imap_password": s.get("imap_password") or "",
+            "imap_starttls": bool(s.get("imap_starttls", True)),
+            "smtp_host": smtp_host,
+            "smtp_port": int(s.get("smtp_port") or 465),
+            "smtp_user": s.get("smtp_user") or "",
+            "smtp_password": s.get("smtp_password") or "",
+            "from_address": s.get("email_from") or "",
+            "created_at": now,
+            "updated_at": now,
+        })
+        db.commit()
+        logger.info("Seeded email_accounts 'Default' from settings.json")
     except Exception as e:
-        logging.getLogger(__name__).warning(f"seed email account migration: {e}")
+        if db is not None:
+            db.rollback()
+        logger.warning("seed email account migration: %s", e)
+    finally:
+        if db is not None:
+            db.close()
 
 
 # WARNING: Foreign-key enforcement is enabled globally for all SQLite connections.
@@ -1819,6 +2065,41 @@ def init_db():
     """
     _migrate_model_endpoints()
     Base.metadata.create_all(bind=engine)
+    # Lock the DB file (and any SQLite sidecars) to 0o600 — it holds bearer-token
+    # + bcrypt hashes and encrypted provider keys. POSIX only; safe_chmod no-ops
+    # on Windows (ACL-restricted profile dir) and the path helper returns None for
+    # Postgres / in-memory. Must stay AFTER create_all: the file is born here at
+    # the umask default, and nothing below resets the mode. The path comes from
+    # engine.url (SQLAlchemy's parsed URL), so a driver-qualified or query-tagged
+    # DATABASE_URL still resolves to the real file instead of slipping through.
+    db_path = _sqlite_db_path(engine.url)
+    if db_path is not None:
+        # Fail closed-loud on the main file: this is the only access control on
+        # it, so if the chmod genuinely fails (read-only FS, foreign owner) an
+        # operator should hear about it. safe_chmod also returns False as a
+        # Windows no-op, so guard on IS_WINDOWS to avoid a spurious warning there.
+        if not safe_chmod(db_path, 0o600) and not IS_WINDOWS:
+            logger.warning(
+                "Could not restrict %s to 0o600; it holds secrets and may be "
+                "world-readable. Check filesystem permissions and ownership.",
+                db_path,
+            )
+        # Re-lock any sidecars present at startup. New ones inherit the main
+        # file's mode (now 0o600, since we set it first), and they're usually
+        # absent here, but a stale -wal/-shm/-journal left by an older 0o644
+        # install could still expose secret pages. Absent sidecars are the
+        # normal case, not an error — only a failed chmod warrants a warning.
+        for suffix in _SQLITE_SIDECARS:
+            sidecar = db_path + suffix
+            if (
+                os.path.exists(sidecar)
+                and not safe_chmod(sidecar, 0o600)
+                and not IS_WINDOWS
+            ):
+                logger.warning(
+                    "Could not restrict %s to 0o600; it may expose DB pages.",
+                    sidecar,
+                )
     _migrate_add_hidden_models_column()
     _migrate_add_cached_models_column()
     _migrate_add_pinned_models_column()
@@ -1853,6 +2134,7 @@ def init_db():
     _migrate_add_crew_member_id()
     _migrate_add_assistant_columns()
     _migrate_add_email_smtp_security()
+    _migrate_email_account_default_invariant()
     _migrate_seed_email_account()
     _migrate_add_calendar_metadata()
     _migrate_add_calendar_is_utc()
@@ -1904,6 +2186,20 @@ def _migrate_chat_messages_fts():
     conn = None
     try:
         conn = sqlite3.connect(db_path)
+        fts_content_expr_new = (
+            "CASE WHEN instr(COALESCE(new.content, ''), ';base64,') > 0 "
+            "OR instr(COALESCE(new.content, ''), 'data:image/') > 0 "
+            "OR instr(COALESCE(new.content, ''), 'data:audio/') > 0 "
+            "THEN '[inline media omitted from search index]' "
+            "ELSE COALESCE(new.content, '') END"
+        )
+        fts_content_expr_cm = (
+            "CASE WHEN instr(COALESCE(cm.content, ''), ';base64,') > 0 "
+            "OR instr(COALESCE(cm.content, ''), 'data:image/') > 0 "
+            "OR instr(COALESCE(cm.content, ''), 'data:audio/') > 0 "
+            "THEN '[inline media omitted from search index]' "
+            "ELSE COALESCE(cm.content, '') END"
+        )
         try:
             conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)")
             conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe")
@@ -1912,7 +2208,7 @@ def _migrate_chat_messages_fts():
             return
 
         conn.executescript(
-            """
+            f"""
             CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5(
                 content,
                 message_id UNINDEXED,
@@ -1920,10 +2216,14 @@ def _migrate_chat_messages_fts():
                 role UNINDEXED
             );
 
+            DROP TRIGGER IF EXISTS chat_messages_fts_ai;
+            DROP TRIGGER IF EXISTS chat_messages_fts_ad;
+            DROP TRIGGER IF EXISTS chat_messages_fts_au;
+
             CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai
             AFTER INSERT ON chat_messages BEGIN
                 INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-                VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
+                VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
             END;
 
             CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad
@@ -1935,14 +2235,14 @@ def _migrate_chat_messages_fts():
             AFTER UPDATE ON chat_messages BEGIN
                 DELETE FROM chat_messages_fts WHERE message_id = old.id;
                 INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-                VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
+                VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role);
             END;
             """
         )
         conn.execute(
-            """
+            f"""
             INSERT INTO chat_messages_fts(content, message_id, session_id, role)
-            SELECT COALESCE(cm.content, ''), cm.id, cm.session_id, cm.role
+            SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role
             FROM chat_messages cm
             WHERE NOT EXISTS (
                 SELECT 1 FROM chat_messages_fts fts
@@ -1950,6 +2250,7 @@ def _migrate_chat_messages_fts():
             )
             """
         )
+        _scrub_legacy_chat_message_fts_media(conn)
         conn.commit()
     except Exception as e:
         logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}")
@@ -1960,6 +2261,37 @@ def _migrate_chat_messages_fts():
             pass
 
 
+def _scrub_legacy_chat_message_fts_media(conn) -> None:
+    """Replace already-indexed inline media rows with searchable text only."""
+    try:
+        from src.attachment_refs import search_index_text
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"chat_messages FTS media scrub skipped: {e}")
+        return
+
+    try:
+        rows = conn.execute(
+            """
+            SELECT id, session_id, role, content
+            FROM chat_messages
+            WHERE instr(COALESCE(content, ''), ';base64,') > 0
+               OR instr(COALESCE(content, ''), 'data:image/') > 0
+               OR instr(COALESCE(content, ''), 'data:audio/') > 0
+            """
+        ).fetchall()
+        for message_id, session_id, role, content in rows:
+            conn.execute("DELETE FROM chat_messages_fts WHERE message_id = ?", (message_id,))
+            conn.execute(
+                """
+                INSERT INTO chat_messages_fts(content, message_id, session_id, role)
+                VALUES (?, ?, ?, ?)
+                """,
+                (search_index_text(content), message_id, session_id, role),
+            )
+    except Exception as e:
+        logging.getLogger(__name__).warning(f"chat_messages FTS media scrub failed: {e}")
+
+
 def _migrate_add_email_smtp_security():
     """Add explicit SMTP security mode for Proton Bridge/custom local SMTP."""
     import sqlite3

core/middleware.py

@@ -3,10 +3,14 @@
 
 import os
 import secrets
+from collections.abc import Mapping
 
 from fastapi import HTTPException, Request
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import Response
+from starlette.routing import get_route_path
+
+from src.owner_identity import INTERNAL_TOOL_USER, auth_disabled
 
 
 # Per-process token that lets the in-app tool layer hit admin-gated
@@ -15,8 +19,30 @@ from starlette.responses import Response
 # same value from this module. Never persisted or exposed externally.
 INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32)
 INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token"
-# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved.
-INTERNAL_TOOL_USER = "internal-tool"
+
+
+def get_application_route_path(scope: Mapping[str, object]) -> str:
+    """Return the application-relative path used by Starlette routing.
+
+    Uvicorn prefixes ``scope["path"]`` with a configured ASGI ``root_path``;
+    Starlette removes that prefix before matching routes. Middleware policy
+    must use the same path form or a deployment prefix can change which policy
+    applies to an otherwise unchanged application route.
+    """
+    return get_route_path(scope)
+
+
+def with_asgi_root_path(scope: Mapping[str, object], path: str) -> str:
+    """Prefix an application path for a client-facing redirect target."""
+    root_path = scope.get("root_path", "")
+    if not isinstance(root_path, str) or not root_path:
+        return path
+    return f"{root_path.rstrip('/')}{path}"
+
+
+def path_is_route_or_child(path: str, prefix: str) -> bool:
+    """Return whether ``path`` is exactly ``prefix`` or below that route."""
+    return path == prefix or path.startswith(prefix + "/")
 
 
 def is_cors_preflight(method: str, headers) -> bool:
@@ -47,7 +73,7 @@ def require_admin(request: Request):
         pass
 
     auth_mgr = getattr(request.app.state, "auth_manager", None)
-    if os.getenv("AUTH_ENABLED", "true").lower() == "false":
+    if auth_disabled():
         return
     if not auth_mgr or not auth_mgr.is_configured:
         raise HTTPException(403, "Admin only")

core/models.py

@@ -8,6 +8,11 @@ These are simple datacontainers. All persistence is handled by SessionManager.
 from dataclasses import dataclass
 from typing import Dict, List, Any, Optional, TYPE_CHECKING
 
+from src.tool_approval_scopes import (
+    CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
+    CHAT_SESSION_APPROVAL_DECISION,
+)
+
 if TYPE_CHECKING:
     from .session_manager import SessionManager
 
@@ -31,6 +36,35 @@ set_session_manager = set_session_manager_instance
 get_session_manager = get_session_manager_instance
 
 
+def _history_grants_chat_session_approval(
+    history: List["ChatMessage"],
+    session_id: str,
+) -> bool:
+    """Return whether this exact chat has a resolved session-scope grant."""
+
+    expected_session = str(session_id or "")
+    if not expected_session:
+        return False
+    for message in reversed(history or []):
+        metadata = getattr(message, "metadata", None)
+        if not isinstance(metadata, dict):
+            continue
+        tool_events = metadata.get("tool_events")
+        if not isinstance(tool_events, list):
+            continue
+        for event in reversed(tool_events):
+            ask_user = event.get("ask_user") if isinstance(event, dict) else None
+            if not isinstance(ask_user, dict):
+                continue
+            if (
+                ask_user.get("kind") == "tool_approval"
+                and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
+                and str(ask_user.get("session_id") or "") == expected_session
+            ):
+                return True
+    return False
+
+
 @dataclass
 class ChatMessage:
     """A single chat message."""
@@ -116,11 +150,27 @@ class Session:
         the model. Display/history-load paths use the raw ``history`` and are
         unaffected.
         """
-        return [
+        messages = [
             msg.to_dict()
             for msg in self.history
             if (msg.metadata or {}).get("source") != "slash"
         ]
+        if not _history_grants_chat_session_approval(self.history, self.id):
+            return messages
+
+        # Keep the grant close to the latest user request so route-neutral
+        # compaction/trimming preserves it. Copy the metadata instead of
+        # mutating the durable transcript object.
+        for index in range(len(messages) - 1, -1, -1):
+            if messages[index].get("role") != "user":
+                continue
+            message = dict(messages[index])
+            metadata = dict(message.get("metadata") or {})
+            metadata[CHAT_SESSION_APPROVAL_CONTEXT_MARKER] = True
+            message["metadata"] = metadata
+            messages[index] = message
+            break
+        return messages
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""

core/session_manager.py

@@ -14,8 +14,12 @@ import logging
 from datetime import datetime, timezone, timedelta
 from typing import Dict, Optional
 
+from sqlalchemy import func
+
 from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
 from .models import Session, ChatMessage
+from src.attachment_refs import persistable_message_content
+from src.upload_handler import reserve_message_upload_references
 
 # Re-export singleton accessors from models for convenience
 from .models import set_session_manager_instance, get_session_manager_instance
@@ -72,6 +76,7 @@ class SessionManager:
     def __init__(self, sessions_file: str = None):
         # sessions_file kept for backward compat, not used
         self.sessions: Dict[str, Session] = {}
+        self.upload_handler = None
         self.load_sessions()
 
     # ------------------------------------------------------------------
@@ -89,14 +94,28 @@ class SessionManager:
         try:
             db_sessions = db.query(DbSession).filter(
                 DbSession.archived == False,
-                DbSession.message_count > 0,
+                DbSession.messages.any(),
             ).order_by(DbSession.last_accessed.desc()).limit(100).all()
 
+            # message_count is derived metadata and can drift after interrupted
+            # or legacy writes. Count only the bounded discovery set so startup
+            # remains metadata-only while lazy hydration sees an authoritative
+            # positive count for every discovered non-empty session.
+            message_counts = {}
+            if db_sessions:
+                message_counts = dict(
+                    db.query(DbChatMessage.session_id, func.count(DbChatMessage.id))
+                    .filter(DbChatMessage.session_id.in_([row.id for row in db_sessions]))
+                    .group_by(DbChatMessage.session_id)
+                    .all()
+                )
+
             loaded_count = 0
             for db_session in db_sessions:
                 try:
                     session = self._db_to_session_meta(db_session)
                     if session is not None:
+                        session.message_count = message_counts[db_session.id]
                         self.sessions[db_session.id] = session
                         loaded_count += 1
                 except Exception as e:
@@ -191,7 +210,12 @@ class SessionManager:
             is_important=getattr(db_session, 'is_important', False) or False,
         )
 
-        session.message_count = getattr(db_session, 'message_count', len(history))
+        # The rows just loaded are the whole transcript, so they — not the
+        # denormalized sessions.message_count column — are the truth for this
+        # cached object. get_session's hydration gate compares against this
+        # number; seeding it from a drifted column would ask for a reload that
+        # can never close the gap.
+        session.message_count = len(history)
         return session
 
     # ------------------------------------------------------------------
@@ -230,17 +254,26 @@ class SessionManager:
                 logger.warning("Dropping message for deleted session %s", session_id)
                 return
 
+            missing_upload_id = reserve_message_upload_references(
+                getattr(self, "upload_handler", None),
+                getattr(db_session, "owner", None),
+                message.content,
+                message.metadata,
+            )
+            if missing_upload_id:
+                raise ValueError(
+                    f"Referenced upload is no longer available: {missing_upload_id}"
+                )
+
             msg_id = str(uuid.uuid4())
             msg_time = datetime.utcnow()
             if message.metadata is None:
                 message.metadata = {}
             message.metadata.setdefault('timestamp', _message_timestamp_iso(msg_time))
-            # Multimodal content (image/audio attachments) is a list — serialize
-            # to JSON so the Text column can store it.  On reload, _db_to_session
-            # detects the JSON-array prefix and parses it back.
-            _content = message.content
-            if isinstance(_content, list):
-                _content = json.dumps(_content)
+            # Multimodal content may contain provider data URLs for the live
+            # model call. Persist only readable text plus attachment references
+            # so chat_messages/FTS do not duplicate upload bytes.
+            _content = persistable_message_content(message.content, message.metadata)
             db_message = DbChatMessage(
                 id=msg_id,
                 session_id=session_id,
@@ -322,6 +355,28 @@ class SessionManager:
         session = self.get_session(session_id)
         db = SessionLocal()
         try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if db_session is None:
+                logger.warning("Cannot replace history for missing session %s", session_id)
+                return False
+
+            # Reserve every incoming attachment before removing any durable
+            # message row. reserve_upload() shares the upload lifecycle lock
+            # with cleanup, so an upload cannot be deleted between this
+            # ownership check/access touch and the replacement transaction.
+            # A failed reservation must leave the existing transcript intact.
+            for message in messages:
+                missing_upload_id = reserve_message_upload_references(
+                    getattr(self, "upload_handler", None),
+                    getattr(db_session, "owner", None),
+                    message.content,
+                    message.metadata,
+                )
+                if missing_upload_id:
+                    raise ValueError(
+                        f"Referenced upload is no longer available: {missing_upload_id}"
+                    )
+
             db.query(DbChatMessage).filter(DbChatMessage.session_id == session_id).delete()
             now = datetime.now(timezone.utc)
             for i, message in enumerate(messages):
@@ -330,15 +385,9 @@ class SessionManager:
                     id=msg_id,
                     session_id=session_id,
                     role=message.role,
-                    # Multimodal content (image/audio attachments) is a list;
-                    # serialize to JSON so the Text column round-trips via
-                    # _parse_msg_content. Storing the raw list let SQLAlchemy
-                    # bind its single-quoted repr, which _parse_msg_content
-                    # cannot parse (it looks for double-quoted "type"), so the
-                    # attachment was destroyed on reload. Mirrors _persist_message.
-                    content=(json.dumps(message.content)
-                             if isinstance(message.content, list)
-                             else message.content),
+                    # Mirrors _persist_message: keep raw media bytes out of the
+                    # persisted transcript and search index.
+                    content=persistable_message_content(message.content, message.metadata),
                     meta_data=json.dumps(message.metadata) if message.metadata else None,
                     timestamp=now + timedelta(microseconds=i),
                 )
@@ -347,12 +396,10 @@ class SessionManager:
                     message.metadata = {}
                 message.metadata["_db_id"] = msg_id
 
-            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-            if db_session:
-                db_session.message_count = len(messages)
-                db_session.updated_at = now
-                db_session.last_accessed = now
-                db_session.last_message_at = now
+            db_session.message_count = len(messages)
+            db_session.updated_at = now
+            db_session.last_accessed = now
+            db_session.last_message_at = now
 
             db.commit()
             session.history = list(messages)
@@ -372,30 +419,50 @@ class SessionManager:
     # ------------------------------------------------------------------
 
     def get_session(self, session_id: str) -> Session:
-        """Get a session by ID, loading from DB if needed.
+        """Get a session by ID, loading complete DB history when needed.
 
-        Sessions seeded by `load_sessions` start with empty history. The
-        first read here hydrates them with the message rows.
+        Sessions seeded by ``load_sessions`` start with empty history, and a
+        cached session can also become partially stale. Refresh metadata first,
+        then hydrate whenever the cached transcript is short of the stored rows.
+        Model-send routes enter through this method before building context,
+        while paginated display history reads SQLite directly.
+
+        The gate compares against ``sync_session_metadata``'s reconciled count
+        (the real ``chat_messages`` total), never the denormalized column, so a
+        hydrate always closes the gap and the next read is a cache hit.
         """
         if session_id not in self.sessions:
             self._load_session_from_db(session_id)
-        else:
-            cached = self.sessions[session_id]
-            # Lazy hydrate: metadata-only entries get their messages on first read.
-            if not cached.history and getattr(cached, "message_count", 0) > 0:
-                self._load_session_from_db(session_id)
 
         # Keep model/endpoint metadata fresh. Endpoint deletion can clear the
-        # DB row while a session object is still cached in RAM.
+        # DB row while a session object is still cached in RAM. Refreshing first
+        # also exposes the authoritative message count before completeness is
+        # checked.
         self.sync_session_metadata(session_id)
 
+        cached = self.sessions[session_id]
+        cached_count = len(cached.history or [])
+        stored_count = int(getattr(cached, "message_count", 0) or 0)
+        if cached_count < stored_count:
+            self._load_session_from_db(session_id)
+
         # Update last_accessed
         self._touch_session(session_id)
 
         return self.sessions[session_id]
 
     def sync_session_metadata(self, session_id: str) -> bool:
-        """Refresh non-message session fields from the DB into the cached object."""
+        """Refresh non-message session fields from the DB into the cached object.
+
+        ``message_count`` is reconciled against the real ``chat_messages`` rows
+        rather than copied from the denormalized ``sessions.message_count``
+        column. That column drifts in normal operation — ``_persist_message``
+        swallows a failed insert but ``add_message`` has already appended in
+        memory, so the next successful persist writes rows+1, and a persist for
+        an uncached session writes 0. Hydration keys off this number: a
+        drifted-high column would reload the whole transcript on every warm
+        read, and a drifted-low one would leave the model a truncated one.
+        """
         session = self.sessions.get(session_id)
         if session is None:
             return False
@@ -418,7 +485,11 @@ class SessionManager:
             session.archived = db_session.archived
             session.owner = getattr(db_session, "owner", None)
             session.is_important = getattr(db_session, "is_important", False) or False
-            session.message_count = getattr(db_session, "message_count", session.message_count) or 0
+            session.message_count = (
+                db.query(DbChatMessage)
+                .filter(DbChatMessage.session_id == session_id)
+                .count()
+            )
             return True
         except Exception as e:
             logger.error(f"Error syncing session metadata {session_id}: {e}")
@@ -517,6 +588,12 @@ class SessionManager:
         """Permanently delete a session and all its messages."""
         db = SessionLocal()
         try:
+            try:
+                from src.session_image_cleanup import cleanup_session_images
+                cleanup_session_images(session_id, db=db)
+            except Exception as e:
+                logger.warning(f"Image cleanup failed while deleting session {session_id}: {e}")
+
             # Detach documents so they survive as orphans in the library
             db.query(DbDocument).filter(DbDocument.session_id == session_id).update(
                 {DbDocument.session_id: None}, synchronize_session=False

docker-compose.gpu-amd.yml

@@ -46,10 +46,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -67,9 +68,18 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
+      - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
+      - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
+      - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -125,12 +135,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.gpu-nvidia.yml

@@ -45,10 +45,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -66,9 +67,18 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
+      - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
+      - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
+      - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -128,12 +138,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docker-compose.yml

@@ -34,10 +34,11 @@ services:
       - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db}
       - AUTH_ENABLED=${AUTH_ENABLED:-true}
       - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false}
+      - COMPANION_BASE_URL=${COMPANION_BASE_URL:-}
       - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin}
       - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-}
       - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1}
-      - SECURE_COOKIES=${SECURE_COOKIES:-false}
+      - SECURE_COOKIES=${SECURE_COOKIES:-}
       - EMBEDDING_URL=${EMBEDDING_URL:-}
       - EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
       - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
@@ -55,9 +56,18 @@ services:
       - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
       - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
       - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
+      - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES}
       - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
       - GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
       - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}
+      - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
+      - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
+      - GOOGLE_OAUTH_REDIRECT_URI=${GOOGLE_OAUTH_REDIRECT_URI:-}
+      # Externally reachable origin for MCP OAuth callbacks. The container
+      # always listens on 7000 and cannot see the host port map above, so
+      # remote MCP OAuth needs this set whenever the browser reaches
+      # Odysseus on anything other than http://localhost:7000.
+      - OAUTH_REDIRECT_BASE_URL=${OAUTH_REDIRECT_BASE_URL:-}
       - TAVILY_API_KEY=${TAVILY_API_KEY:-}
       - SERPER_API_KEY=${SERPER_API_KEY:-}
       # PUID / PGID — the user/group the container drops to before
@@ -106,12 +116,17 @@ services:
           fi
           sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml
         fi
+        # Advisory: a settings file the migration cannot parse or rewrite must
+        # not be what stops searxng from booting. It explains itself on stderr
+        # and we carry on, letting searxng report anything genuinely wrong.
+        /usr/local/searxng/.venv/bin/python /tmp/migrate-searxng-settings.py /etc/searxng/settings.yml || true
         exec /usr/local/searxng/entrypoint.sh
     ports:
       - "127.0.0.1:8080:8080"
     volumes:
       - searxng-data:/etc/searxng
       - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z
+      - ./scripts/migrate_searxng_settings.py:/tmp/migrate-searxng-settings.py:ro,z
     environment:
       - SEARXNG_BASE_URL=http://localhost:8080/
       - SEARXNG_SECRET=${SEARXNG_SECRET:-}

docs/attachments.md

@@ -0,0 +1,85 @@
+# Attachment References and Upload Storage
+
+Odysseus stores uploaded bytes once under the configured upload directory and
+passes stable references through chat history, tools, and future artifact work.
+The goal is to avoid duplicating large inline media payloads in
+`chat_messages.content` or the SQLite FTS index.
+
+## Reference Shape
+
+Attachment references use this minimum shape:
+
+```json
+{
+  "type": "attachment_ref",
+  "attachment_id": "32hex-or-32hex.ext",
+  "name": "original-filename.png",
+  "mime": "image/png",
+  "size": 12345,
+  "checksum_sha256": "hex-digest",
+  "created_at": "2026-07-09T12:00:00"
+}
+```
+
+Optional fields such as `width`, `height`, `vision`, `vision_model`, and
+`gallery_id` may be present when the uploader or preprocessing path knows them.
+
+## Persistence
+
+The live model call may still receive provider-specific multimodal blocks for
+the current turn. Persistence is different:
+
+- `chat_messages.content` stores readable text plus compact attachment reference
+  lines, never raw `data:*;base64,...` upload bytes.
+- `chat_messages.metadata.attachments` stores structured attachment reference
+  metadata for UI reloads and future processing.
+- The SQLite FTS migration recreates chat-message FTS triggers so new rows do
+  not index inline media payloads, and it scrubs legacy rows that were already
+  indexed with data URLs.
+
+## Tool Access
+
+Agent/tool context receives upload entries as `attachment_ref` manifests with an
+`odysseus://attachment/<id>` URI and `read_policy: "owner_checked_upload"`.
+
+For compatibility with existing built-in tools, a local `path` may be included
+only after all of these checks pass:
+
+- the upload ID resolves through `UploadHandler.resolve_upload`;
+- the requested owner is allowed to read the upload;
+- the file remains inside the configured upload directory;
+- the file path is inside the tool-readable roots.
+
+External MCP/custom tools should treat the URI and attachment ID as the stable
+contract and request bytes through an owner-checked server path, not by assuming
+host filesystem layout.
+
+## Retention and Deletion
+
+Current retention behavior is conservative:
+
+- uploads are indexed in `uploads.json` with owner, checksum, MIME type, size,
+  and creation time;
+- admin cleanup first scans persisted chat metadata/content, document versions,
+  PDF source markers, gallery hashes, notes, and calendar records for live
+  references;
+- cleanup fails closed if that reference scan cannot complete, and the lower-level
+  cleanup API removes nothing unless it receives a complete reference snapshot;
+- expired, unreferenced uploads are removed during the completed scan, while
+  attachment-bearing writers must first take an owner-checked reservation that
+  serializes with deletion and refreshes the upload's access timestamp;
+- deliberate removal atomically drops matching `uploads.json` rows before deleting
+  the bytes and restores those rows if filesystem removal fails;
+- deleting a chat removes the chat rows but does not immediately delete shared
+  upload bytes, because the same upload may also be referenced by gallery items,
+  documents, duplicate-upload rows, or future artifact records.
+
+There is no distinct artifact table in the current schema. Artifact-like upload
+references persisted in chat or document text are covered by the canonical
+attachment-ID scan; any future artifact store must be added to reference discovery
+before cleanup is allowed to consider its uploads unreferenced.
+
+Cleanup and write reservations share the upload-index lock. This closes the
+scan/write/delete race in the documented single-worker deployment; a future
+multi-process deployment must add an inter-process lock or move lifecycle state
+into the database before enabling destructive cleanup in more than one worker.

docs/index.html

@@ -409,7 +409,7 @@
         <a href="#testimonials">Testimonials</a>
         <a href="#how">How it started</a>
         <a href="#start">Get started</a>
-        <a class="btn" href="https://github.com/pewdiepie-archdaemon/odysseus" target="_blank">
+        <a class="btn" href="https://github.com/odysseus-dev/odysseus" target="_blank">
           <svg width="16" height="16" viewBox="0 0 24 24" fill="currentColor"><path d="M12 .5C5.7.5.5 5.7.5 12c0 5.1 3.3 9.4 7.9 10.9.6.1.8-.2.8-.6v-2c-3.2.7-3.9-1.5-3.9-1.5-.5-1.3-1.3-1.7-1.3-1.7-1-.7.1-.7.1-.7 1.2.1 1.8 1.2 1.8 1.2 1 1.8 2.7 1.3 3.4 1 .1-.8.4-1.3.7-1.6-2.6-.3-5.3-1.3-5.3-5.7 0-1.3.5-2.3 1.2-3.1-.1-.3-.5-1.5.1-3.1 0 0 1-.3 3.3 1.2a11.5 11.5 0 0 1 6 0C17.3 4.7 18.3 5 18.3 5c.6 1.6.2 2.8.1 3.1.8.8 1.2 1.8 1.2 3.1 0 4.4-2.7 5.4-5.3 5.7.4.4.8 1.1.8 2.2v3.3c0 .4.2.7.8.6 4.6-1.5 7.9-5.8 7.9-10.9C23.5 5.7 18.3.5 12 .5z"/></svg>
           GitHub
         </a>
@@ -437,7 +437,7 @@
       </p>
       <div class="hero-cta">
         <a class="btn primary" href="#start">Get started</a>
-        <a class="btn" href="https://github.com/pewdiepie-archdaemon/odysseus" target="_blank">View on GitHub</a>
+        <a class="btn" href="https://github.com/odysseus-dev/odysseus" target="_blank">View on GitHub</a>
       </div>
 
     </div>
@@ -678,9 +678,9 @@
         <div class="eyebrow"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 17l6-5-6-5"/><path d="M12 19h8"/></svg>Get started</div>
         <h2 class="h" style="margin-bottom:6px;">Odysseus is yours.</h2>
         <p class="sub center" style="margin:0 auto;">It's open source and free. No sales team, no demo request, no Trojan horse.</p>
-        <div class="codeblock"><span><span class="prompt">$</span> git clone https://github.com/pewdiepie-archdaemon/odysseus.git &amp;&amp; cd odysseus</span><button class="copy-btn" data-copy="git clone https://github.com/pewdiepie-archdaemon/odysseus.git && cd odysseus">Copy</button></div>
+        <div class="codeblock"><span><span class="prompt">$</span> git clone https://github.com/odysseus-dev/odysseus.git &amp;&amp; cd odysseus</span><button class="copy-btn" data-copy="git clone https://github.com/odysseus-dev/odysseus.git && cd odysseus">Copy</button></div>
         <div>
-          <a class="btn primary" href="https://github.com/pewdiepie-archdaemon/odysseus" target="_blank" style="margin-top:14px;">View on GitHub</a>
+          <a class="btn primary" href="https://github.com/odysseus-dev/odysseus" target="_blank" style="margin-top:14px;">View on GitHub</a>
         </div>
         <div class="pill-row">
           <span class="pill">Self-hosted</span>

docs/security-ci.md

@@ -7,10 +7,9 @@ benefit.
 
 ## What runs, and why
 
-Most checks live in files under `.github/workflows/`. CodeQL is configured
-through GitHub's code scanning default setup, so it appears as a dynamic GitHub
-workflow instead of a checked-in workflow file. They run automatically; you do
-not start them.
+Most checks live in files under `.github/workflows/`. CodeQL uses the
+checked-in advanced configuration in `.github/workflows/codeql.yml`. They run
+automatically; you do not start them.
 
 | Check | What it protects against | Blocks a merge? |
 |---|---|---|
@@ -90,14 +89,9 @@ let the workflows run on one pull request first, then add them here.
 2. Turn on **Dependency graph** (usually on by default for public repos) -- this
    powers Dependency review and Dependabot.
 3. Turn on **Dependabot alerts** and **Dependabot security updates**.
-4. Under **Code scanning**, use **Set up -> Default** for CodeQL. GitHub then
-   runs CodeQL as a dynamic workflow without the fork-token limitations that
-   affect checked-in advanced workflows.
-
-   Do not also add a checked-in CodeQL workflow while default setup is enabled:
-   GitHub rejects advanced CodeQL uploads when default setup is active. If the
-   project later needs an advanced CodeQL workflow, disable default setup first
-   and keep only one CodeQL publishing path active.
+4. Under **Code scanning**, keep **Default setup** disabled. CodeQL is
+   configured by `.github/workflows/codeql.yml`; enabling default setup at the
+   same time causes GitHub to reject uploads from the checked-in workflow.
 
 ## Keeping it current
 

docs/setup.md

@@ -4,7 +4,7 @@ This page keeps the detailed install, deployment, troubleshooting, and configura
 
 ## Quick Start
 
-> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main).
+> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/odysseus-dev/odysseus/tree/main).
 
 Defaults work out of the box: clone, run, then configure models/search/email
 inside **Settings**. Only edit `.env` for deployment-level overrides like
@@ -20,7 +20,7 @@ pull request guidelines.
 
 ### Docker (recommended)
 ```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 cp .env.example .env       # optional, but recommended for explicit defaults
 docker compose up -d --build
@@ -38,7 +38,7 @@ only when you intentionally want LAN/reverse-proxy access.
 
 ### Native Linux / macOS
 ```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 python3 -m venv venv
 source venv/bin/activate
@@ -56,7 +56,7 @@ Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an
 M-series Mac, run Odysseus natively:
 
 ```bash
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 ./start-macos.sh
 ```
@@ -150,6 +150,79 @@ scripts/check-docker-gpu.sh --enable-nvidia-overlay
 # Full assisted setup — install toolkit, then enable overlay if passthrough works:
 scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
 ```
+#### Arch Linux NVIDIA Docker notes
+
+On Arch Linux, verify the host NVIDIA driver and Docker GPU passthrough before enabling the Odysseus NVIDIA overlay.
+
+Install the required packages:
+
+```bash
+sudo pacman -Syu
+sudo pacman -S docker docker-compose nvidia-container-toolkit nvidia-utils
+sudo systemctl enable --now docker
+```
+
+Configure Docker to use the NVIDIA container runtime:
+
+```bash
+sudo nvidia-ctk runtime configure --runtime=docker
+sudo systemctl restart docker
+```
+
+Verify the host GPU:
+
+```bash
+nvidia-smi
+```
+
+Verify Docker GPU passthrough:
+
+```bash
+docker run --rm --gpus all nvidia/cuda:12.9.0-base-ubuntu22.04 nvidia-smi
+```
+
+Then enable the Odysseus NVIDIA compose overlay:
+
+```env
+COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
+```
+
+Rebuild and verify the GPU inside the Odysseus container:
+
+```bash
+docker compose up -d --build
+docker compose exec odysseus nvidia-smi -L
+```
+
+For first-time local model testing on 8 GB laptop GPUs, start with GGUF/Q4 models on llama.cpp before trying GPTQ/AWQ models on vLLM or SGLang. This keeps the first run simpler while confirming GPU passthrough works.
+
+**WSL2 + snap Docker.** If the NVIDIA check fails with this error, Docker may be
+installed via snap:
+
+```text
+failed to fulfil mount request: open /usr/lib/wsl/lib/libdxcore.so: no such file or directory
+```
+
+Check with `snap list docker` or:
+
+```bash
+docker info --format '{{.DockerRootDir}}'
+```
+
+A Docker root under `/var/snap/docker/` means snap confinement can prevent
+Docker from seeing WSL2's `/usr/lib/wsl/lib` GPU libraries even when the files
+exist on the host. Reinstalling or reconfiguring `nvidia-container-toolkit` will
+not fix that. Remove snap Docker, install the official apt-based Docker Engine
+([Docker docs](https://docs.docker.com/engine/install/ubuntu/)), then configure
+the NVIDIA runtime again:
+
+```bash
+sudo snap remove docker
+sudo nvidia-ctk runtime configure --runtime=docker
+sudo systemctl restart docker
+```
+
+Then re-run `scripts/check-docker-gpu.sh`.
 
 Safety notes:
 - The app never installs host GPU runtime automatically.
@@ -236,6 +309,32 @@ container. Cookbook **Serve** is a separate workflow for serving downloaded
 models through Odysseus/llama.cpp, so Windows users with an existing Ollama
 install usually only need to add the endpoint in Settings.
 
+**Tool calls not firing on a manually-added Ollama `/v1` endpoint.** By
+design, a local Ollama `/v1` endpoint defaults to the conservative
+text-based (fenced-block) tool-calling path rather than native structured
+tool calls, since some locally-served models mishandle native schemas (see
+#1567). This is correct for most local setups, but if you know your specific
+model reliably supports native tool calling (check `ollama show <model>` for
+`tools` under Capabilities), you can opt that endpoint in explicitly. There
+is currently no UI control for this on manually-added endpoints (see #5192);
+the flag can still be set directly against the existing API, from a browser
+console on an authenticated admin session:
+
+```js
+fetch('/api/model-endpoints/<endpoint-id>', {
+  method: 'PATCH',
+  credentials: 'same-origin',
+  headers: {'Content-Type': 'application/json'},
+  body: JSON.stringify({supports_tools: true})
+}).then(r => r.json()).then(console.log)
+```
+
+Find `<endpoint-id>` by inspecting the `/api/model-endpoints` response (or
+your browser's network tab while Settings loads the endpoint list). Send
+`supports_tools: false` to disable native structured tool calls and force the
+conservative fenced/text path, or `supports_tools: null` to return the endpoint
+to the Auto heuristic.
+
 **Useful checks.**
 
 ```bash
@@ -257,7 +356,7 @@ do not run on macOS. MLX-only models are not served by Odysseus.
 server; safe to re-run):
 
 ```powershell
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
 ```
@@ -265,7 +364,7 @@ powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
 Or do it by hand:
 
 ```powershell
-git clone https://github.com/pewdiepie-archdaemon/odysseus.git
+git clone https://github.com/odysseus-dev/odysseus.git
 cd odysseus
 py -3.11 -m venv venv
 venv\Scripts\Activate.ps1
@@ -342,10 +441,19 @@ A grab-bag of small gotchas that otherwise turn into long debugging sessions.
 | Package | Feature unlocked |
 |---------|-----------------|
 | `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
+| `kokoro`, `soundfile` | Local Kokoro-82M text-to-speech on a CUDA GPU. The pinned Kokoro release supports Odysseus installs on Python 3.11-3.12; these packages are intentionally skipped on Python 3.13+ (including the Python 3.14 container image). |
 | `ddgs` | DuckDuckGo as a search provider option. |
 | `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
 | `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
 
+Install the optional set only when you need these features:
+
+```bash
+pip install -r requirements-optional.txt
+```
+
+The default Docker image currently uses Python 3.14, while Kokoro 0.9.4 declares Python `>=3.10,<3.13`. Odysseus itself continues to support Python 3.11+, but this pinned optional local-TTS feature requires a native Python 3.11 or 3.12 environment. Kokoro declares `torch`, but the local provider only activates when that torch build has CUDA and a GPU is visible; install the CUDA build appropriate for your host. Browser and configured endpoint TTS remain available on Python 3.13+ and in the container image.
+
 ### Faster, reproducible installs with uv (optional)
 [uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the
 venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use:
@@ -376,7 +484,7 @@ Odysseus is a self-hosted workspace with powerful local tools: shell access, fil
 
 - Keep `AUTH_ENABLED=true` for any network-accessible deployment.
 - Keep `LOCALHOST_BYPASS=false` outside local development.
-- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
+- Leave `SECURE_COOKIES` unset unless you need to override it: session cookies are marked `Secure` whenever the request arrives over HTTPS. Use `SECURE_COOKIES=true` to force it on for a proxy whose scheme Odysseus cannot see, or `SECURE_COOKIES=false` to force it off while you still serve plain HTTP alongside HTTPS.
 - Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
 - Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
 - Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
@@ -387,6 +495,14 @@ Odysseus is a self-hosted workspace with powerful local tools: shell access, fil
 - Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
 - Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
 
+> **Upgrading an existing install:** `SECURE_COOKIES` used to default to
+> `false`, so an install set up before scheme derivation may still carry
+> `SECURE_COOKIES=false` in its own `.env`. That explicit value stays
+> authoritative, so HTTPS logins keep getting a non-`Secure` session cookie.
+> Pulling this change updates the tracked Compose files, but nothing rewrites
+> your `.env` — drop the line from it unless you deliberately serve plain HTTP
+> alongside HTTPS and want the escape hatch.
+
 ### Private or proxied deployments
 Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
 
@@ -395,9 +511,162 @@ Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and th
 3. Put the authenticated Odysseus web/API entrypoint behind that layer.
 4. Keep raw service and model ports internal-only.
 
-Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
+Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true` and `LOCALHOST_BYPASS=false`. Any proxy that forwards `X-Forwarded-Proto: https` gets `Secure` session cookies without configuration, so `SECURE_COOKIES` only needs setting when you want to override that — force it on for a proxy that forwards no scheme at all, or off while you still serve plain HTTP.
 `ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry.
 
+#### Faster over the network: HTTP/2
+
+The frontend is raw ES modules with no bundler, so a page load is a few hundred
+small same-origin requests. Over HTTP/1.1 browsers typically allow only a small
+number of concurrent connections per host (commonly around six), so many of
+those requests are serialized across multiple round trips. On localhost that
+costs almost nothing. Over a LAN, VPN, or remote link it can become a major
+part of load time, especially as latency increases.
+
+HTTP/2 multiplexes them onto one connection and the serialisation disappears.
+Odysseus needs no changes for this — uvicorn keeps speaking HTTP/1.1 on
+loopback and the proxy speaks HTTP/2 to the browser. Mainstream browsers
+negotiate HTTP/2 for normal web pages over TLS; they do not use the cleartext
+h2c mode here, so browser-facing HTTP/2 requires a certificate. The
+`--ssl-certfile` route in *HTTPS + LAN/Tailscale exposure* above gives you
+HTTPS but not HTTP/2 — uvicorn does not speak it.
+
+**1. Install Caddy.** See the [install docs](https://caddyserver.com/docs/install)
+for your platform; on macOS, `brew install caddy`.
+
+**2. Write a `Caddyfile`.** Pick the block that matches how you reach the
+machine. Replace `7000` if Odysseus listens elsewhere — the macOS start script
+uses `7860`.
+
+Public domain, Caddy obtains and renews the certificate itself:
+
+```
+odysseus.example.com {
+	reverse_proxy 127.0.0.1:7000
+}
+```
+
+Tailscale, no public DNS needed — `tailscale cert` issues a browser-trusted
+certificate for a tailnet name and writes `<domain>.crt` and `<domain>.key`:
+
+```bash
+tailscale cert myhost.tailnet-name.ts.net
+```
+
+```
+myhost.tailnet-name.ts.net {
+	tls /path/to/myhost.tailnet-name.ts.net.crt /path/to/myhost.tailnet-name.ts.net.key
+	reverse_proxy 127.0.0.1:7000
+}
+```
+
+LAN with your own certificate — same shape, your own files:
+
+```
+odysseus.lan {
+	tls /path/to/cert.pem /path/to/key.pem
+	reverse_proxy 127.0.0.1:7000
+}
+```
+
+Give `tls` absolute paths: a service starts in a working directory you did not
+choose. If port 443 is already taken, append a port to the site address
+(`odysseus.example.com:8443`) and use it in the URL. That alone does not free
+port 80 — Caddy still binds it for the HTTP-to-HTTPS redirect, and fails to
+start with `listen tcp :80: bind: address already in use` if something else
+holds it. Turn the redirect off with a global block at the top of the file:
+
+```
+{
+	auto_https disable_redirects
+}
+```
+
+**3. Run it in the foreground first:**
+
+```bash
+caddy run --config ./Caddyfile
+```
+
+Once that works, run it as a service:
+
+```bash
+brew services start caddy          # macOS — reads $(brew --prefix)/etc/Caddyfile, not ./Caddyfile
+sudo systemctl enable --now caddy  # Linux, if your package installed the unit
+```
+
+Odysseus's own service is unchanged; the proxy runs alongside it. Under Docker,
+run the proxy as another container, or on the host pointing at the published
+port.
+
+**4. Point Odysseus at the new origin** in `.env`, then restart it.
+
+A proxy that exposes the HTTPS request scheme to Odysseus needs no `SECURE_COOKIES` setting. Only force it on when the proxy cannot expose that scheme:
+
+```bash
+# only if the proxy cannot expose the external HTTPS scheme to Odysseus:
+SECURE_COOKIES=true
+# only if you use remote MCP servers with OAuth:
+OAUTH_REDIRECT_BASE_URL=https://odysseus.example.com
+```
+
+Gmail OAuth needs nothing here when the proxy runs on the same host: the
+redirect URI is built from the incoming request, and uvicorn rewrites the
+scheme from `X-Forwarded-Proto` for proxies it trusts — by default only
+`127.0.0.1`. A proxy in a separate container or on another machine is not
+trusted, so pin the URI there:
+
+```bash
+GOOGLE_OAUTH_REDIRECT_URI=https://odysseus.example.com/api/email/oauth/google/callback
+```
+
+(uvicorn's own `FORWARDED_ALLOW_IPS` widens that trust, but it has to be in the
+environment uvicorn starts with — `.env` is read by the app afterwards, too
+late for it to take effect.)
+
+**5. Confirm HTTP/2 is really on:**
+
+```bash
+curl -s -o /dev/null -w '%{http_version}\n' https://odysseus.example.com/
+# 2
+```
+
+The status code is not the thing to check here — a logged-out request redirects
+to the login page, so `curl -I` shows `HTTP/2 302`, and the `HTTP/2` prefix is
+the part that matters. The browser reports the same in the Network panel's
+Protocol column (`h2`); in Chrome and Firefox that column is hidden until you
+enable it by right-clicking the column headers.
+
+Three things bite when moving an existing install behind TLS:
+
+- Leave `SECURE_COOKIES` unset when Odysseus can see the external HTTPS scheme;
+  the cookie then follows the request automatically. If your proxy cannot expose
+  that scheme, set `SECURE_COOKIES=true` **at the same time** you stop serving
+  plain HTTP, not before. An explicit `true` applies to every login, so while an
+  HTTP entrypoint is still reachable the browser will reject the `Secure` cookie
+  there and login will appear to loop.
+- `OAUTH_REDIRECT_BASE_URL` defaults to `http://localhost:7000`. Unlike the
+  Gmail redirect URI it cannot be derived from a request — it is registered
+  with each MCP authorization server up front — so set it to the external
+  origin if you use remote MCP servers over OAuth.
+- Odysseus sends `Strict-Transport-Security` once it sees `X-Forwarded-Proto:
+  https`. HSTS applies to the whole hostname and ignores the port, so any other
+  plain-HTTP service on that same hostname becomes unreachable in browsers that
+  have visited Odysseus. Give Odysseus its own hostname, or strip the header at
+  the proxy (`header_down -Strict-Transport-Security` in Caddy).
+
+Server-sent events are not buffered by this configuration, so chat streaming
+arrives token by token; add `flush_interval -1` inside the `reverse_proxy`
+block if you want that pinned explicitly. nginx needs `proxy_buffering off;`
+for the same reason.
+
+Changing the external origin also affects state scoped to it. Service workers
+and their caches are origin-scoped, so moving to a different origin starts with
+a cold load. Cookies follow their own domain/path/security rules rather than
+being port-scoped: changing the hostname normally requires a new login, while
+changing only the scheme or port does not by itself guarantee that existing
+cookies disappear.
+
 Common internal-only ports from the default docs/compose setup:
 
 | Port | Service |
@@ -428,7 +697,7 @@ Key settings:
 | `AUTH_ENABLED` | `true` | Enable/disable login |
 | `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
 | `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. |
-| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
+| `SECURE_COOKIES` | derived from the request scheme | Marks session cookies `Secure` on HTTPS requests. Set true to force it on, false to force it off. |
 | `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
 | `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
 | `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |

launch-windows.ps1

@@ -163,6 +163,10 @@ if (Test-Path $cudaBase) {
 }
 
 # 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
+# -Port only reaches uvicorn as a flag. Everything that builds a URL for this
+# instance - internal_api_base(), companion pairing, the MCP OAuth callback -
+# reads APP_PORT, so set it too or they all assume 7000.
+$env:APP_PORT = $Port
 Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port)
 Write-Host "Press Ctrl+C to stop."
 Write-Host ""

licenses/KaTeX-MIT-LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013-2020 Khan Academy and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

licenses/Mermaid-MIT-LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 - 2022 Knut Sveidqvist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mcp_servers/email_server.py

@@ -24,6 +24,7 @@ from pathlib import Path
 from datetime import datetime, timedelta
 import uuid
 from contextvars import ContextVar
+from urllib.parse import parse_qs, unquote, urlparse
 
 from mcp.server import Server
 from mcp.server.stdio import stdio_server
@@ -129,20 +130,36 @@ def _mcp_owner_required(rows: list[dict] | None = None) -> bool:
     return _has_owner_scoped_accounts(rows)
 
 
-def _load_email_writing_style() -> str:
-    """Return the existing Settings > Email > Writing Style value."""
+def _load_email_writing_style(account: str | None = None) -> str:
+    """Return the saved Settings > Email > Writing Style value.
+
+    Prefer the selected account's style when one exists; fall back to the
+    legacy global style so older installs keep behaving as before.
+    """
     try:
         settings_path = DATA_DIR / "settings.json"
         if not settings_path.exists():
             return ""
         settings = json.loads(settings_path.read_text(encoding="utf-8"))
+        account_id = ""
+        if account:
+            try:
+                cfg = _load_config(account)
+                account_id = str(cfg.get("account_id") or account or "").strip()
+            except Exception:
+                account_id = str(account or "").strip()
+        by_account = settings.get("email_writing_styles_by_account") or {}
+        if account_id and isinstance(by_account, dict):
+            style = by_account.get(account_id)
+            if isinstance(style, str) and style.strip():
+                return style.strip()
         return str(settings.get("email_writing_style") or "").strip()
     except Exception:
         return ""
 
 
-def _writing_style_guidance() -> str:
-    style = _load_email_writing_style()
+def _writing_style_guidance(account: str | None = None) -> str:
+    style = _load_email_writing_style(account)
     if not style:
         return (
             "No saved writing style is configured in Settings > Email > Writing Style. "
@@ -509,6 +526,251 @@ def _decode_header(raw):
         return "".join(decoded)
 
 
+def _uid_from_fetch_meta(meta_b: bytes) -> str:
+    m = re.search(rb"UID\s+(\d+)", meta_b or b"")
+    return m.group(1).decode("ascii", errors="ignore") if m else ""
+
+
+def _parse_list_unsubscribe_header(value: str | None) -> list[dict]:
+    raw = str(value or "").strip()
+    if not raw:
+        return []
+    pieces = re.findall(r"<([^>]+)>", raw)
+    if not pieces:
+        pieces = [p.strip() for p in raw.split(",") if p.strip()]
+    out: list[dict] = []
+    seen = set()
+    for piece in pieces:
+        target = piece.strip().strip("<>").strip()
+        if not target:
+            continue
+        parsed = urlparse(target)
+        scheme = parsed.scheme.lower()
+        key = target.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        if scheme == "mailto":
+            addr = unquote(parsed.path or "").strip()
+            if not addr or "\r" in addr or "\n" in addr:
+                continue
+            query = parse_qs(parsed.query or "", keep_blank_values=True)
+            subject = unquote((query.get("subject") or ["unsubscribe"])[0] or "unsubscribe")
+            body = unquote((query.get("body") or ["unsubscribe"])[0] or "unsubscribe")
+            subject = re.sub(r"[\r\n]+", " ", subject).strip() or "unsubscribe"
+            body = re.sub(r"[\r\n]+", "\n", body).strip() or "unsubscribe"
+            out.append({
+                "kind": "mailto",
+                "target": addr,
+                "subject": subject[:200],
+                "body": body[:1000],
+                "executable": True,
+            })
+        elif scheme in {"http", "https"}:
+            out.append({
+                "kind": "url",
+                "target": target,
+                "executable": False,
+            })
+    return out
+
+
+def _email_unsubscribe_candidate_from_msg(msg, uid: str, folder: str) -> dict | None:
+    sender = _decode_header(msg.get("From", ""))
+    sender_name, sender_addr = email.utils.parseaddr(sender)
+    subject = _decode_header(msg.get("Subject", "(no subject)"))
+    list_id = _decode_header(msg.get("List-Id", ""))
+    precedence = (msg.get("Precedence") or "").strip().lower()
+    auto_submitted = (msg.get("Auto-Submitted") or "").strip().lower()
+    methods = _parse_list_unsubscribe_header(msg.get("List-Unsubscribe"))
+    if not methods:
+        return None
+    reasons: list[str] = ["has unsubscribe header"]
+    score = 45
+    if list_id:
+        score += 20
+        reasons.append("mailing-list header")
+    if precedence in {"bulk", "junk", "list"}:
+        score += 20
+        reasons.append(f"precedence={precedence}")
+    if auto_submitted and auto_submitted != "no":
+        score += 10
+        reasons.append(f"auto-submitted={auto_submitted}")
+    if re.search(r"\b(unsubscribe|newsletter|sale|discount|offer|promo|limited time)\b", (subject or "").lower()):
+        score += 10
+        reasons.append("promotional subject")
+    executable = [m for m in methods if m.get("executable")]
+    return {
+        "uid": str(uid),
+        "folder": folder,
+        "message_id": (msg.get("Message-ID") or "").strip(),
+        "subject": subject,
+        "from_name": sender_name or sender_addr,
+        "from_address": sender_addr,
+        "list_id": list_id,
+        "score": min(score, 100),
+        "reasons": reasons[:5],
+        "methods": methods,
+        "can_execute": bool(executable),
+        "recommended_method": executable[0] if executable else methods[0],
+    }
+
+
+def _unsubscribe_candidate_dedupe_key(candidate: dict) -> tuple[str, str, str]:
+    list_id = str(candidate.get("list_id") or "").strip().lower()
+    method = candidate.get("recommended_method") or {}
+    method_kind = str(method.get("kind") or "").strip().lower()
+    method_target = str(method.get("target") or "").strip().lower()
+    sender = str(candidate.get("from_address") or "").strip().lower()
+    if list_id:
+        return ("list", list_id, method_target or sender)
+    if method_target:
+        return ("method", method_kind, method_target)
+    return ("sender", sender, str(candidate.get("subject") or "").strip().lower())
+
+
+def _dedupe_unsubscribe_candidates(candidates: list[dict]) -> list[dict]:
+    deduped: dict[tuple[str, str, str], dict] = {}
+    for candidate in candidates or []:
+        key = _unsubscribe_candidate_dedupe_key(candidate)
+        existing = deduped.get(key)
+        if not existing:
+            copy = dict(candidate)
+            copy["duplicate_count"] = 1
+            copy["duplicate_uids"] = [str(candidate.get("uid") or "")]
+            deduped[key] = copy
+            continue
+        existing["duplicate_count"] = int(existing.get("duplicate_count") or 1) + 1
+        uid = str(candidate.get("uid") or "")
+        if uid:
+            existing.setdefault("duplicate_uids", []).append(uid)
+        if int(candidate.get("score") or 0) > int(existing.get("score") or 0):
+            keep_count = existing.get("duplicate_count")
+            keep_uids = existing.get("duplicate_uids")
+            replacement = dict(candidate)
+            replacement["duplicate_count"] = keep_count
+            replacement["duplicate_uids"] = keep_uids
+            deduped[key] = replacement
+    return list(deduped.values())
+
+
+def _scan_unsubscribe_candidates(folder="INBOX", account=None, limit=25, max_scan=150) -> dict:
+    limit = max(1, min(int(limit or 25), 100))
+    max_scan = max(limit, min(int(max_scan or 150), 500))
+    folder = folder or "INBOX"
+    candidates: list[dict] = []
+    conn = _imap_connect(account)
+    try:
+        status, _ = conn.select(_q(folder), readonly=True)
+        if status != "OK":
+            return {"success": False, "error": f"Folder not found: {folder}", "candidates": []}
+        status, data = conn.uid("SEARCH", None, "ALL")
+        if status != "OK" or not data or not data[0]:
+            return {"success": True, "candidates": [], "total": 0, "scanned": 0, "folder": folder}
+        uids = []
+        for raw_uid in data[0].split():
+            try:
+                uids.append(int(raw_uid))
+            except Exception:
+                continue
+        uids = sorted(uids, reverse=True)[:max_scan]
+        if not uids:
+            return {"success": True, "candidates": [], "total": 0, "scanned": 0, "folder": folder}
+        status, msg_data = conn.uid("FETCH", _b(",".join(str(u) for u in uids)), "(UID RFC822.HEADER)")
+    finally:
+        try:
+            conn.logout()
+        except Exception:
+            pass
+    if status != "OK":
+        return {"success": False, "error": "Failed to fetch email headers", "candidates": []}
+    for item in msg_data or []:
+        if not isinstance(item, tuple) or len(item) < 2:
+            continue
+        meta_b = item[0] if isinstance(item[0], bytes) else str(item[0]).encode()
+        uid = _uid_from_fetch_meta(meta_b)
+        if not uid:
+            continue
+        try:
+            msg = email.message_from_bytes(item[1] or b"")
+        except Exception:
+            continue
+        candidate = _email_unsubscribe_candidate_from_msg(msg, uid, folder)
+        if candidate:
+            candidates.append(candidate)
+    raw_total = len(candidates)
+    candidates = _dedupe_unsubscribe_candidates(candidates)
+    candidates.sort(key=lambda c: (int(c.get("score") or 0), int(c.get("duplicate_count") or 1), int(c.get("uid") or 0)), reverse=True)
+    return {
+        "success": True,
+        "candidates": candidates[:limit],
+        "total": len(candidates),
+        "raw_total": raw_total,
+        "scanned": len(uids),
+        "folder": folder,
+        "account": account or "",
+    }
+
+
+def _unsubscribe_email(uid, folder="INBOX", account=None, method_index=0, allow_web=False) -> dict:
+    uid = str(uid or "").strip()
+    if not uid:
+        return {"success": False, "error": "uid is required"}
+    conn = _imap_connect(account)
+    try:
+        status, _ = conn.select(_q(folder), readonly=True)
+        if status != "OK":
+            return {"success": False, "error": f"Folder not found: {folder}"}
+        status, msg_data = conn.uid("FETCH", _b(uid), "(UID RFC822.HEADER)")
+    finally:
+        try:
+            conn.logout()
+        except Exception:
+            pass
+    if status != "OK" or not msg_data:
+        return {"success": False, "error": f"Email not found: {uid}"}
+    raw_header = b""
+    for item in msg_data or []:
+        if isinstance(item, tuple) and len(item) >= 2:
+            raw_header = item[1] or b""
+            break
+    msg = email.message_from_bytes(raw_header)
+    candidate = _email_unsubscribe_candidate_from_msg(msg, uid, folder)
+    if not candidate:
+        return {"success": False, "error": "No List-Unsubscribe header found"}
+    methods = candidate.get("methods") or []
+    method_index = int(method_index or 0)
+    method = methods[method_index] if 0 <= method_index < len(methods) else (candidate.get("recommended_method") or methods[0])
+    if method.get("kind") == "url":
+        return {
+            "success": False,
+            "requires_browser": True,
+            "url": method.get("target"),
+            "candidate": candidate,
+            "instructions": (
+                "This unsubscribe is a web link. Ask the user for approval, then use the browser/web tool "
+                "to open the exact URL and complete the unsubscribe page. Do not fetch unrelated links."
+            ),
+        }
+    if method.get("kind") != "mailto" or not method.get("executable"):
+        return {"success": False, "error": "Unsupported unsubscribe method", "candidate": candidate}
+    result = _send_email(
+        to=method.get("target"),
+        subject=method.get("subject") or "unsubscribe",
+        body=method.get("body") or "unsubscribe",
+        account=account,
+    )
+    if "error" in result:
+        return {"success": False, "error": result["error"], "candidate": candidate}
+    return {
+        "success": True,
+        "method": method,
+        "candidate": candidate,
+        "send_result": result,
+        "pending": bool(result.get("pending")),
+    }
+
+
 def _extract_text(msg):
     """Extract plain text body from email message."""
     if msg.is_multipart():
@@ -1540,14 +1802,12 @@ async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account
         from src.endpoint_resolver import (
             resolve_endpoint,
             resolve_utility_fallback_candidates,
-            resolve_chat_fallback_candidates,
         )
         from src.llm_core import llm_call_async_with_fallback
     except Exception as exc:
         return {"error": f"AI reply helpers unavailable: {exc}"}
 
-    settings = _load_settings()
-    style = settings.get("email_writing_style", "")
+    style = _load_email_writing_style(account)
     system_prompt = _EMAIL_REPLY_SYS_PROMPT_BASE
     if style:
         system_prompt += f"\n\nWRITING STYLE TO MATCH:\n{style}"
@@ -1582,13 +1842,6 @@ async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account
         utility_fallbacks = resolve_utility_fallback_candidates() or []
     for cand in utility_fallbacks:
         _add(*cand)
-    try:
-        chat_fallbacks = resolve_chat_fallback_candidates(owner=None) or []
-    except TypeError:
-        chat_fallbacks = resolve_chat_fallback_candidates() or []
-    for cand in chat_fallbacks:
-        _add(*cand)
-
     if not candidates:
         return {"error": "No LLM endpoint configured for AI reply"}
 
@@ -1890,6 +2143,45 @@ async def list_tools() -> list[Tool]:
                 "required": [],
             },
         ),
+        Tool(
+            name="scan_email_unsubscribes",
+            description=(
+                "Scan recent email headers for likely spam/newsletter unsubscribe candidates. "
+                "Returns reviewable candidates with UID, sender, subject, score, reasons, and "
+                "List-Unsubscribe methods. This does not unsubscribe anything. For mailto "
+                "methods, use unsubscribe_email after user approval. For web URL methods, use "
+                "browser/web tools after user approval to open the exact URL and complete the page."
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "folder": {"type": "string", "description": "IMAP folder to scan", "default": "INBOX"},
+                    "limit": {"type": "integer", "description": "Maximum candidates to return", "default": 25},
+                    "max_scan": {"type": "integer", "description": "How many newest messages to inspect", "default": 150},
+                    **ACCOUNT_PROP,
+                },
+                "required": [],
+            },
+        ),
+        Tool(
+            name="unsubscribe_email",
+            description=(
+                "Execute one approved unsubscribe action for an email UID. Supports safe mailto "
+                "List-Unsubscribe directly. If the selected method is a web URL, this returns "
+                "requires_browser with the exact URL; use browser/web tools only after user approval."
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "uid": {"type": "string", "description": "Email UID from scan_email_unsubscribes/list_emails"},
+                    "folder": {"type": "string", "description": "IMAP folder", "default": "INBOX"},
+                    "method_index": {"type": "integer", "description": "Unsubscribe method index from scan_email_unsubscribes", "default": 0},
+                    "allow_web": {"type": "boolean", "description": "Return web unsubscribe URL instructions when the method is URL", "default": False},
+                    **ACCOUNT_PROP,
+                },
+                "required": ["uid"],
+            },
+        ),
         Tool(
             name="download_attachment",
             description=(
@@ -2264,6 +2556,69 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
                 lines.append(line)
             return [TextContent(type="text", text="\n\n".join(lines))]
 
+        elif name == "scan_email_unsubscribes":
+            try:
+                result = _scan_unsubscribe_candidates(
+                    folder=arguments.get("folder", "INBOX"),
+                    account=acct,
+                    limit=arguments.get("limit", 25),
+                    max_scan=arguments.get("max_scan", 150),
+                )
+            except Exception as e:
+                return [TextContent(type="text", text=f"Unsubscribe scan failed: {e}")]
+            if not result.get("success"):
+                return [TextContent(type="text", text=f"Unsubscribe scan failed: {result.get('error', 'unknown error')}")]
+            candidates = result.get("candidates") or []
+            if not candidates:
+                return [TextContent(type="text", text=f"No unsubscribe candidates found in {result.get('scanned', 0)} recent emails.")]
+            lines = [
+                f"Found {len(candidates)} unsubscribe candidate(s) from {result.get('scanned', 0)} recent emails.",
+                "Review these with the user before executing. Mailto methods can use unsubscribe_email; URL methods require browser/web tools after approval.\n",
+            ]
+            for i, cand in enumerate(candidates, 1):
+                lines.append(
+                    f"{i}. **{cand.get('subject') or '(no subject)'}**\n"
+                    f"   From: {cand.get('from_name') or cand.get('from_address') or ''} ({cand.get('from_address') or ''})\n"
+                    f"   UID: {cand.get('uid')}  Folder: {cand.get('folder')}\n"
+                    f"   Score: {cand.get('score')}  Matching emails: {cand.get('duplicate_count', 1)}  Reasons: {', '.join(cand.get('reasons') or [])}"
+                )
+                for j, method in enumerate(cand.get("methods") or []):
+                    if method.get("kind") == "mailto":
+                        lines.append(f"   Method {j}: mailto {method.get('target')} (executable via unsubscribe_email)")
+                    elif method.get("kind") == "url":
+                        lines.append(f"   Method {j}: web URL {method.get('target')} (use browser/web tools after approval)")
+            return [TextContent(type="text", text="\n".join(lines))]
+
+        elif name == "unsubscribe_email":
+            result = _unsubscribe_email(
+                uid=arguments.get("uid"),
+                folder=arguments.get("folder", "INBOX"),
+                account=acct,
+                method_index=arguments.get("method_index", 0),
+                allow_web=bool(arguments.get("allow_web", False)),
+            )
+            if result.get("requires_browser"):
+                return [TextContent(
+                    type="text",
+                    text=(
+                        "Web unsubscribe requires browser/web navigation.\n"
+                        f"URL: {result.get('url')}\n"
+                        f"{result.get('instructions')}"
+                    ),
+                )]
+            if not result.get("success"):
+                return [TextContent(type="text", text=f"Unsubscribe failed: {result.get('error', 'unknown error')}")]
+            method = result.get("method") or {}
+            if result.get("pending"):
+                return [TextContent(
+                    type="text",
+                    text=(
+                        f"Unsubscribe email staged for approval to {method.get('target')}. "
+                        "Nothing has been sent until the user approves the pending email."
+                    ),
+                )]
+            return [TextContent(type="text", text=f"Unsubscribe email sent to {method.get('target')}.")]
+
         elif name == "download_attachment":
             uid = arguments.get("uid")
             index = arguments.get("index")

mcp_servers/image_gen_server.py

@@ -81,7 +81,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
             if not model_spec:
                 return [TextContent(type="text", text="Error: No image model found. Configure one in Admin.")]
 
-        url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec)
+        try:
+            url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec, model_type="image")
+        except ValueError:
+            _lower_model_spec = model_spec.lower()
+            if not any(_name in _lower_model_spec for _name in ("gpt-image", "dall-e")):
+                raise
+            url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec)
 
         is_gpt_image = "gpt-image" in model_id.lower()
         base_url = url.replace("/chat/completions", "").replace("/v1/messages", "").rstrip("/")

mcp_servers/memory_server.py

@@ -17,6 +17,8 @@ from mcp.types import Tool, TextContent
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
+from src.memory import MemoryStoreUnreadable
+
 server = Server("memory")
 
 # Late-initialized managers (set during first tool call)
@@ -29,6 +31,10 @@ _OWNER_SCOPE_ERROR = (
     "Error: Memory MCP owner is not configured for an owner-scoped memory store. "
     "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
 )
+_UNREADABLE_STORE_ERROR = (
+    "Error: Memory store is temporarily unreadable — nothing was saved. "
+    "Repair or restore memory.json, then retry."
+)
 
 
 def _configured_owner() -> str | None:
@@ -51,9 +57,21 @@ def _owner_scoped_store(entries: list[dict]) -> bool:
     return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
 
 
-def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
-    """Return configured owner, all entries, visible entries, and optional error."""
-    entries = _memory_manager.load_all()
+def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]:
+    """Return configured owner, all entries, visible entries, and optional error.
+
+    ``for_update=True`` is for read-modify-write callers. They save the ``all
+    entries`` list back, so an unreadable store must be reported as an error
+    instead of degrading to ``[]`` — otherwise the save writes their one new
+    entry over the whole store (issue #5673).
+    """
+    if for_update:
+        try:
+            entries = _memory_manager.load_all_for_update()
+        except MemoryStoreUnreadable as e:
+            return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})"
+    else:
+        entries = _memory_manager.load_all()
     owner = _configured_owner()
     if owner is None and _owner_scoped_store(entries):
         return None, entries, [], _OWNER_SCOPE_ERROR
@@ -161,7 +179,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
         category = arguments.get("category", "fact")
         if not text:
             return _text_result("Error: Memory text cannot be empty")
-        owner, memories, _visible, scope_error = _scope_entries()
+        owner, memories, _visible, scope_error = _scope_entries(for_update=True)
         if scope_error:
             return _text_result(scope_error)
         entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)

package.json

@@ -1,7 +1,7 @@
 {
   "repository": {
     "type": "git",
-    "url": "https://github.com/pewdiepie-archdaemon/odysseus.git"
+    "url": "https://github.com/odysseus-dev/odysseus.git"
   },
   "devDependencies": {
     "@antithesishq/bombadil": "^0.6.1"

requirements-optional.txt

@@ -12,6 +12,16 @@
 # GPU-accelerated transcription — it's auto-detected, CPU is used otherwise.
 faster-whisper
 
+# Local text-to-speech via Kokoro-82M for the "local" TTS provider.
+# Kokoro 0.9.4 declares Python >=3.10,<3.13; Odysseus itself requires 3.11+,
+# so pip installs these extras on 3.11-3.12 and deliberately skips them on
+# Python 3.13+ (including the Python 3.14 container image). Kokoro declares
+# torch; the local provider still
+# requires a CUDA-enabled torch build and GPU at runtime. SoundFile is separate
+# in Kokoro's official install instructions and is not a transitive dependency.
+kokoro==0.9.4; python_version >= "3.11" and python_version < "3.13"
+soundfile; python_version >= "3.11" and python_version < "3.13"
+
 # DuckDuckGo as a search provider option.
 # Install if you want DDG in the search-provider dropdown.
 # Alternatives: SearXNG, Brave, Tavily, Serper, Google PSE.

requirements.txt

@@ -38,7 +38,10 @@ python-dateutil
 caldav
 cryptography
 bcrypt
-mcp
+# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a
+# breaking rewrite, so keep fresh installs on the maintained v1 line until the
+# servers are migrated together.
+mcp<2
 pyotp
 qrcode[pil]
 croniter

routes/admin_wipe/__init__.py

@@ -0,0 +1,5 @@
+"""Admin wipe route domain package (slice 2h, #4082/#4071).
+
+Contains admin_wipe_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/admin_wipe_routes.py re-exports from here.
+"""

routes/admin_wipe/admin_wipe_routes.py

@@ -0,0 +1,176 @@
+"""Admin Danger Zone — per-category wipes.
+
+Each endpoint is admin-only and truncates exactly one domain so the
+user can selectively reset memory / skills / notes / etc. without
+nuking everything. The catch-all `chats` endpoint mirrors the
+existing /api/sessions/all so the Danger Zone speaks one URL pattern.
+
+URL shape: DELETE /api/admin/wipe/{kind}
+Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar.
+"""
+
+import json
+import logging
+import os
+import shutil
+from fastapi import APIRouter, HTTPException, Request
+
+from core.middleware import require_admin
+from core.database import (
+    SessionLocal,
+    Session as DbSession,
+    ChatMessage as DbChatMessage,
+    Memory,
+    Note,
+    ScheduledTask,
+    TaskRun,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    GalleryAlbum,
+    CalendarEvent,
+    CalendarCal,
+)
+from src.constants import DATA_DIR, SKILLS_DIR, SKILLS_FILE, GALLERY_DIR, GALLERY_UPLOADS_DIR
+
+logger = logging.getLogger(__name__)
+
+
+def _wipe_memory_files():
+    """Blank memory.json + drop the per-owner tidy-state sidecar so the
+    next audit doesn't try to diff against gone memories."""
+    for name in ("memory.json", "memory_tidy_state.json"):
+        p = os.path.join(DATA_DIR, name)
+        if not os.path.exists(p):
+            continue
+        try:
+            if name == "memory.json":
+                with open(p, "w", encoding="utf-8") as f:
+                    json.dump([], f)
+            else:
+                os.remove(p)
+        except OSError as e:
+            logger.warning(f"Could not reset {name}: {e}")
+
+
+def _rmtree_quiet(path: str):
+    """rmtree that doesn't crash if the path doesn't exist."""
+    if os.path.isdir(path):
+        try:
+            shutil.rmtree(path)
+        except OSError as e:
+            logger.warning(f"Could not remove {path}: {e}")
+
+
+def setup_admin_wipe_routes(session_manager):
+    """The session_manager is passed in so we can also clear its
+    in-memory cache when wiping chats — without it the DB is empty
+    but the next /api/sessions returns stale entries."""
+    router = APIRouter(prefix="/api/admin")
+
+    @router.delete("/wipe/{kind}")
+    def wipe(kind: str, request: Request):
+        require_admin(request)
+        kind = (kind or "").strip().lower()
+
+        db = SessionLocal()
+        try:
+            if kind == "chats":
+                count = db.query(DbSession).count()
+                db.query(DbChatMessage).delete()
+                db.query(DbSession).delete()
+                db.commit()
+                try:
+                    session_manager.sessions.clear()
+                except Exception:
+                    pass
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "memory":
+                count = db.query(Memory).count()
+                db.query(Memory).delete()
+                db.commit()
+                _wipe_memory_files()
+                # Drop the vector store too so semantic search doesn't
+                # return ghosts. Lazy import — chromadb may not be
+                # initialised in every deployment.
+                try:
+                    from src.memory_vector import get_memory_vector_store
+                    mv = get_memory_vector_store()
+                    if mv and hasattr(mv, "clear"):
+                        mv.clear()
+                except Exception as e:
+                    logger.info(f"Memory vector clear skipped: {e}")
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "skills":
+                # Skills live as SKILL.md files under data/skills/. Drop
+                # the entire directory; the SkillsManager re-creates the
+                # tree on next write.
+                skills_dir = SKILLS_DIR
+                count = 0
+                if os.path.isdir(skills_dir):
+                    # Count SKILL.md files for the response — quick walk.
+                    for _, _, files in os.walk(skills_dir):
+                        count += sum(1 for f in files if f == "SKILL.md")
+                    _rmtree_quiet(skills_dir)
+                # Legacy fallback file
+                legacy = SKILLS_FILE
+                if os.path.exists(legacy):
+                    try:
+                        os.remove(legacy)
+                    except OSError:
+                        pass
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "notes":
+                count = db.query(Note).count()
+                db.query(Note).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "tasks":
+                # TaskRun rows reference tasks via FK — clear them first.
+                db.query(TaskRun).delete()
+                count = db.query(ScheduledTask).count()
+                db.query(ScheduledTask).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "documents":
+                # DocumentVersion FKs Document — clear children first.
+                db.query(DocumentVersion).delete()
+                count = db.query(Document).count()
+                db.query(Document).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "gallery":
+                count = db.query(GalleryImage).count() + db.query(GalleryAlbum).count()
+                db.query(GalleryImage).delete()
+                db.query(GalleryAlbum).delete()
+                db.commit()
+                # Also drop the upload dir so disk doesn't keep orphans.
+                _rmtree_quiet(GALLERY_DIR)
+                _rmtree_quiet(GALLERY_UPLOADS_DIR)
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "calendar":
+                # Events FK calendars — clear children first, then both.
+                db.query(CalendarEvent).delete()
+                count = db.query(CalendarCal).count()
+                db.query(CalendarCal).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            raise HTTPException(400, f"Unknown wipe kind: {kind!r}")
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.exception(f"Wipe {kind} failed")
+            raise HTTPException(500, f"Wipe {kind} failed: {e}")
+        finally:
+            db.close()
+
+    return router

routes/admin_wipe_routes.py

@@ -1,176 +1,17 @@
-"""Admin Danger Zone — per-category wipes.
+"""Backward-compat shim — canonical location is routes/admin_wipe/admin_wipe_routes.py.
 
-Each endpoint is admin-only and truncates exactly one domain so the
-user can selectively reset memory / skills / notes / etc. without
-nuking everything. The catch-all `chats` endpoint mirrors the
-existing /api/sessions/all so the Danger Zone speaks one URL pattern.
-
-URL shape: DELETE /api/admin/wipe/{kind}
-Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar.
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.admin_wipe_routes``, ``from routes.admin_wipe_routes
+import X``, ``importlib.import_module("routes.admin_wipe_routes")``, and the
+``import ... as admin_wipe_routes`` + ``monkeypatch.setattr(admin_wipe_routes,
+"SessionLocal", ...)`` / ``"require_admin"`` pattern used by
+test_admin_wipe_gallery.py all operate on the *same* object the application
+actually uses. Keeps existing import paths working after slice 2h
+(#4082/#4071).
 """
 
-import json
-import logging
-import os
-import shutil
-from fastapi import APIRouter, HTTPException, Request
+import sys as _sys
 
-from core.middleware import require_admin
-from core.database import (
-    SessionLocal,
-    Session as DbSession,
-    ChatMessage as DbChatMessage,
-    Memory,
-    Note,
-    ScheduledTask,
-    TaskRun,
-    Document,
-    DocumentVersion,
-    GalleryImage,
-    GalleryAlbum,
-    CalendarEvent,
-    CalendarCal,
-)
-from src.constants import DATA_DIR, SKILLS_DIR, SKILLS_FILE, GALLERY_DIR, GALLERY_UPLOADS_DIR
+from routes.admin_wipe import admin_wipe_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-def _wipe_memory_files():
-    """Blank memory.json + drop the per-owner tidy-state sidecar so the
-    next audit doesn't try to diff against gone memories."""
-    for name in ("memory.json", "memory_tidy_state.json"):
-        p = os.path.join(DATA_DIR, name)
-        if not os.path.exists(p):
-            continue
-        try:
-            if name == "memory.json":
-                with open(p, "w", encoding="utf-8") as f:
-                    json.dump([], f)
-            else:
-                os.remove(p)
-        except OSError as e:
-            logger.warning(f"Could not reset {name}: {e}")
-
-
-def _rmtree_quiet(path: str):
-    """rmtree that doesn't crash if the path doesn't exist."""
-    if os.path.isdir(path):
-        try:
-            shutil.rmtree(path)
-        except OSError as e:
-            logger.warning(f"Could not remove {path}: {e}")
-
-
-def setup_admin_wipe_routes(session_manager):
-    """The session_manager is passed in so we can also clear its
-    in-memory cache when wiping chats — without it the DB is empty
-    but the next /api/sessions returns stale entries."""
-    router = APIRouter(prefix="/api/admin")
-
-    @router.delete("/wipe/{kind}")
-    def wipe(kind: str, request: Request):
-        require_admin(request)
-        kind = (kind or "").strip().lower()
-
-        db = SessionLocal()
-        try:
-            if kind == "chats":
-                count = db.query(DbSession).count()
-                db.query(DbChatMessage).delete()
-                db.query(DbSession).delete()
-                db.commit()
-                try:
-                    session_manager.sessions.clear()
-                except Exception:
-                    pass
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "memory":
-                count = db.query(Memory).count()
-                db.query(Memory).delete()
-                db.commit()
-                _wipe_memory_files()
-                # Drop the vector store too so semantic search doesn't
-                # return ghosts. Lazy import — chromadb may not be
-                # initialised in every deployment.
-                try:
-                    from src.memory_vector import get_memory_vector_store
-                    mv = get_memory_vector_store()
-                    if mv and hasattr(mv, "clear"):
-                        mv.clear()
-                except Exception as e:
-                    logger.info(f"Memory vector clear skipped: {e}")
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "skills":
-                # Skills live as SKILL.md files under data/skills/. Drop
-                # the entire directory; the SkillsManager re-creates the
-                # tree on next write.
-                skills_dir = SKILLS_DIR
-                count = 0
-                if os.path.isdir(skills_dir):
-                    # Count SKILL.md files for the response — quick walk.
-                    for _, _, files in os.walk(skills_dir):
-                        count += sum(1 for f in files if f == "SKILL.md")
-                    _rmtree_quiet(skills_dir)
-                # Legacy fallback file
-                legacy = SKILLS_FILE
-                if os.path.exists(legacy):
-                    try:
-                        os.remove(legacy)
-                    except OSError:
-                        pass
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "notes":
-                count = db.query(Note).count()
-                db.query(Note).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "tasks":
-                # TaskRun rows reference tasks via FK — clear them first.
-                db.query(TaskRun).delete()
-                count = db.query(ScheduledTask).count()
-                db.query(ScheduledTask).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "documents":
-                # DocumentVersion FKs Document — clear children first.
-                db.query(DocumentVersion).delete()
-                count = db.query(Document).count()
-                db.query(Document).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "gallery":
-                count = db.query(GalleryImage).count() + db.query(GalleryAlbum).count()
-                db.query(GalleryImage).delete()
-                db.query(GalleryAlbum).delete()
-                db.commit()
-                # Also drop the upload dir so disk doesn't keep orphans.
-                _rmtree_quiet(GALLERY_DIR)
-                _rmtree_quiet(GALLERY_UPLOADS_DIR)
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "calendar":
-                # Events FK calendars — clear children first, then both.
-                db.query(CalendarEvent).delete()
-                count = db.query(CalendarCal).count()
-                db.query(CalendarCal).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            raise HTTPException(400, f"Unknown wipe kind: {kind!r}")
-        except HTTPException:
-            raise
-        except Exception as e:
-            db.rollback()
-            logger.exception(f"Wipe {kind} failed")
-            raise HTTPException(500, f"Wipe {kind} failed: {e}")
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/assistant_routes.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 
 from core.database import SessionLocal, CrewMember, ScheduledTask
 from src.auth_helpers import get_current_user
-from core.auth import RESERVED_USERNAMES
+from src.owner_identity import REQUEST_SENTINEL_OWNERS
 from src.task_scheduler import compute_next_run
 
 
@@ -90,11 +90,12 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
     # check-in tasks seeded. Hitting any /assistant route under one of these
     # used to seed a full CrewMember + Morning/Midday/Evening tasks under that
     # owner, which then double-fired alongside the real user's check-ins.
-    # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "".
+    # REQUEST_SENTINEL_OWNERS covers request-only identities; Default/Local is a
+    # reserved login name but remains a valid storage owner.
 
     async def _get_or_create(owner: str) -> CrewMember:
         """Return the per-owner assistant CrewMember, creating it on demand."""
-        if not owner or owner in RESERVED_USERNAMES:
+        if not owner or owner in REQUEST_SENTINEL_OWNERS:
             raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
         db = SessionLocal()
         try:

routes/auth_routes.py

@@ -22,6 +22,8 @@ from src.settings import (
     load_features as _load_features,
     save_features as _save_features,
     DEFAULT_SETTINGS,
+    RETIRED_SETTING_KEYS,
+    without_retired_settings,
 )
 from src.integrations import (
     load_integrations,
@@ -84,6 +86,33 @@ class SetOpenRegistrationRequest(BaseModel):
 SESSION_COOKIE = "odysseus_session"
 
 
+def _secure_cookie(request: Request) -> bool:
+    """Decide the ``Secure`` attribute of the session cookie.
+
+    ``SECURE_COOKIES`` stays authoritative when it holds an explicit value:
+    ``true`` always marks the cookie Secure (the documented knob for a TLS
+    proxy), ``false`` never does, which is the escape hatch for an install
+    that still answers on plain HTTP alongside HTTPS. Anything else —
+    unset, or the present-but-empty value docker-compose injects for a
+    variable the host has not defined — derives it from the request, so an
+    HTTPS login gets a Secure cookie without any configuration.
+
+    Either the connection scheme or ``X-Forwarded-Proto`` saying https is
+    enough, which is the same test ``core/middleware.py`` applies before it
+    sends HSTS. Uvicorn's proxy-headers middleware already folds that header
+    into the scheme for the proxies it trusts, so reading it here only adds
+    the case of a terminator that is not on a trusted address; the cost is
+    that a client talking to the app directly can set the header and lock
+    its own session out over plain HTTP.
+    """
+    configured = os.getenv("SECURE_COOKIES", "").strip().lower()
+    if configured in ("true", "false"):
+        return configured == "true"
+    # A chained proxy sends a list — the client-facing hop comes first.
+    forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0]
+    return request.url.scheme == "https" or forwarded_proto.strip().lower() == "https"
+
+
 def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
     router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -157,7 +186,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             value=token,
             httponly=True,
             samesite="lax",
-            secure=os.getenv("SECURE_COOKIES", "false").lower() == "true",
+            secure=_secure_cookie(request),
             path="/",
         )
         if body.remember:
@@ -345,9 +374,61 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         # docs, email accounts, tasks, etc.
         try:
             from sqlalchemy import func
-            from core.database import Base, SessionLocal
+            from core.database import (
+                Base,
+                EmailAccount,
+                SessionLocal,
+                lock_email_account_owner_mutations,
+            )
             db = SessionLocal()
             try:
+                # Email-account defaults are protected by per-owner mutex rows.
+                # A rename crosses two owner partitions, so lock both in the
+                # shared helper's canonical order before inspecting either.
+                lock_email_account_owner_mutations(
+                    db, old_username, new_username
+                )
+
+                source_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == old_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                destination_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == new_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                if destination_default_ids:
+                    clear_default_ids = (
+                        destination_default_ids[1:] + source_default_ids
+                    )
+                else:
+                    clear_default_ids = source_default_ids[1:]
+                if clear_default_ids:
+                    (
+                        db.query(EmailAccount)
+                        .filter(EmailAccount.id.in_(clear_default_ids))
+                        .update(
+                            {EmailAccount.is_default: False},
+                            synchronize_session=False,
+                        )
+                    )
+
                 for mapper in Base.registry.mappers:
                     model = mapper.class_
                     if not hasattr(model, "owner"):
@@ -637,7 +718,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         a scrubbed copy with secret keys blanked. The frontend uses this
         for keybinds + TTS prefs, so it stays callable without admin."""
         user = _get_current_user(request)
-        settings = _load_settings()
+        settings = without_retired_settings(_load_settings())
         if user and auth_manager.is_admin(user):
             return settings
         return scrub_settings(settings)
@@ -657,6 +738,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
         }
         for key in DEFAULT_SETTINGS:
+            if key in RETIRED_SETTING_KEYS:
+                continue
             if key not in body:
                 continue
             val = body[key]
@@ -669,7 +752,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                 val = max(lo, min(val, hi))
             current[key] = val
         _save_settings(current)
-        return current
+        return without_retired_settings(current)
 
     # ---- Integrations CRUD ----
 

routes/backup_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime
 
 from fastapi import APIRouter, HTTPException, Request, Response
 from core.middleware import require_admin
+from services.memory import MemoryStoreUnreadable
 from src.auth_helpers import get_current_user
 from src.settings import load_settings, save_settings, load_features, save_features
 
@@ -76,7 +77,15 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
 
         # ── Memories ──
         if "memories" in body and isinstance(body["memories"], list):
-            existing = memory_manager.load_all()
+            # Strict load: importing on top of an unreadable store would write
+            # only the incoming rows back and drop everything already saved.
+            try:
+                existing = memory_manager.load_all_for_update()
+            except MemoryStoreUnreadable as e:
+                logger.error("Refusing to import memories: %s", e)
+                raise HTTPException(
+                    503, "Memory store is temporarily unreadable — nothing was imported."
+                )
             # Dedup against THIS user's own memories only. Using every tenant's
             # rows (load_all) meant a memory whose text matched any other
             # user's was silently skipped, so the importing user lost their own

routes/calendar_routes.py

@@ -10,11 +10,13 @@ from typing import Optional, List
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
 from pydantic import BaseModel
 from sqlalchemy import or_, and_
+from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
 from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
-from src.auth_helpers import require_user
+from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
+from src.upload_handler import reserve_upload_references
 
 logger = logging.getLogger(__name__)
 
@@ -220,22 +222,125 @@ class EventUpdate(BaseModel):
 
 # ── Helpers ──
 
+_DEFAULT_CALENDAR_NAMESPACE = uuid.UUID("4840613a-9847-4a3b-bd75-19e6bc5fc3ce")
+
+
+def _default_calendar_id(owner: str, collision_index: int = 0) -> str:
+    """Return one stable primary-key candidate for an owner's lazy default.
+
+    Slot zero preserves the original owner-derived identifier.  Later slots
+    let a username be reused after its prior calendar was migrated to another
+    owner during a rename, without making concurrent first use choose random
+    and therefore divergent identifiers.
+    """
+    if collision_index == 0:
+        candidate_name = owner
+    else:
+        candidate_name = json.dumps(
+            [owner, collision_index],
+            ensure_ascii=False,
+            separators=(",", ":"),
+        )
+    return str(uuid.uuid5(_DEFAULT_CALENDAR_NAMESPACE, candidate_name))
+
+
+def _begin_sqlite_default_write(db) -> None:
+    """Serialize an absent-default check with other SQLite writers.
+
+    SQLite's default deferred transactions allow two workers to both read an
+    empty calendar set before either writes.  ``BEGIN IMMEDIATE`` acquires the
+    writer reservation before the second, authoritative lookup.  We issue it
+    only when the driver has not already opened a write transaction; a caller
+    with a pending write already owns the required reservation.
+    """
+    connection = db.connection()
+    dbapi_connection = connection.connection
+    driver_connection = getattr(
+        dbapi_connection,
+        "driver_connection",
+        dbapi_connection,
+    )
+    if not getattr(driver_connection, "in_transaction", False):
+        connection.exec_driver_sql("BEGIN IMMEDIATE")
+
+
 def _ensure_default_calendar(db, owner: str = None) -> CalendarCal:
-    """Create default calendar if none exist for this owner."""
+    """Return the owner's calendar, staging a default in the caller's transaction.
+
+    A stable owner-derived primary key makes concurrent first-use inserts
+    converge on one row on every SQL backend.  SQLite additionally serializes
+    the absent-row check because its deferred transactions otherwise permit
+    both workers to read the gap before either writes.  Other backends recover
+    a lost insert race inside a savepoint so the caller's event transaction
+    remains usable and atomic.
+    """
     owner = owner or FALLBACK_OWNER
     cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
-    if not cal:
+    if cal:
+        return cal
+
+    dialect = db.get_bind().dialect.name
+    if dialect == "sqlite":
+        _begin_sqlite_default_write(db)
+        # Another worker may have committed while BEGIN IMMEDIATE waited.
+        cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first()
+        if cal:
+            return cal
+
+    collision_index = 0
+    while True:
+        default_id = _default_calendar_id(owner, collision_index)
+
+        if dialect == "sqlite":
+            # BEGIN IMMEDIATE above makes this occupancy check authoritative:
+            # another SQLite writer cannot rename, delete, or claim this slot
+            # until the caller commits or rolls back.
+            occupant = db.query(CalendarCal).filter(
+                CalendarCal.id == default_id,
+            ).first()
+            if occupant is not None:
+                if occupant.owner == owner:
+                    return occupant
+                collision_index += 1
+                continue
+
         cal = CalendarCal(
-            id=str(uuid.uuid4()),
+            id=default_id,
             owner=owner,
             name="Personal",
             color="#5b8abf",
             source="local",
         )
-        db.add(cal)
-        db.commit()
-        db.refresh(cal)
-    return cal
+
+        if dialect == "sqlite":
+            db.add(cal)
+            db.flush()
+            return cal
+
+        try:
+            # A uniqueness failure rolls back only this savepoint, not an event
+            # or reminder already staged by the caller's outer transaction.
+            with db.begin_nested():
+                db.add(cal)
+                db.flush()
+            return cal
+        except IntegrityError:
+            # Use a locking/current read so repeatable-read backends can observe
+            # the row that won after our transaction's original empty snapshot.
+            occupant = db.query(CalendarCal).filter(
+                CalendarCal.id == default_id,
+            ).with_for_update().first()
+            if occupant is None:
+                # Do not misclassify an unrelated integrity failure as an ID
+                # collision and loop forever. A concurrently deleted winner is
+                # safe for the caller to retry as a fresh transaction.
+                raise
+            if occupant.owner == owner:
+                return occupant
+            # A renamed calendar owns this deterministic slot. Advance to the
+            # next stable slot; concurrent callers for this owner will still
+            # converge there.
+            collision_index += 1
 
 
 # Per-request user time context. chat_routes sets this from browser timezone
@@ -697,9 +802,18 @@ def _expand_rrule(
 
 # ── Routes ──
 
-def setup_calendar_routes() -> APIRouter:
+def setup_calendar_routes(upload_handler=None) -> APIRouter:
     router = APIRouter(prefix="/api/calendar", tags=["calendar"])
 
+    def _reserve_calendar_uploads(request: Request, *values) -> None:
+        missing_id = reserve_upload_references(
+            upload_handler,
+            effective_user(request),
+            *values,
+        )
+        if missing_id:
+            raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
+
     # ── CalDAV multi-account helpers ─────────────────────────────────────────
 
     def _get_caldav_accounts(owner: str) -> list:
@@ -913,7 +1027,24 @@ def setup_calendar_routes() -> APIRouter:
             '</d:prop></d:propfind>'
         )
         try:
-            async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False) as cx:
+            # Build an SSL context that trusts the operator's custom CA bundle
+            # (SSL_CERT_FILE / REQUESTS_CA_BUNDLE) so self-signed CalDAV servers
+            # pass the pre-flight the same way they pass the real sync.
+            # trust_env=False is kept to block proxy/auth env leakage; the CA
+            # bundle is loaded explicitly instead.
+            import ssl as _ssl
+            _ssl_ctx = _ssl.create_default_context()
+            # Disable VERIFY_X509_STRICT so certs without a keyUsage extension
+            # (common in self-signed setups) are accepted, matching the
+            # requests/urllib3 behavior used by the CalDAV sync path.
+            _ssl_ctx.verify_flags &= ~_ssl.VERIFY_X509_STRICT
+            _ca_bundle = _os.environ.get("SSL_CERT_FILE") or _os.environ.get("REQUESTS_CA_BUNDLE")
+            if _ca_bundle:
+                if _os.path.isfile(_ca_bundle):
+                    _ssl_ctx.load_verify_locations(_ca_bundle)
+                else:
+                    logger.warning("CalDAV test: CA bundle %s not found, using system CAs", _ca_bundle)
+            async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False, verify=_ssl_ctx) as cx:
                 r = await cx.request(
                     "PROPFIND", url,
                     auth=(user, pw),
@@ -988,6 +1119,9 @@ def setup_calendar_routes() -> APIRouter:
         db = SessionLocal()
         try:
             _ensure_default_calendar(db, owner)
+            # Listing calendars intentionally lazily creates a durable default.
+            # Other callers commit it with the event they are creating.
+            db.commit()
             cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all()
             return {"calendars": [
                 {"name": c.name, "href": c.id, "color": c.color, "source": c.source}
@@ -996,6 +1130,7 @@ def setup_calendar_routes() -> APIRouter:
         except HTTPException:
             raise
         except Exception as e:
+            db.rollback()
             logger.error("Failed to list calendars: %s", e)
             raise HTTPException(500, "Failed to list calendars")
         finally:
@@ -1070,6 +1205,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.post("/events")
     async def create_event(request: Request, data: EventCreate):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, data.color, data.description, data.location)
         db = SessionLocal()
         try:
             cal = None
@@ -1131,6 +1267,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, data.color, data.description, data.location)
         try:
             base_uid = _resolve_base_uid(uid)
         except ValueError as e:
@@ -1224,6 +1361,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.post("/calendars")
     async def create_calendar(request: Request, name: str = "Imported", color: str = "#5b8abf"):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, color)
         db = SessionLocal()
         try:
             cal = CalendarCal(
@@ -1246,6 +1384,7 @@ def setup_calendar_routes() -> APIRouter:
     @router.put("/calendars/{cal_id}")
     async def update_calendar(request: Request, cal_id: str, name: str = None, color: str = None):
         owner = _require_user(request)
+        _reserve_calendar_uploads(request, color)
         db = SessionLocal()
         try:
             cal = _get_or_404_calendar(db, cal_id, owner)

routes/chat_helpers.py

@@ -5,6 +5,7 @@ import json
 import logging
 import os
 import re
+import time
 from dataclasses import dataclass, field
 from typing import Any, Optional
 
@@ -14,9 +15,10 @@ from core.database import Session as DBSession, ModelEndpoint
 from src.llm_core import normalize_model_id
 from src.endpoint_resolver import normalize_base
 from src.context_compactor import maybe_compact, trim_for_context
-from src.model_context import estimate_tokens
+from src.model_context import estimate_tokens, get_context_length
 from src.auth_helpers import effective_user
 from src.prompt_security import untrusted_context_message
+from src.attachment_refs import attachment_ref
 from routes.prefs_routes import _load_for_user as load_prefs_for_user
 
 from fastapi import HTTPException
@@ -55,6 +57,9 @@ def _is_casual_low_signal(text: str) -> bool:
 # the background work (extraction, auto-naming) silently never runs.
 # Mirrors WebhookManager._spawn_tracked from src/webhook_manager.py.
 _BG_TASKS: set[asyncio.Task] = set()
+_INCOGNITO_CONTEXTS: dict[str, dict[str, Any]] = {}
+_INCOGNITO_CONTEXT_TTL_SECONDS = 6 * 60 * 60
+_INCOGNITO_CONTEXT_MAX_MESSAGES = 80
 
 
 def _spawn_bg(coro) -> asyncio.Task:
@@ -65,6 +70,40 @@ def _spawn_bg(coro) -> asyncio.Task:
     return task
 
 
+def _prune_incognito_contexts(now: float | None = None):
+    now = now or time.time()
+    stale = [
+        sid for sid, bundle in _INCOGNITO_CONTEXTS.items()
+        if now - float(bundle.get("updated_at") or 0) > _INCOGNITO_CONTEXT_TTL_SECONDS
+    ]
+    for sid in stale:
+        _INCOGNITO_CONTEXTS.pop(sid, None)
+
+
+def _incognito_messages(session_id: str) -> list[dict[str, Any]]:
+    _prune_incognito_contexts()
+    bundle = _INCOGNITO_CONTEXTS.get(str(session_id or ""))
+    if not bundle:
+        return []
+    return [dict(m) for m in bundle.get("messages", []) if isinstance(m, dict)]
+
+
+def _append_incognito_message(session_id: str, role: str, content: Any, metadata: dict | None = None):
+    sid = str(session_id or "").strip()
+    if not sid:
+        return
+    _prune_incognito_contexts()
+    bundle = _INCOGNITO_CONTEXTS.setdefault(sid, {"messages": [], "updated_at": time.time()})
+    msg: dict[str, Any] = {"role": role, "content": content}
+    if metadata:
+        msg["metadata"] = dict(metadata)
+    messages = bundle.setdefault("messages", [])
+    messages.append(msg)
+    if len(messages) > _INCOGNITO_CONTEXT_MAX_MESSAGES:
+        del messages[:-_INCOGNITO_CONTEXT_MAX_MESSAGES]
+    bundle["updated_at"] = time.time()
+
+
 # ── Data containers ────────────────────────────────────────────────────── #
 
 @dataclass
@@ -113,10 +152,38 @@ class ChatContext:
     # Uploads attached to this user turn, resolved and owner-checked for the
     # agent's private context. This is not emitted to the browser.
     uploaded_files: list = field(default_factory=list)
+    # Route-neutral prompt before any model-window compaction/trimming. This is
+    # retained only when explicit foreground fallbacks are enabled so each
+    # concrete candidate can apply its own context budget independently.
+    route_messages: list = field(default_factory=list)
 
 
 # ── Helpers ────────────────────────────────────────────────────────────── #
 
+def _allowed_models_from_privileges(privs: dict) -> Optional[frozenset[str]]:
+    if privs.get("block_all_models"):
+        return frozenset()
+    allowed_raw = privs.get("allowed_models")
+    allowed = allowed_raw if isinstance(allowed_raw, list) else []
+    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
+    return frozenset(model for model in allowed if isinstance(model, str)) if restricted else None
+
+
+def _allowed_models_for_request(request) -> Optional[frozenset[str]]:
+    """Return the caller's model allowlist, or ``None`` when unrestricted."""
+
+    try:
+        user = effective_user(request)
+    except Exception:
+        user = None
+    if not user:
+        return None
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not auth_manager:
+        return None
+    privs = auth_manager.get_privileges(user) or {}
+    return _allowed_models_from_privileges(privs)
+
 def _enforce_chat_privileges(request, sess) -> None:
     """Apply the per-user privilege gates (allowed_models + max_messages_per_day)
     that both /api/chat and /api/chat_stream must enforce BEFORE any LLM work.
@@ -146,10 +213,8 @@ def _enforce_chat_privileges(request, sess) -> None:
     if privs.get("block_all_models"):
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
-    allowed_raw = privs.get("allowed_models")
-    allowed = allowed_raw if isinstance(allowed_raw, list) else []
-    restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
-    if restricted and sess.model and sess.model not in allowed:
+    allowed_models = _allowed_models_from_privileges(privs)
+    if allowed_models is not None and sess.model and sess.model not in allowed_models:
         raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
 
     cap = int(privs.get("max_messages_per_day") or 0)
@@ -248,96 +313,6 @@ async def auto_name_session(session_manager, sess):
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
-def try_fallback_endpoint(sess, session_id: str) -> dict | None:
-    """Find an alternative working endpoint when the current one fails.
-
-    Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None.
-    """
-    import requests as _req
-    from src.endpoint_resolver import (
-        build_chat_url,
-        build_headers,
-        build_models_url,
-        normalize_base,
-        resolve_endpoint_runtime,
-    )
-    from src.chatgpt_subscription import is_chatgpt_subscription_base
-
-    current_url = sess.endpoint_url or ""
-    owner = getattr(sess, "owner", None)
-    db = SessionLocal()
-    try:
-        q = db.query(ModelEndpoint).filter(
-            ModelEndpoint.is_enabled == True
-        )
-        if owner:
-            from src.auth_helpers import owner_filter
-            q = owner_filter(q, ModelEndpoint, owner)
-        endpoints = q.all()
-    finally:
-        db.close()
-
-    for ep in endpoints:
-        base = normalize_base(ep.base_url)
-        # Skip current endpoint
-        if current_url and base in current_url:
-            continue
-        try:
-            base, api_key = resolve_endpoint_runtime(ep, owner=owner)
-        except Exception:
-            continue
-        ping_url = build_models_url(base)
-        headers = build_headers(api_key, base)
-        try:
-            if ping_url:
-                r = _req.get(ping_url, headers=headers, timeout=5)
-                r.raise_for_status()
-                data = r.json()
-                models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
-                if not models:
-                    models = [
-                        m.get("name") or m.get("model")
-                        for m in (data.get("models") or [])
-                        if m.get("name") or m.get("model")
-                    ]
-            else:
-                models = json.loads(ep.cached_models or "[]")
-            if not models:
-                continue
-            # Found a working endpoint — update session
-            new_model = models[0]
-            chat_url = build_chat_url(base)
-            new_headers = build_headers(api_key, base)
-            persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers
-
-            sess.model = new_model
-            sess.endpoint_url = chat_url
-            sess.headers = new_headers
-
-            # Persist
-            _db = SessionLocal()
-            try:
-                _db.query(DBSession).filter(DBSession.id == session_id).update({
-                    "model": new_model,
-                    "endpoint_url": chat_url,
-                    "headers": persisted_headers,
-                })
-                _db.commit()
-            finally:
-                _db.close()
-
-            logger.info(f"Fallback: switched session {session_id} from {current_url} to {ep.name} ({new_model})")
-            return {
-                "model": new_model,
-                "endpoint_url": chat_url,
-                "endpoint_name": ep.name,
-            }
-        except Exception:
-            continue
-
-    return None
-
-
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
     temperature, max_tokens, system_prompt, char_name = (
@@ -418,24 +393,28 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
             except Exception:
                 path = None
 
-        manifest.append({
-            "id": info.get("id") or str(att_id),
-            "name": info.get("name") or info.get("original_name") or str(att_id),
-            "mime": info.get("mime", ""),
-            "size": info.get("size", 0),
+        ref = attachment_ref({**info, "id": info.get("id") or str(att_id)})
+        ref.update({
+            "id": ref["attachment_id"],
+            "uri": f"odysseus://attachment/{ref['attachment_id']}",
+            "read_policy": "owner_checked_upload",
+            # Transitional compatibility: existing built-in tools can still use
+            # this path, but only after owner, upload-root, and tool-root checks.
             "path": path,
         })
+        manifest.append(ref)
     return manifest
 
 
 def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
     """Add user message to session history and update session name.
-    In incognito mode, still add to in-memory history (for conversation context)
-    but skip session name update (which would persist)."""
+    Incognito messages must not mutate persistent session history, even in
+    memory, because a later normal turn can persist the same session object."""
+    if incognito:
+        return
     user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
     sess.add_message(ChatMessage("user", preprocessed.user_content, metadata=user_meta))
-    if not incognito:
-        chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
+    chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
 
 
 def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False):
@@ -644,6 +623,9 @@ async def build_chat_context(
     use_enhanced_message: bool = False,
     agent_mode: bool = False,
     allow_tool_preprocessing: bool = True,
+    defer_context_shaping: bool = False,
+    continuation_context_message: str | None = None,
+    persist_user_message: bool = True,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -664,11 +646,17 @@ async def build_chat_context(
         allow_tool_preprocessing=allow_tool_preprocessing,
     )
 
-    # Add user message to history
-    add_user_message(sess, chat_handler, preprocessed, incognito=incognito)
+    # Add user message to history. Nobody/incognito uses a request-local
+    # transcript store instead of session history so stale saved chats cannot
+    # bleed into context and the turn is not persisted.
+    if persist_user_message and incognito:
+        user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
+        _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
+    elif persist_user_message:
+        add_user_message(sess, chat_handler, preprocessed, incognito=False)
 
     # Fire events
-    if not incognito:
+    if persist_user_message and not incognito:
         fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
 
     # Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
@@ -680,7 +668,12 @@ async def build_chat_context(
         getattr(chat_handler, "upload_handler", None),
         getattr(sess, "owner", None),
     )
-    casual_low_signal = _is_casual_low_signal(message)
+    context_message = (
+        str(continuation_context_message).strip()
+        if continuation_context_message
+        else message
+    )
+    casual_low_signal = _is_casual_low_signal(context_message)
 
     # Memory enabled?
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
@@ -717,7 +710,15 @@ async def build_chat_context(
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
     # the sync path uses text_for_context.
-    _ctx_msg = preprocessed.enhanced_message if use_enhanced_message else preprocessed.text_for_context
+    _ctx_msg = (
+        context_message
+        if continuation_context_message
+        else (
+            preprocessed.enhanced_message
+            if use_enhanced_message
+            else preprocessed.text_for_context
+        )
+    )
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
@@ -756,8 +757,10 @@ async def build_chat_context(
     if norm:
         sess.model = norm
 
-    # Build messages
-    messages = preface + sess.get_context_messages()
+    # Build messages. In Nobody/incognito mode, never read saved session
+    # history: the session id may be a temporary wrapper or, in buggy clients, a
+    # stale normal session id. Only the ephemeral incognito transcript is safe.
+    messages = preface + (_incognito_messages(session_id) if incognito else sess.get_context_messages())
 
     # Current date/time — injected as a standalone *user*-role context message
     # placed immediately before the latest user turn, NOT folded into the
@@ -779,13 +782,22 @@ async def build_chat_context(
         except Exception:
             logger.debug("Failed to add current date/time context", exc_info=True)
 
-    # Auto-compact
-    messages, context_length, was_compacted = await maybe_compact(
-        sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
-    )
+    route_messages = list(messages)
+    # Explicit fallback routing must shape from the same route-neutral prompt
+    # for every candidate. Running selected-model compaction here would mutate
+    # session history before we know which route can answer and would make a
+    # later larger-context candidate unable to recover discarded history.
+    if defer_context_shaping:
+        context_length = get_context_length(sess.endpoint_url, sess.model)
+        was_compacted = False
+    else:
+        messages, context_length, was_compacted = await maybe_compact(
+            sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
+        )
     _before_trim_messages = len(messages)
     _before_trim_tokens = estimate_tokens(messages)
-    messages = trim_for_context(messages, context_length)
+    if not defer_context_shaping:
+        messages = trim_for_context(messages, context_length)
     _after_trim_messages = len(messages)
     _after_trim_tokens = estimate_tokens(messages)
     _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
@@ -809,6 +821,7 @@ async def build_chat_context(
         context_tokens_after_trim=_after_trim_tokens,
         auto_opened_docs=auto_opened_docs,
         uploaded_files=uploaded_files,
+        route_messages=route_messages,
     )
 
 
@@ -1023,7 +1036,12 @@ def save_assistant_response(
     tool_events: list = None,
     incognito: bool = False,
 ):
-    """Add assistant response to session history. In incognito mode, keeps in-memory context but skips DB persistence."""
+    """Add assistant response to session history.
+
+    Incognito responses are intentionally not added to the session object. The
+    session may later be saved by a normal turn, so "in-memory only" is not
+    private enough.
+    """
     md = dict(last_metrics) if last_metrics else {}
     def _model_value(value) -> str:
         if value is None:
@@ -1063,19 +1081,18 @@ def save_assistant_response(
         _content = _think_info["reply"]
     else:
         _content = full_response
+    if incognito:
+        _append_incognito_message(session_id, "assistant", _content, md)
+        return None
     sess.add_message(ChatMessage("assistant", _content, metadata=md))
 
-    if not incognito:
-        from core.database import update_session_last_accessed
-        update_session_last_accessed(session_id)
-        session_manager.save_sessions()
+    from core.database import update_session_last_accessed
+    update_session_last_accessed(session_id)
+    session_manager.save_sessions()
 
     # Return the persisted message's DB id so the stream can wire it onto the
     # freshly-rendered bubble — lets the user edit/delete a just-streamed reply
-    # without reloading. Incognito returns None: those messages are ephemeral,
-    # so we don't hand out an edit/delete handle for them.
-    if incognito:
-        return None
+    # without reloading.
     try:
         _last = sess.history[-1]
         _meta = getattr(_last, "metadata", None)

routes/cleanup/__init__.py

@@ -0,0 +1,5 @@
+"""Cleanup route domain package (slice 2g, #4082/#4071).
+
+Contains cleanup_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/cleanup_routes.py re-exports from here.
+"""

routes/cleanup/cleanup_routes.py

@@ -0,0 +1,60 @@
+# routes/cleanup_routes.py
+"""Routes for cleanup operations."""
+import logging
+from fastapi import APIRouter, HTTPException, Request
+from src.cleanup_service import get_cleanup_preview, cleanup_sessions
+from src.auth_helpers import get_current_user
+
+logger = logging.getLogger(__name__)
+
+def setup_cleanup_routes(session_manager):
+    """
+    Setup cleanup-related routes.
+
+    Args:
+        session_manager: SessionManager instance
+
+    Returns:
+        APIRouter instance with cleanup routes
+    """
+    router = APIRouter(prefix="/api/cleanup")
+
+    @router.get("/preview")
+    async def cleanup_preview(request: Request):
+        """
+        Preview what would be cleaned up without making any changes.
+
+        Returns:
+            JSON response with lists of sessions that would be archived/deleted and estimated space savings
+        """
+        user = get_current_user(request)
+        try:
+            preview = await get_cleanup_preview(owner=user)
+            return preview
+        except Exception as e:
+            logger.error(f"Cleanup preview failed: {e}")
+            raise HTTPException(500, "Cleanup preview generation failed")
+
+    @router.post("")
+    async def cleanup_endpoint(request: Request):
+        """
+        Perform cleanup operations:
+        1. Archive inactive sessions (not accessed for 7 days)
+        2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages)
+
+        Returns:
+            JSON response with counts of deleted and archived sessions, and space freed
+        """
+        user = get_current_user(request)
+        try:
+            archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user)
+            return {
+                "archived_count": archived_count,
+                "deleted_count": deleted_count,
+                "space_freed_mb": round(space_freed_mb, 2)
+            }
+        except Exception as e:
+            logger.error(f"Cleanup failed: {e}")
+            raise HTTPException(500, "Cleanup operation failed")
+
+    return router

routes/cleanup_routes.py

@@ -1,60 +1,17 @@
-# routes/cleanup_routes.py
-"""Routes for cleanup operations."""
-import logging
-from fastapi import APIRouter, HTTPException, Request
-from src.cleanup_service import get_cleanup_preview, cleanup_sessions
-from src.auth_helpers import get_current_user
+"""Backward-compat shim — canonical location is routes/cleanup/cleanup_routes.py.
 
-logger = logging.getLogger(__name__)
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.cleanup_routes``, ``from routes.cleanup_routes import X``,
+``importlib.import_module("routes.cleanup_routes")``, and the string-targeted
+``monkeypatch.setattr("routes.cleanup_routes.get_cleanup_preview", ...)`` /
+``"routes.cleanup_routes.get_current_user"`` / ``"routes.cleanup_routes.
+cleanup_sessions"`` pattern used by test_cleanup_owner_scope.py all operate
+on the *same* object the application actually uses. Keeps existing import
+paths working after slice 2g (#4082/#4071).
+"""
 
-def setup_cleanup_routes(session_manager):
-    """
-    Setup cleanup-related routes.
+import sys as _sys
 
-    Args:
-        session_manager: SessionManager instance
+from routes.cleanup import cleanup_routes as _canonical  # noqa: F401
 
-    Returns:
-        APIRouter instance with cleanup routes
-    """
-    router = APIRouter(prefix="/api/cleanup")
-
-    @router.get("/preview")
-    async def cleanup_preview(request: Request):
-        """
-        Preview what would be cleaned up without making any changes.
-
-        Returns:
-            JSON response with lists of sessions that would be archived/deleted and estimated space savings
-        """
-        user = get_current_user(request)
-        try:
-            preview = await get_cleanup_preview(owner=user)
-            return preview
-        except Exception as e:
-            logger.error(f"Cleanup preview failed: {e}")
-            raise HTTPException(500, "Cleanup preview generation failed")
-
-    @router.post("")
-    async def cleanup_endpoint(request: Request):
-        """
-        Perform cleanup operations:
-        1. Archive inactive sessions (not accessed for 7 days)
-        2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages)
-
-        Returns:
-            JSON response with counts of deleted and archived sessions, and space freed
-        """
-        user = get_current_user(request)
-        try:
-            archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user)
-            return {
-                "archived_count": archived_count,
-                "deleted_count": deleted_count,
-                "space_freed_mb": round(space_freed_mb, 2)
-            }
-        except Exception as e:
-            logger.error(f"Cleanup failed: {e}")
-            raise HTTPException(500, "Cleanup operation failed")
-
-    return router
+_sys.modules[__name__] = _canonical

routes/compare/__init__.py

@@ -0,0 +1,5 @@
+"""Compare route domain package (slice 2i, #4082/#4071).
+
+Contains compare_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/compare_routes.py re-exports from here.
+"""

routes/compare/compare_routes.py

@@ -0,0 +1,365 @@
+# routes/compare_routes.py
+"""Model A/B comparison routes."""
+import json
+import uuid
+import random
+from datetime import datetime
+from fastapi import APIRouter, Form, HTTPException, Request
+from typing import List
+from pydantic import BaseModel
+import logging
+
+from core.database import Comparison, SessionLocal
+from core.session_manager import SessionManager
+from src.auth_helpers import get_current_user
+from routes.session_routes import _reject_raw_endpoint_url_for_non_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/compare", tags=["compare"])
+
+
+def _owned_endpoint_by_url(db, base_url, owner):
+    """ModelEndpoint whose base_url == `base_url` and is VISIBLE to `owner`
+    (their own rows + legacy null-owner "shared" rows); None otherwise.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. start_comparison copies the matched row's api_key
+    into the caller-owned [CMP] session's headers, which then drives that session's
+    /api/chat_stream calls — so an UNSCOPED base_url match would let a user mint a
+    comparison bound to ANOTHER user's private endpoint and spend that owner's
+    api_key / reach whatever base_url they configured. Mirrors
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from core.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _owned_endpoint_by_id(db, endpoint_id, owner):
+    """ModelEndpoint whose id == `endpoint_id` and is VISIBLE to `owner` (their
+    own rows + legacy null-owner "shared" rows); None otherwise.
+
+    Preferred over _owned_endpoint_by_url for credential resolution: two visible
+    endpoints can share the same base_url but hold DIFFERENT api_keys (e.g. two
+    accounts on the same provider). A base_url-only match returns whichever row
+    sorts first, so it can copy the WRONG owner-scoped key into the [CMP] session.
+    An id pins the exact registered endpoint, so /api/compare/start prefers it and
+    only falls back to URL matching for legacy / admin raw-URL callers. Owner
+    scoping is identical to _owned_endpoint_by_url (a null/empty owner is a no-op).
+    """
+    from core.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+class RecordVoteRequest(BaseModel):
+    prompt: str
+    models: List[str]
+    winner: str           # model name or "tie"
+    is_blind: bool = True
+
+
+def setup_compare_routes(session_manager: SessionManager):
+    """Setup comparison routes."""
+
+    @router.post("/start")
+    def start_comparison(
+        request: Request,
+        prompt: str = Form(...),
+        model_a: str = Form(...),
+        model_b: str = Form(...),
+        endpoint_a: str = Form(""),
+        endpoint_b: str = Form(""),
+        endpoint_a_id: str = Form(""),
+        endpoint_b_id: str = Form(""),
+        is_blind: str = Form("true"),
+    ):
+        """Create two ephemeral sessions and a comparison record.
+
+        Returns the comparison ID and the two session IDs so the client
+        can fire two independent SSE streams to /api/chat_stream.
+        """
+        user = getattr(request.state, 'current_user', None)
+        comp_id = str(uuid.uuid4())
+        sid_a = str(uuid.uuid4())
+        sid_b = str(uuid.uuid4())
+
+        # Blind mapping: randomly assign left/right
+        blind = str(is_blind).lower() == "true"
+        if blind:
+            mapping = {"left": "a", "right": "b"}
+            if random.random() > 0.5:
+                mapping = {"left": "b", "right": "a"}
+        else:
+            mapping = {"left": "a", "right": "b"}
+
+        # Map session IDs to left/right based on blind mapping
+        session_left = sid_a if mapping["left"] == "a" else sid_b
+        session_right = sid_a if mapping["right"] == "a" else sid_b
+
+        # In blind mode, name the helper sessions by their neutral slot
+        # ("Model A" / "Model B") instead of the real model. Otherwise the
+        # session name leaks the model in the sidebar and GET /api/sessions,
+        # de-anonymizing the comparison before the user votes (issue #1285).
+        slot_name = {session_left: "Model A", session_right: "Model B"}
+
+        # SECURITY: resolve and validate BOTH endpoints before creating any
+        # session. Compare copies a registered endpoint's Authorization header
+        # into the [CMP] session, so validating one endpoint while creating its
+        # session, then rejecting the other, would leave a partial compare
+        # session behind with that header attached. Doing all the owner-scope
+        # resolution + raw-URL rejection up front means a 403 on either endpoint
+        # aborts the whole request with nothing created and no header copied.
+        from src.endpoint_resolver import build_chat_url, build_headers, normalize_base
+        resolved = []
+        db = SessionLocal()
+        try:
+            for sid, model, endpoint, endpoint_id in [
+                (sid_a, model_a, endpoint_a, endpoint_a_id),
+                (sid_b, model_b, endpoint_b, endpoint_b_id),
+            ]:
+                # Prefer an explicit endpoint id: it pins the EXACT registered
+                # endpoint (and its api_key), even when two endpoints visible to
+                # the caller share a base_url with different keys — a URL-only
+                # match would copy whichever row sorts first, i.e. possibly the
+                # wrong key. Fall back to URL resolution only for legacy / admin
+                # raw-URL callers that don't send an id.
+                eid = endpoint_id.strip() if isinstance(endpoint_id, str) else ""
+                if eid:
+                    ep = _owned_endpoint_by_id(db, eid, user)
+                    if ep is None:
+                        # An id the caller can't see (wrong owner / deleted) must
+                        # NOT silently fall back to a same-URL row with a different
+                        # key — that's exactly the mix-up ids exist to prevent.
+                        raise HTTPException(404, "Model endpoint not found")
+                    # The id already resolved the endpoint; ignore any raw URL the
+                    # caller also sent and dial the stored config instead.
+                    endpoint = ep.base_url
+                elif not endpoint:
+                    raise HTTPException(
+                        422, "endpoint_a/endpoint_b or endpoint_a_id/endpoint_b_id is required"
+                    )
+                else:
+                    # Resolve the supplied URL to a ModelEndpoint the caller owns
+                    # (their own rows + legacy null-owner shared rows), scoped so a
+                    # comparison can't borrow another user's private endpoint key.
+                    base = normalize_base(endpoint)
+                    ep = _owned_endpoint_by_url(db, base, user)
+                # Reject *unregistered* raw URLs for signed-in non-admins; a
+                # matched registered endpoint supplies an id so the caller can
+                # still compare endpoints they own. Blanket-rejecting here (the
+                # earlier `endpoint_id=None` call) locked non-admins out of
+                # compare entirely, since compare resolves endpoints by URL with
+                # no endpoint_id. Mirrors the gallery inpaint/harmonize checks.
+                # Raised here (phase 1), before any session exists.
+                _reject_raw_endpoint_url_for_non_admin(
+                    request, user, str(ep.id) if ep is not None else None, endpoint
+                )
+                # Bind the [CMP] session to the RESOLVED endpoint, not the raw
+                # caller-supplied string. When the URL matches a registered
+                # endpoint visible to the caller, use that row's own normalized
+                # base URL (the same value owner scoping + endpoint validation
+                # already vetted) so the session dials exactly where the stored
+                # config points. The raw `endpoint` only survives for callers
+                # allowed to pass one — admins / single-user mode, where
+                # `_reject_raw_endpoint_url_for_non_admin` is a no-op and `ep`
+                # is None. Mirrors the registered-endpoint path in session_routes.
+                session_endpoint_url = (
+                    build_chat_url(normalize_base(ep.base_url)) if ep is not None else endpoint
+                )
+                # Headers come only from a matched endpoint's key; None when
+                # `ep` is None (raw admin URL or no match), so a comparison can
+                # never inherit another user's key/headers.
+                headers = build_headers(ep.api_key, ep.base_url) if (ep and ep.api_key) else None
+                resolved.append((sid, model, session_endpoint_url, headers))
+        finally:
+            db.close()
+
+        # Both endpoints validated — only now create the ephemeral [CMP]
+        # sessions and copy any resolved headers.
+        for sid, model, session_endpoint_url, headers in resolved:
+            name = f"[CMP] {slot_name[sid]}" if blind else f"[CMP] {model.split('/')[-1]}"
+            session_manager.create_session(
+                session_id=sid,
+                name=name,
+                endpoint_url=session_endpoint_url,
+                model=model,
+                rag=False,
+                owner=user,
+            )
+            if headers:
+                s = session_manager.sessions.get(sid)
+                if s:
+                    s.headers = headers
+
+        # Store comparison record
+        db = SessionLocal()
+        try:
+            comp = Comparison(
+                id=comp_id,
+                prompt=prompt,
+                model_a=model_a,
+                model_b=model_b,
+                # Record the URL the session actually dials. For URL callers this
+                # is their raw input; for id-only callers (empty endpoint_a/_b)
+                # fall back to the resolved endpoint URL so the column stays
+                # meaningful and non-null. resolved is in [a, b] order.
+                endpoint_a=endpoint_a or resolved[0][2],
+                endpoint_b=endpoint_b or resolved[1][2],
+                is_blind=blind,
+                blind_mapping=json.dumps(mapping),
+                owner=user,
+            )
+            db.add(comp)
+            db.commit()
+        finally:
+            db.close()
+
+        # In blind mode, withhold the model identities AND the left/right
+        # mapping from the response. The client already knows model_a/model_b
+        # (it sent them), so returning either would defeat blind mode. They are
+        # revealed by POST /api/compare/{id}/vote once the user has voted (#1285).
+        return {
+            "id": comp_id,
+            "session_left": session_left,
+            "session_right": session_right,
+            "model_left": None if blind else (model_a if mapping["left"] == "a" else model_b),
+            "model_right": None if blind else (model_a if mapping["right"] == "a" else model_b),
+            "is_blind": blind,
+            "mapping": None if blind else mapping,
+        }
+
+    @router.post("/{comp_id}/vote")
+    def vote_comparison(
+        request: Request,
+        comp_id: str,
+        winner: str = Form(...),  # "left", "right", or "tie"
+    ):
+        """Record the user's vote and reveal model names if blind."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
+            if not comp:
+                raise HTTPException(404, "Comparison not found")
+            # SECURITY: strict ownership — null-owner Comparisons were
+            # accessible to every user.
+            if user and comp.owner != user:
+                raise HTTPException(404, "Comparison not found")
+            if comp.winner:
+                raise HTTPException(400, "Already voted")
+
+            mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"}
+
+            if winner == "tie":
+                comp.winner = "tie"
+            elif winner == "left":
+                comp.winner = mapping["left"]
+            elif winner == "right":
+                comp.winner = mapping["right"]
+            else:
+                raise HTTPException(400, "winner must be 'left', 'right', or 'tie'")
+
+            comp.voted_at = datetime.utcnow()
+            db.commit()
+
+            return {
+                "winner": comp.winner,
+                "model_a": comp.model_a,
+                "model_b": comp.model_b,
+                "revealed": {
+                    "left": comp.model_a if mapping["left"] == "a" else comp.model_b,
+                    "right": comp.model_a if mapping["right"] == "a" else comp.model_b,
+                },
+            }
+        finally:
+            db.close()
+
+    @router.post("/record")
+    def record_comparison(request: Request, body: RecordVoteRequest):
+        """Lightweight endpoint to record a comparison vote from the frontend."""
+        user = get_current_user(request)
+        comp_id = str(uuid.uuid4())
+
+        model_a = body.models[0] if len(body.models) > 0 else ""
+        model_b = body.models[1] if len(body.models) > 1 else ""
+
+        # For N>2 models, store the full list as JSON in blind_mapping
+        if len(body.models) > 2:
+            blind_mapping = json.dumps({"models": body.models})
+        else:
+            blind_mapping = None
+
+        db = SessionLocal()
+        try:
+            comp = Comparison(
+                id=comp_id,
+                prompt=body.prompt[:500],
+                model_a=model_a,
+                model_b=model_b,
+                endpoint_a="",
+                endpoint_b="",
+                winner=body.winner,
+                is_blind=body.is_blind,
+                blind_mapping=blind_mapping,
+                voted_at=datetime.utcnow(),
+                owner=user,
+            )
+            db.add(comp)
+            db.commit()
+        finally:
+            db.close()
+
+        return {"status": "ok", "id": comp_id}
+
+    @router.get("/history")
+    def list_comparisons(request: Request):
+        """List past comparisons."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            q = db.query(Comparison)
+            if user:
+                q = q.filter(Comparison.owner == user)
+            comps = q.order_by(Comparison.created_at.desc()).limit(50).all()
+            return [
+                {
+                    "id": c.id,
+                    "prompt": c.prompt[:100],
+                    "model_a": c.model_a,
+                    "model_b": c.model_b,
+                    "winner": c.winner,
+                    "is_blind": c.is_blind,
+                    "voted_at": c.voted_at.isoformat() if c.voted_at else None,
+                    "created_at": c.created_at.isoformat() if c.created_at else None,
+                }
+                for c in comps
+            ]
+        finally:
+            db.close()
+
+    @router.delete("/{comp_id}")
+    def delete_comparison(request: Request, comp_id: str):
+        """Delete a comparison and its ephemeral sessions."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
+            if not comp:
+                raise HTTPException(404, "Comparison not found")
+            # SECURITY: strict ownership — null-owner Comparisons were
+            # accessible to every user.
+            if user and comp.owner != user:
+                raise HTTPException(404, "Comparison not found")
+            db.delete(comp)
+            db.commit()
+            return {"status": "deleted"}
+        finally:
+            db.close()
+
+    return router

routes/compare_routes.py

@@ -1,365 +1,18 @@
-# routes/compare_routes.py
-"""Model A/B comparison routes."""
-import json
-import uuid
-import random
-from datetime import datetime
-from fastapi import APIRouter, Form, HTTPException, Request
-from typing import List
-from pydantic import BaseModel
-import logging
+"""Backward-compat shim — canonical location is routes/compare/compare_routes.py.
 
-from core.database import Comparison, SessionLocal
-from core.session_manager import SessionManager
-from src.auth_helpers import get_current_user
-from routes.session_routes import _reject_raw_endpoint_url_for_non_admin
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.compare_routes``, ``from routes.compare_routes import X``,
+``importlib.import_module("routes.compare_routes")``, and the
+``import ... as cr`` + ``monkeypatch.setattr(cr, "SessionLocal", ...)`` /
+``"_owned_endpoint_by_url"`` / ``"_owned_endpoint_by_id"`` pattern used by
+test_endpoint_owner_scope_followup.py all operate on the *same* object the
+application actually uses. Keeps existing import paths working after
+slice 2i (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
-logger = logging.getLogger(__name__)
+import sys as _sys
 
-router = APIRouter(prefix="/api/compare", tags=["compare"])
+from routes.compare import compare_routes as _canonical  # noqa: F401
 
-
-def _owned_endpoint_by_url(db, base_url, owner):
-    """ModelEndpoint whose base_url == `base_url` and is VISIBLE to `owner`
-    (their own rows + legacy null-owner "shared" rows); None otherwise.
-
-    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
-    owner = private, "the model picker only shows the endpoint to that user") and
-    holds a decrypted `api_key`. start_comparison copies the matched row's api_key
-    into the caller-owned [CMP] session's headers, which then drives that session's
-    /api/chat_stream calls — so an UNSCOPED base_url match would let a user mint a
-    comparison bound to ANOTHER user's private endpoint and spend that owner's
-    api_key / reach whatever base_url they configured. Mirrors
-    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
-    legacy mode).
-    """
-    from core.database import ModelEndpoint
-    from src.auth_helpers import owner_filter
-    q = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url)
-    return owner_filter(q, ModelEndpoint, owner).first()
-
-
-def _owned_endpoint_by_id(db, endpoint_id, owner):
-    """ModelEndpoint whose id == `endpoint_id` and is VISIBLE to `owner` (their
-    own rows + legacy null-owner "shared" rows); None otherwise.
-
-    Preferred over _owned_endpoint_by_url for credential resolution: two visible
-    endpoints can share the same base_url but hold DIFFERENT api_keys (e.g. two
-    accounts on the same provider). A base_url-only match returns whichever row
-    sorts first, so it can copy the WRONG owner-scoped key into the [CMP] session.
-    An id pins the exact registered endpoint, so /api/compare/start prefers it and
-    only falls back to URL matching for legacy / admin raw-URL callers. Owner
-    scoping is identical to _owned_endpoint_by_url (a null/empty owner is a no-op).
-    """
-    from core.database import ModelEndpoint
-    from src.auth_helpers import owner_filter
-    q = db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id)
-    return owner_filter(q, ModelEndpoint, owner).first()
-
-
-class RecordVoteRequest(BaseModel):
-    prompt: str
-    models: List[str]
-    winner: str           # model name or "tie"
-    is_blind: bool = True
-
-
-def setup_compare_routes(session_manager: SessionManager):
-    """Setup comparison routes."""
-
-    @router.post("/start")
-    def start_comparison(
-        request: Request,
-        prompt: str = Form(...),
-        model_a: str = Form(...),
-        model_b: str = Form(...),
-        endpoint_a: str = Form(""),
-        endpoint_b: str = Form(""),
-        endpoint_a_id: str = Form(""),
-        endpoint_b_id: str = Form(""),
-        is_blind: str = Form("true"),
-    ):
-        """Create two ephemeral sessions and a comparison record.
-
-        Returns the comparison ID and the two session IDs so the client
-        can fire two independent SSE streams to /api/chat_stream.
-        """
-        user = getattr(request.state, 'current_user', None)
-        comp_id = str(uuid.uuid4())
-        sid_a = str(uuid.uuid4())
-        sid_b = str(uuid.uuid4())
-
-        # Blind mapping: randomly assign left/right
-        blind = str(is_blind).lower() == "true"
-        if blind:
-            mapping = {"left": "a", "right": "b"}
-            if random.random() > 0.5:
-                mapping = {"left": "b", "right": "a"}
-        else:
-            mapping = {"left": "a", "right": "b"}
-
-        # Map session IDs to left/right based on blind mapping
-        session_left = sid_a if mapping["left"] == "a" else sid_b
-        session_right = sid_a if mapping["right"] == "a" else sid_b
-
-        # In blind mode, name the helper sessions by their neutral slot
-        # ("Model A" / "Model B") instead of the real model. Otherwise the
-        # session name leaks the model in the sidebar and GET /api/sessions,
-        # de-anonymizing the comparison before the user votes (issue #1285).
-        slot_name = {session_left: "Model A", session_right: "Model B"}
-
-        # SECURITY: resolve and validate BOTH endpoints before creating any
-        # session. Compare copies a registered endpoint's Authorization header
-        # into the [CMP] session, so validating one endpoint while creating its
-        # session, then rejecting the other, would leave a partial compare
-        # session behind with that header attached. Doing all the owner-scope
-        # resolution + raw-URL rejection up front means a 403 on either endpoint
-        # aborts the whole request with nothing created and no header copied.
-        from src.endpoint_resolver import build_chat_url, build_headers, normalize_base
-        resolved = []
-        db = SessionLocal()
-        try:
-            for sid, model, endpoint, endpoint_id in [
-                (sid_a, model_a, endpoint_a, endpoint_a_id),
-                (sid_b, model_b, endpoint_b, endpoint_b_id),
-            ]:
-                # Prefer an explicit endpoint id: it pins the EXACT registered
-                # endpoint (and its api_key), even when two endpoints visible to
-                # the caller share a base_url with different keys — a URL-only
-                # match would copy whichever row sorts first, i.e. possibly the
-                # wrong key. Fall back to URL resolution only for legacy / admin
-                # raw-URL callers that don't send an id.
-                eid = endpoint_id.strip() if isinstance(endpoint_id, str) else ""
-                if eid:
-                    ep = _owned_endpoint_by_id(db, eid, user)
-                    if ep is None:
-                        # An id the caller can't see (wrong owner / deleted) must
-                        # NOT silently fall back to a same-URL row with a different
-                        # key — that's exactly the mix-up ids exist to prevent.
-                        raise HTTPException(404, "Model endpoint not found")
-                    # The id already resolved the endpoint; ignore any raw URL the
-                    # caller also sent and dial the stored config instead.
-                    endpoint = ep.base_url
-                elif not endpoint:
-                    raise HTTPException(
-                        422, "endpoint_a/endpoint_b or endpoint_a_id/endpoint_b_id is required"
-                    )
-                else:
-                    # Resolve the supplied URL to a ModelEndpoint the caller owns
-                    # (their own rows + legacy null-owner shared rows), scoped so a
-                    # comparison can't borrow another user's private endpoint key.
-                    base = normalize_base(endpoint)
-                    ep = _owned_endpoint_by_url(db, base, user)
-                # Reject *unregistered* raw URLs for signed-in non-admins; a
-                # matched registered endpoint supplies an id so the caller can
-                # still compare endpoints they own. Blanket-rejecting here (the
-                # earlier `endpoint_id=None` call) locked non-admins out of
-                # compare entirely, since compare resolves endpoints by URL with
-                # no endpoint_id. Mirrors the gallery inpaint/harmonize checks.
-                # Raised here (phase 1), before any session exists.
-                _reject_raw_endpoint_url_for_non_admin(
-                    request, user, str(ep.id) if ep is not None else None, endpoint
-                )
-                # Bind the [CMP] session to the RESOLVED endpoint, not the raw
-                # caller-supplied string. When the URL matches a registered
-                # endpoint visible to the caller, use that row's own normalized
-                # base URL (the same value owner scoping + endpoint validation
-                # already vetted) so the session dials exactly where the stored
-                # config points. The raw `endpoint` only survives for callers
-                # allowed to pass one — admins / single-user mode, where
-                # `_reject_raw_endpoint_url_for_non_admin` is a no-op and `ep`
-                # is None. Mirrors the registered-endpoint path in session_routes.
-                session_endpoint_url = (
-                    build_chat_url(normalize_base(ep.base_url)) if ep is not None else endpoint
-                )
-                # Headers come only from a matched endpoint's key; None when
-                # `ep` is None (raw admin URL or no match), so a comparison can
-                # never inherit another user's key/headers.
-                headers = build_headers(ep.api_key, ep.base_url) if (ep and ep.api_key) else None
-                resolved.append((sid, model, session_endpoint_url, headers))
-        finally:
-            db.close()
-
-        # Both endpoints validated — only now create the ephemeral [CMP]
-        # sessions and copy any resolved headers.
-        for sid, model, session_endpoint_url, headers in resolved:
-            name = f"[CMP] {slot_name[sid]}" if blind else f"[CMP] {model.split('/')[-1]}"
-            session_manager.create_session(
-                session_id=sid,
-                name=name,
-                endpoint_url=session_endpoint_url,
-                model=model,
-                rag=False,
-                owner=user,
-            )
-            if headers:
-                s = session_manager.sessions.get(sid)
-                if s:
-                    s.headers = headers
-
-        # Store comparison record
-        db = SessionLocal()
-        try:
-            comp = Comparison(
-                id=comp_id,
-                prompt=prompt,
-                model_a=model_a,
-                model_b=model_b,
-                # Record the URL the session actually dials. For URL callers this
-                # is their raw input; for id-only callers (empty endpoint_a/_b)
-                # fall back to the resolved endpoint URL so the column stays
-                # meaningful and non-null. resolved is in [a, b] order.
-                endpoint_a=endpoint_a or resolved[0][2],
-                endpoint_b=endpoint_b or resolved[1][2],
-                is_blind=blind,
-                blind_mapping=json.dumps(mapping),
-                owner=user,
-            )
-            db.add(comp)
-            db.commit()
-        finally:
-            db.close()
-
-        # In blind mode, withhold the model identities AND the left/right
-        # mapping from the response. The client already knows model_a/model_b
-        # (it sent them), so returning either would defeat blind mode. They are
-        # revealed by POST /api/compare/{id}/vote once the user has voted (#1285).
-        return {
-            "id": comp_id,
-            "session_left": session_left,
-            "session_right": session_right,
-            "model_left": None if blind else (model_a if mapping["left"] == "a" else model_b),
-            "model_right": None if blind else (model_a if mapping["right"] == "a" else model_b),
-            "is_blind": blind,
-            "mapping": None if blind else mapping,
-        }
-
-    @router.post("/{comp_id}/vote")
-    def vote_comparison(
-        request: Request,
-        comp_id: str,
-        winner: str = Form(...),  # "left", "right", or "tie"
-    ):
-        """Record the user's vote and reveal model names if blind."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
-            if not comp:
-                raise HTTPException(404, "Comparison not found")
-            # SECURITY: strict ownership — null-owner Comparisons were
-            # accessible to every user.
-            if user and comp.owner != user:
-                raise HTTPException(404, "Comparison not found")
-            if comp.winner:
-                raise HTTPException(400, "Already voted")
-
-            mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"}
-
-            if winner == "tie":
-                comp.winner = "tie"
-            elif winner == "left":
-                comp.winner = mapping["left"]
-            elif winner == "right":
-                comp.winner = mapping["right"]
-            else:
-                raise HTTPException(400, "winner must be 'left', 'right', or 'tie'")
-
-            comp.voted_at = datetime.utcnow()
-            db.commit()
-
-            return {
-                "winner": comp.winner,
-                "model_a": comp.model_a,
-                "model_b": comp.model_b,
-                "revealed": {
-                    "left": comp.model_a if mapping["left"] == "a" else comp.model_b,
-                    "right": comp.model_a if mapping["right"] == "a" else comp.model_b,
-                },
-            }
-        finally:
-            db.close()
-
-    @router.post("/record")
-    def record_comparison(request: Request, body: RecordVoteRequest):
-        """Lightweight endpoint to record a comparison vote from the frontend."""
-        user = get_current_user(request)
-        comp_id = str(uuid.uuid4())
-
-        model_a = body.models[0] if len(body.models) > 0 else ""
-        model_b = body.models[1] if len(body.models) > 1 else ""
-
-        # For N>2 models, store the full list as JSON in blind_mapping
-        if len(body.models) > 2:
-            blind_mapping = json.dumps({"models": body.models})
-        else:
-            blind_mapping = None
-
-        db = SessionLocal()
-        try:
-            comp = Comparison(
-                id=comp_id,
-                prompt=body.prompt[:500],
-                model_a=model_a,
-                model_b=model_b,
-                endpoint_a="",
-                endpoint_b="",
-                winner=body.winner,
-                is_blind=body.is_blind,
-                blind_mapping=blind_mapping,
-                voted_at=datetime.utcnow(),
-                owner=user,
-            )
-            db.add(comp)
-            db.commit()
-        finally:
-            db.close()
-
-        return {"status": "ok", "id": comp_id}
-
-    @router.get("/history")
-    def list_comparisons(request: Request):
-        """List past comparisons."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            q = db.query(Comparison)
-            if user:
-                q = q.filter(Comparison.owner == user)
-            comps = q.order_by(Comparison.created_at.desc()).limit(50).all()
-            return [
-                {
-                    "id": c.id,
-                    "prompt": c.prompt[:100],
-                    "model_a": c.model_a,
-                    "model_b": c.model_b,
-                    "winner": c.winner,
-                    "is_blind": c.is_blind,
-                    "voted_at": c.voted_at.isoformat() if c.voted_at else None,
-                    "created_at": c.created_at.isoformat() if c.created_at else None,
-                }
-                for c in comps
-            ]
-        finally:
-            db.close()
-
-    @router.delete("/{comp_id}")
-    def delete_comparison(request: Request, comp_id: str):
-        """Delete a comparison and its ephemeral sessions."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
-            if not comp:
-                raise HTTPException(404, "Comparison not found")
-            # SECURITY: strict ownership — null-owner Comparisons were
-            # accessible to every user.
-            if user and comp.owner != user:
-                raise HTTPException(404, "Comparison not found")
-            db.delete(comp)
-            db.commit()
-            return {"status": "deleted"}
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/cookbook_helpers.py

@@ -463,14 +463,22 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "        if sz == 0 and os.path.isdir(snap):",
         "            sz2, nf2, ic2 = snapshot_size()",
         "            sz, nf, ic = sz2, nf2, ic or ic2",
-        "        is_diffusion = False; gguf_files = []",
+        "        is_video = bool(re.search(r'(?i)(^|/)Lightricks/LTX-|(^|/)LTX[-_/]|video|text-to-video|image-to-video', rid))",
+        "        is_diffusion = is_video; is_adapter = bool(re.search(r'(?i)(lora|adapter|peft|qlora|control[-_]?lora|diffusion[-_]?lora)', rid)); gguf_files = []",
         "        if os.path.isdir(snap):",
         "            for sd in os.listdir(snap):",
         "                sf = os.path.join(snap, sd)",
         "                if not os.path.isdir(sf): continue",
         "                if os.path.exists(os.path.join(sf, 'model_index.json')): is_diffusion = True",
+        "                if os.path.exists(os.path.join(sf, 'adapter_config.json')) or os.path.exists(os.path.join(sf, 'adapter_model.safetensors')): is_adapter = True",
+        "                for _root, _dirs, _fns in safe_walk(sf):",
+        "                    for _fn in _fns:",
+        "                        _lfn = _fn.lower()",
+        "                        if _lfn.endswith('.safetensors') and re.search(r'(?i)(ltx|video|upscaler)', _lfn): is_video = True; is_diffusion = True",
+        "                        if _lfn in ('adapter_config.json','adapter_model.safetensors','pytorch_lora_weights.safetensors') or 'lora' in _lfn:",
+        "                            is_adapter = True",
         "                for f in collect_ggufs(sf): f['rel_path'] = sd + '/' + f['rel_path']; gguf_files.append(f)",
-        "        models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
+        "        models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_video':is_video,'is_adapter':is_adapter,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
         "def hf_cache_paths():",
         "    candidates = []",
         "    def add(p):",
@@ -505,11 +513,12 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "        fp = os.path.join(p, d)",
         "        if not os.path.isdir(fp) or os.path.islink(fp) or not safe_path(fp): continue",
         "        if d in seen: continue",
-        "        is_model = False; gguf_files = []",
+        "        is_model = False; is_adapter = bool(re.search(r'(?i)(lora|adapter|peft|qlora|control[-_]?lora|diffusion[-_]?lora)', d)); gguf_files = []",
         "        for root, dirs, fns in safe_walk(fp):",
         "            for fn in fns:",
         "                if fn.lower().endswith('.gguf'): is_model = True",
         "                elif fn == 'config.json' or fn.endswith('.safetensors') or fn.endswith('.bin'): is_model = True",
+        "                if fn in ('adapter_config.json','adapter_model.safetensors','pytorch_lora_weights.safetensors') or 'lora' in fn.lower(): is_adapter = True",
         "            if is_model: break",
         "        if not is_model: continue",
         "        gguf_files = collect_ggufs(fp)",
@@ -520,7 +529,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache:
         "                try: nf += 1; sz += os.path.getsize(os.path.join(dp, fn))",
         "                except Exception: pass",
         "        is_diff = os.path.exists(os.path.join(fp, 'model_index.json'))",
-        "        models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
+        "        models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_adapter':is_adapter,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
         "def parse_size(num, unit):",
         "    try: n = float(num)",
         "    except Exception: return 0",
@@ -1195,6 +1204,41 @@ def _safe_env_prefix(ep: str | None) -> str | None:
     return f'[ -f "{path}" ] && source "{path}" || true'
 
 
+def _local_windows_bash_env_prefix(ep: str | None) -> str | None:
+    """Convert a frontend PowerShell venv prefix for the local Git Bash runner."""
+    if not ep:
+        return ep
+
+    prefix = ep.strip()
+    if not prefix.startswith("&"):
+        return ep
+
+    raw_path = prefix[1:].lstrip()
+    if not raw_path:
+        return ep
+    if raw_path.startswith("'"):
+        if len(raw_path) < 2 or not raw_path.endswith("'"):
+            return ep
+        quoted_path = raw_path[1:-1]
+        if "'" in quoted_path.replace("''", ""):
+            return ep
+        path = quoted_path.replace("''", "'")
+    else:
+        path = raw_path.rstrip()
+        if "'" in path or '"' in path:
+            return ep
+    if any(c in path for c in "\r\n;&|`$<>"):
+        return ep
+    if not path.replace("\\", "/").casefold().endswith("/scripts/activate.ps1"):
+        return ep
+
+    bash_path = _git_bash_path(path)
+    if "\\" in bash_path:
+        return ep
+    bash_path = bash_path[: -len("Activate.ps1")] + "activate"
+    return "source " + shlex.quote(bash_path)
+
+
 def _ssh_ps(host, script_path, port=None):
     """Build SSH command to run a PowerShell script on a Windows remote."""
     pf = f"-p {port} " if port and port != "22" else ""
@@ -1320,6 +1364,26 @@ def _diagnose_serve_output(text: str) -> dict | None:
             "MLX LM is not installed on this server.",
             [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
         ),
+        (
+            r"OmniGen2Pipeline|module diffusers has no attribute .*Pipeline|custom_pipeline=.*failed",
+            "This image model uses a custom Diffusers pipeline that the launch environment does not know yet.",
+            [{"label": "update Diffusers image dependencies", "op": "dependency", "package": "diffusers transformers accelerate"}],
+        ),
+        (
+            r"mflux-generate-qwen.*not found|mflux-generate.*not found|MLX image serving requires mflux|No module named ['\"]?mflux",
+            "MLX image serving requires mflux on this Apple Silicon server.",
+            [{"label": "install mflux in Cookbook Dependencies", "op": "dependency", "package": "mflux"}],
+        ),
+        (
+            r"mlx-lama-swift|odysseus-mlx-inpaint|mlx-lama-serve|LaMa / MI-GAN MLX inpainting models require",
+            "LaMa / MI-GAN MLX inpainting requires an Odysseus-compatible mlx-lama-swift bridge on this Apple Silicon server.",
+            [{"label": "build mlx-lama-swift bridge and put odysseus-mlx-inpaint or mlx-lama-serve on PATH", "op": "dependency", "package": "mlx_lama_swift"}],
+        ),
+        (
+            r"mlx-ddcolor-swift|odysseus-mlx-colorize|mlx-ddcolor-serve|DDColor MLX models require",
+            "DDColor MLX colorization requires an Odysseus-compatible mlx-ddcolor-swift bridge on this Apple Silicon server.",
+            [{"label": "build mlx-ddcolor-swift bridge and put odysseus-mlx-colorize or mlx-ddcolor-serve on PATH", "op": "dependency", "package": "mlx_ddcolor_swift"}],
+        ),
         (
             r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
             "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
@@ -1358,9 +1422,9 @@ def _diagnose_serve_output(text: str) -> dict | None:
             [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}],
         ),
         (
-            r"No module named 'torch'|No module named torch|No module named 'diffusers'|No module named diffusers",
-            "Diffusion serving requires PyTorch and diffusers.",
-            [{"label": "install diffusers[torch] in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch]"}],
+            r"No module named 'torch'|No module named torch|No module named 'torchvision'|No module named torchvision|No module named 'diffusers'|No module named diffusers|No module named 'scipy'|No module named scipy|install scipy if you want to use beta sigmas|requires the Torchvision library",
+            "Diffusion serving requires PyTorch, Torchvision, Diffusers, Accelerate, and SciPy.",
+            [{"label": "install Diffusers image deps in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch] torchvision accelerate scipy python-multipart"}],
         ),
         (
             r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review",

routes/cookbook_routes.py

@@ -50,7 +50,7 @@ from routes.cookbook_helpers import (
     _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
     _validate_local_dir, _validate_gpus, _shell_path,
     _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
-    _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
+    _safe_env_prefix, _local_windows_bash_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
     _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
     load_stored_hf_token,
     _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
@@ -73,6 +73,47 @@ _HF_TOKEN_STATUS_SNIPPET = (
 )
 
 
+def _windows_local_pid_record_line(pid_path: Path, ready_path: Path) -> str:
+    """Build the Git Bash prelude that records a Win32-stoppable PID.
+
+    Python publishes the detached outer process's Win32 PID first, then touches
+    ``ready_path``. The inner Git Bash runner waits for that publication before
+    replacing the fallback with its own Win32 PID from /proc/<msys-pid>/winpid.
+
+    Missing, malformed, or late mappings leave the valid outer PID untouched.
+    """
+    pp = shlex.quote(pid_path.as_posix())
+    rp = shlex.quote(ready_path.as_posix())
+    return (
+        "i=0; "
+        f"while [ ! -e {rp} ] && [ \"$i\" -lt 500 ]; do "
+        "i=$((i+1)); sleep 0.01; done; "
+        f"if [ -e {rp} ]; then "
+        "winpid=\"$(cat /proc/$$/winpid 2>/dev/null || true)\"; "
+        "case \"$winpid\" in ''|*[!0-9]*) ;; "
+        f"*) printf '%s\\n' \"$winpid\" > {pp} ;; esac; "
+        "fi; "
+        f"rm -f {rp}"
+    )
+
+
+def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
+    """Write the MLX image API helper next to the tmux runner on remote hosts."""
+    script_path = Path(__file__).resolve().parents[1] / "scripts" / "mlx_image_server.py"
+    try:
+        script = script_path.read_text(encoding="utf-8")
+    except Exception as e:
+        logger.warning("Failed to read mlx_image_server.py: %s", e)
+        runner_lines.append('echo "ERROR: Odysseus could not prepare the MLX image server helper."')
+        runner_lines.append('ODYSSEUS_PREFLIGHT_EXIT=127')
+        return
+    runner_lines.append('mkdir -p scripts')
+    runner_lines.append("cat > scripts/mlx_image_server.py <<'PY'")
+    runner_lines.extend(script.splitlines())
+    runner_lines.append("PY")
+    runner_lines.append('chmod +x scripts/mlx_image_server.py 2>/dev/null || true')
+
+
 def _venv_root_from_serve_cmd(cmd: str) -> str:
     """Best-effort venv root from an absolute venv python in a serve command."""
     try:
@@ -492,6 +533,11 @@ def setup_cookbook_routes() -> APIRouter:
                 "MLX LM is not installed on this server.",
                 [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
             ),
+            (
+                r"OmniGen2Pipeline|module diffusers has no attribute .*Pipeline|custom_pipeline=.*failed",
+                "This image model uses a custom Diffusers pipeline that the launch environment does not know yet.",
+                [{"label": "update Diffusers image dependencies", "op": "dependency", "package": "diffusers transformers accelerate"}],
+            ),
             (
                 r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
                 "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
@@ -530,9 +576,9 @@ def setup_cookbook_routes() -> APIRouter:
                 [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}],
             ),
             (
-                r"No module named 'torch'|No module named torch|No module named 'diffusers'|No module named diffusers",
-                "Diffusion serving requires PyTorch and diffusers.",
-                [{"label": "install diffusers[torch] in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch]"}],
+                r"No module named 'torch'|No module named torch|No module named 'torchvision'|No module named torchvision|No module named 'diffusers'|No module named diffusers|No module named 'scipy'|No module named scipy|install scipy if you want to use beta sigmas|requires the Torchvision library",
+                "Diffusion serving requires PyTorch, Torchvision, Diffusers, Accelerate, and SciPy.",
+                [{"label": "install Diffusers image deps in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch] torchvision accelerate scipy python-multipart"}],
             ),
             (
                 r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review",
@@ -956,15 +1002,18 @@ def setup_cookbook_routes() -> APIRouter:
         directly (simple commands only). Returns the launched job record."""
         log_path = TMUX_LOG_DIR / f"{session_id}.log"
         pid_path = TMUX_LOG_DIR / f"{session_id}.pid"
+        pid_ready_path: Path | None = None
         bash = find_bash()
         if bash:
             # Run the existing bash wrapper verbatim through Git Bash, redirecting
             # all output to the log the poller reads. Paths handed to bash use
             # POSIX form + shell-quoting so drive paths / spaces survive.
             inner = TMUX_LOG_DIR / f"{session_id}_run.sh"
-            pp = shlex.quote(pid_path.as_posix())
+            pid_ready_path = TMUX_LOG_DIR / f"{session_id}.pid.ready"
+            pid_ready_path.unlink(missing_ok=True)
             inner.write_text(
-                f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n",
+                _windows_local_pid_record_line(pid_path, pid_ready_path) + "\n"
+                + "\n".join(bash_lines) + "\n",
                 encoding="utf-8",
             )
             lp = shlex.quote(log_path.as_posix())
@@ -998,7 +1047,18 @@ def setup_cookbook_routes() -> APIRouter:
             env=env,
             **detached_popen_kwargs(),
         )
+        # Publish a valid Win32 ancestor first. The Git Bash runner may then
+        # replace it with its own Win32 pid, but never before this fallback exists.
         pid_path.write_text(str(proc.pid), encoding="utf-8")
+        if pid_ready_path is not None:
+            try:
+                pid_ready_path.touch()
+            except OSError as e:
+                logger.warning(
+                    "Could not publish Windows local PID handoff for %s: %s",
+                    session_id,
+                    e,
+                )
         return {"pid": proc.pid, "log_path": str(log_path)}
 
     @router.post("/api/model/download")
@@ -1276,7 +1336,7 @@ def setup_cookbook_routes() -> APIRouter:
             # Local: run hf download in the background (tmux on POSIX, a detached
             # process + logfile on Windows where tmux doesn't exist).
             if req.env_prefix:
-                lines.append(_safe_env_prefix(req.env_prefix))
+                lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 lines.append("deactivate 2>/dev/null; hash -r")
             # Show whether the HF token reached this run (masked) — tells a gated
@@ -1433,9 +1493,11 @@ def setup_cookbook_routes() -> APIRouter:
                     "nb_files": m["nb_files"],
                     "has_incomplete": m["has_incomplete"],
                     "status": "downloading" if m["has_incomplete"] else "ready",
-                    "path": m.get("path", ""),
-                    "is_diffusion": m.get("is_diffusion", False),
-                }
+	                    "path": m.get("path", ""),
+	                    "is_diffusion": m.get("is_diffusion", False),
+	                    "is_video": m.get("is_video", False),
+	                    "is_adapter": m.get("is_adapter", False),
+	                }
                 if m.get("is_local_dir"):
                     entry["is_local_dir"] = True
                 if m.get("is_gguf"):
@@ -1460,6 +1522,7 @@ def setup_cookbook_routes() -> APIRouter:
         """Register a diffusion model as an image endpoint so it appears in the model selector."""
         import re
         from core.database import SessionLocal, ModelEndpoint
+        from src.settings import load_settings, save_settings
 
         # Parse port from command (--port NNNN), default 8100 for diffusion_server
         port_match = re.search(r'--port\s+(\d+)', req.cmd)
@@ -1477,6 +1540,7 @@ def setup_cookbook_routes() -> APIRouter:
         # Friendly display name from repo_id
         short_name = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id
         display_name = f"{short_name} (image)"
+        pinned_models = [req.repo_id] if req.repo_id else []
 
         db = SessionLocal()
         try:
@@ -1486,7 +1550,16 @@ def setup_cookbook_routes() -> APIRouter:
                 existing.is_enabled = True
                 existing.model_type = "image"
                 existing.name = display_name
+                existing.endpoint_kind = "local"
+                existing.model_refresh_mode = "manual"
+                if pinned_models:
+                    existing.cached_models = json.dumps(pinned_models)
+                    existing.pinned_models = json.dumps(pinned_models)
                 db.commit()
+                settings = load_settings()
+                if settings.get("image_gen_enabled") is not True:
+                    settings["image_gen_enabled"] = True
+                    save_settings(settings)
                 logger.info(f"Updated existing image endpoint: {base_url}")
                 return existing.id
 
@@ -1498,9 +1571,18 @@ def setup_cookbook_routes() -> APIRouter:
                 api_key=None,
                 is_enabled=True,
                 model_type="image",
+                endpoint_kind="local",
+                model_refresh_mode="manual",
+                cached_models=json.dumps(pinned_models) if pinned_models else None,
+                pinned_models=json.dumps(pinned_models) if pinned_models else None,
             )
             db.add(ep)
             db.commit()
+            settings = load_settings()
+            settings["image_gen_enabled"] = True
+            if not settings.get("image_model"):
+                settings["image_model"] = req.repo_id
+            save_settings(settings)
             logger.info(f"Auto-registered image endpoint: {display_name} @ {base_url}")
             return ep_id
         except Exception as e:
@@ -2084,7 +2166,7 @@ def setup_cookbook_routes() -> APIRouter:
             if req.gpus:
                 runner_lines.append(f"export CUDA_VISIBLE_DEVICES='{req.gpus}'")
             if req.env_prefix:
-                runner_lines.append(_safe_env_prefix(req.env_prefix))
+                runner_lines.append(_safe_env_prefix(_local_windows_bash_env_prefix(req.env_prefix) if local_windows else req.env_prefix))
             else:
                 runner_lines.append("deactivate 2>/dev/null; hash -r")
             _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)
@@ -2356,24 +2438,8 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('fi')
             elif "sglang.launch_server" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
-                runner_lines.append('if ! command -v sglang &>/dev/null; then')
-                runner_lines.append('  echo "ERROR: SGLang is not installed."')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
-                runner_lines.append('elif ! ODYSSEUS_SGLANG_IMPORT_ERROR="$(python3 -c "import sglang" 2>&1)"; then')
-                runner_lines.append('  echo "ERROR: SGLang is installed but failed to import."')
-                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_SGLANG_IMPORT_ERROR"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
-                runner_lines.append('fi')
-            elif "mlx_lm.server" in req.cmd:
-                runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
-                runner_lines.append('if ! ODYSSEUS_MLX_IMPORT_ERROR="$(python3 -c "import mlx_lm" 2>&1)"; then')
-                runner_lines.append('  echo "ERROR: MLX LM is not installed."')
-                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_MLX_IMPORT_ERROR"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
-                runner_lines.append('fi')
                 runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
-                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then')
-                runner_lines.append('  ODYSSEUS_MLX_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
+                runner_lines.append('ODYSSEUS_SGLANG_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                 runner_lines.append('import shlex, sys')
                 runner_lines.append('parts = shlex.split(sys.argv[1])')
                 runner_lines.append('py = "python3"')
@@ -2384,6 +2450,36 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('print(py)')
                 runner_lines.append('PY')
                 runner_lines.append(')"')
+                runner_lines.append('if ! "$ODYSSEUS_SGLANG_CMD_PY" -c "import sglang" &>/dev/null; then')
+                runner_lines.append('  if ! command -v sglang &>/dev/null; then')
+                runner_lines.append('    echo "ERROR: SGLang is not installed."')
+                runner_lines.append('  else')
+                runner_lines.append('    echo "ERROR: SGLang is installed but failed to import in the launch Python."')
+                runner_lines.append('  fi')
+                runner_lines.append('  ODYSSEUS_SGLANG_IMPORT_ERROR="$("$ODYSSEUS_SGLANG_CMD_PY" -c "import sglang" 2>&1)"')
+                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_SGLANG_IMPORT_ERROR"')
+                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('fi')
+            elif "mlx_lm.server" in req.cmd:
+                runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
+                runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
+                runner_lines.append('ODYSSEUS_MLX_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
+                runner_lines.append('import shlex, sys')
+                runner_lines.append('parts = shlex.split(sys.argv[1])')
+                runner_lines.append('py = "python3"')
+                runner_lines.append('for i, part in enumerate(parts):')
+                runner_lines.append('    if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:')
+                runner_lines.append('        py = part')
+                runner_lines.append('        break')
+                runner_lines.append('print(py)')
+                runner_lines.append('PY')
+                runner_lines.append(')"')
+                runner_lines.append('if ! ODYSSEUS_MLX_IMPORT_ERROR="$("$ODYSSEUS_MLX_CMD_PY" -c "import mlx_lm" 2>&1)"; then')
+                runner_lines.append('  echo "ERROR: MLX LM is not installed in the launch Python: $ODYSSEUS_MLX_CMD_PY"')
+                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_MLX_IMPORT_ERROR"')
+                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('fi')
+                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then')
                 runner_lines.append('  ODYSSEUS_SERVE_CMD="$("$ODYSSEUS_MLX_CMD_PY" - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                 runner_lines.append('import json, os, shlex, sys')
                 runner_lines.append('from pathlib import Path')
@@ -2474,10 +2570,111 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('PY')
                 runner_lines.append(')"')
                 runner_lines.append('fi')
+            elif "scripts/mlx_image_server.py" in req.cmd or ".mlx_image_server.py" in req.cmd:
+                _append_mlx_image_server_script(runner_lines)
+                runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
+                runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
+                runner_lines.append('ODYSSEUS_MLX_IMAGE_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
+                runner_lines.append('import shlex, sys')
+                runner_lines.append('parts = shlex.split(sys.argv[1])')
+                runner_lines.append('py = "python3"')
+                runner_lines.append('for part in parts:')
+                runner_lines.append('    if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:')
+                runner_lines.append('        py = part')
+                runner_lines.append('        break')
+                runner_lines.append('print(py)')
+                runner_lines.append('PY')
+                runner_lines.append(')"')
+                runner_lines.append('ODYSSEUS_MLX_IMAGE_BIN_DIR="$(dirname "$ODYSSEUS_MLX_IMAGE_CMD_PY" 2>/dev/null || true)"')
+                runner_lines.append('if [ -n "$ODYSSEUS_MLX_IMAGE_BIN_DIR" ]; then export PATH="$ODYSSEUS_MLX_IMAGE_BIN_DIR:$PATH"; fi')
+                runner_lines.append('if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import fastapi, uvicorn, multipart" >/dev/null 2>&1; then')
+                runner_lines.append('  echo "ERROR: MLX image serving requires FastAPI + uvicorn + python-multipart in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY. Install the MLX image dependencies in Cookbook Dependencies."')
+                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('fi')
+                runner_lines.append('ODYSSEUS_MLX_IMAGE_MODEL="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
+                runner_lines.append('import shlex, sys')
+                runner_lines.append('parts = shlex.split(sys.argv[1])')
+                runner_lines.append('model = ""')
+                runner_lines.append('for i, part in enumerate(parts):')
+                runner_lines.append('    if part == "--model" and i + 1 < len(parts):')
+                runner_lines.append('        model = parts[i + 1]')
+                runner_lines.append('        break')
+                runner_lines.append('print(model)')
+                runner_lines.append('PY')
+                runner_lines.append(')"')
+                runner_lines.append('if printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; then')
+                runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import mlx, mlx_vlm, transformers, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: HiDream MLX serving needs the model requirements in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart mlx mlx-vlm \'transformers>=4.57.0,<6.0\' huggingface_hub safetensors numpy pillow tqdm sentencepiece hf_transfer"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; then')
+                runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import boogu_image_mlx, mlx, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: Boogu MLX serving needs boogu-image-mlx in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U git+https://github.com/xocialize/boogu-image-mlx.git fastapi uvicorn python-multipart pillow"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; then')
+                runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: DDColor MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('  if ! command -v odysseus-mlx-colorize >/dev/null 2>&1 && ! command -v mlx-ddcolor-serve >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: DDColor MLX serving requires the Odysseus mlx-ddcolor-swift bridge on PATH: odysseus-mlx-colorize or mlx-ddcolor-serve."')
+                runner_lines.append('    echo "Build it from swift/odysseus-mlx-image-bridge in Cookbook Dependencies."')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('  ODYSSEUS_DDCOLOR_BIN="$(command -v odysseus-mlx-colorize 2>/dev/null || command -v mlx-ddcolor-serve 2>/dev/null || true)"')
+                runner_lines.append('  if [ -n "$ODYSSEUS_DDCOLOR_BIN" ]; then')
+                runner_lines.append('    ODYSSEUS_DDCOLOR_DIR="$(dirname "$ODYSSEUS_DDCOLOR_BIN")"')
+                runner_lines.append('    if [ ! -f "$ODYSSEUS_DDCOLOR_DIR/mlx.metallib" ] && [ ! -f "$ODYSSEUS_DDCOLOR_DIR/default.metallib" ]; then')
+                runner_lines.append('      echo "ERROR: DDColor MLX serving found the Swift runner, but mlx.metallib/default.metallib is missing next to it."')
+                runner_lines.append('      echo "Run the DDColor MLX image editing dependency install again; it copies mlx.metallib from the launch Python MLX package."')
+                runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('    fi')
+                runner_lines.append('  fi')
+                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; then')
+                runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('  if ! command -v odysseus-mlx-inpaint >/dev/null 2>&1 && ! command -v mlx-lama-serve >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving requires the Odysseus mlx-lama-swift bridge on PATH: odysseus-mlx-inpaint or mlx-lama-serve."')
+                runner_lines.append('    echo "Build it from swift/odysseus-mlx-image-bridge in Cookbook Dependencies."')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
+                runner_lines.append('  ODYSSEUS_INPAINT_BIN="$(command -v odysseus-mlx-inpaint 2>/dev/null || command -v mlx-lama-serve 2>/dev/null || true)"')
+                runner_lines.append('  if [ -n "$ODYSSEUS_INPAINT_BIN" ]; then')
+                runner_lines.append('    ODYSSEUS_INPAINT_DIR="$(dirname "$ODYSSEUS_INPAINT_BIN")"')
+                runner_lines.append('    if [ ! -f "$ODYSSEUS_INPAINT_DIR/mlx.metallib" ] && [ ! -f "$ODYSSEUS_INPAINT_DIR/default.metallib" ]; then')
+                runner_lines.append('      echo "ERROR: LaMa / MI-GAN MLX serving found the Swift runner, but mlx.metallib/default.metallib is missing next to it."')
+                runner_lines.append('      echo "Run the LaMa / MI-GAN MLX image editing dependency install again; it copies mlx.metallib from the launch Python MLX package."')
+                runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('    fi')
+                runner_lines.append('  fi')
+                runner_lines.append('elif ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
+                runner_lines.append('  echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('  echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
+                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('fi')
             elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
-                runner_lines.append('if ! ODYSSEUS_DIFFUSION_IMPORT_ERROR="$(python3 -c "import torch, diffusers" 2>&1)"; then')
-                runner_lines.append('  echo "ERROR: Diffusion serving requires PyTorch + diffusers."')
+                runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
+                runner_lines.append('ODYSSEUS_DIFFUSION_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
+                runner_lines.append('import shlex, sys')
+                runner_lines.append('parts = shlex.split(sys.argv[1])')
+                runner_lines.append('py = "python3"')
+                runner_lines.append('for part in parts:')
+                runner_lines.append('    if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:')
+                runner_lines.append('        py = part')
+                runner_lines.append('        break')
+                runner_lines.append('print(py)')
+                runner_lines.append('PY')
+                runner_lines.append(')"')
+                runner_lines.append('if ! ODYSSEUS_DIFFUSION_IMPORT_ERROR="$("$ODYSSEUS_DIFFUSION_CMD_PY" -c "import torch, torchvision, diffusers" 2>&1)"; then')
+                runner_lines.append('  echo "ERROR: Diffusion serving requires PyTorch + Torchvision + diffusers in the launch Python: $ODYSSEUS_DIFFUSION_CMD_PY."')
                 runner_lines.append('  printf "%s\\n" "$ODYSSEUS_DIFFUSION_IMPORT_ERROR"')
                 runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('fi')
@@ -2588,8 +2785,8 @@ def setup_cookbook_routes() -> APIRouter:
         # endpoint; any other real model serve (i.e. not a pip-install task) gets
         # a local LLM endpoint pointed at its /v1.
         endpoint_id = None
-        is_diffusion = "diffusion_server.py" in req.cmd
-        if is_diffusion:
+        is_image_endpoint = "diffusion_server.py" in req.cmd or "mlx_image_server.py" in req.cmd
+        if is_image_endpoint:
             endpoint_id = _auto_register_image_endpoint(req, remote)
         elif not is_pip_install:
             endpoint_id = _auto_register_llm_endpoint(req, remote)
@@ -2605,7 +2802,7 @@ def setup_cookbook_routes() -> APIRouter:
         # if N != 0 within the watch window, delete the endpoint we just
         # created. Skipped for diffusion (different image-endpoint cleanup
         # path) and pip-install tasks (no endpoint to drop).
-        if endpoint_id and not is_diffusion and not is_pip_install:
+        if endpoint_id and not is_image_endpoint and not is_pip_install:
             asyncio.create_task(_serve_crash_watchdog(
                 endpoint_id=endpoint_id,
                 session_id=session_id,

routes/document/__init__.py

@@ -0,0 +1,6 @@
+"""Document route domain package (slice 2m, #4082/#4071).
+
+Contains document_routes.py and document_helpers.py, migrated from the flat
+routes/ directory. Backward-compat shims at routes/document_routes.py and
+routes/document_helpers.py re-export from here.
+"""

routes/document/document_helpers.py

@@ -0,0 +1,243 @@
+"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+
+"""Document routes — CRUD for living documents with version history."""
+
+import logging
+import os
+import re
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException, Request
+from pydantic import BaseModel
+
+from core.database import Document, DocumentVersion
+from core.database import Session as DbSession
+from src.auth_helpers import _auth_disabled
+from src.upload_handler import UploadHandler
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class DocumentCreate(BaseModel):
+    session_id: Optional[str] = None
+    title: str = "Untitled"
+    language: Optional[str] = None
+    content: str = ""
+
+class DocumentUpdate(BaseModel):
+    content: str
+    summary: Optional[str] = None
+    force_version: bool = False
+
+class DocumentPatch(BaseModel):
+    title: Optional[str] = None
+    language: Optional[str] = None
+    session_id: Optional[str] = None  # link/unlink document to a session
+
+
+# ---- Helpers ----
+
+def _doc_to_dict(doc: Document) -> Dict[str, Any]:
+    return {
+        "id": doc.id,
+        "session_id": doc.session_id,
+        "title": doc.title,
+        "language": doc.language,
+        "current_content": doc.current_content,
+        "version_count": doc.version_count,
+        "is_active": doc.is_active,
+        "archived": bool(getattr(doc, "archived", False)),
+        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
+        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
+        # Source-email provenance (set when doc was created from an email
+        # attachment) — drives the "Send signed reply" menu item.
+        "source_email_uid":        getattr(doc, "source_email_uid", None),
+        "source_email_folder":     getattr(doc, "source_email_folder", None),
+        "source_email_account_id": getattr(doc, "source_email_account_id", None),
+        "source_email_message_id": getattr(doc, "source_email_message_id", None),
+    }
+
+def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
+    return {
+        "id": v.id,
+        "document_id": v.document_id,
+        "version_number": v.version_number,
+        "content": v.content,
+        "summary": v.summary,
+        "source": v.source,
+        "created_at": v.created_at.isoformat() if v.created_at else None,
+    }
+
+
+def _verify_doc_owner(db, doc: Document, user: str):
+    """Verify `user` owns this document. Raise 404 if not.
+
+    Documents now carry their own `owner` column, so a doc whose session
+    was deleted (session_id → NULL) can still prove ownership and stay
+    openable / cloneable. We trust that column first and only fall back to
+    the session join for any not-yet-backfilled legacy row.
+    """
+    if user is None:
+        if _auth_disabled():
+            return  # Single-user / no-auth mode: allow access
+        raise HTTPException(403, "Authentication required")
+    if doc.owner is not None:
+        if doc.owner != user:
+            raise HTTPException(404, "Document not found")
+        return
+    # Legacy fallback: derive ownership from the linked session.
+    if not doc.session_id:
+        raise HTTPException(404, "Document not found")
+    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
+    if not session or session.owner != user:
+        raise HTTPException(404, "Document not found")
+
+
+def _owner_session_filter(q, user):
+    """Restrict a documents query to those owned by `user`.
+
+    Documents now carry their own `owner` column (backfilled at boot from
+    the linked session, or assigned to the admin user for legacy/orphaned
+    docs). We filter on that directly rather than on a session join, so a
+    document whose session was deleted (session_id → NULL) still shows up
+    for its owner instead of silently vanishing from the Library + search.
+
+    The owner backfill runs in init_db before the app serves requests, so
+    by the time this filter is live there are no NULL-owner rows to leak;
+    we therefore match the owner strictly for authenticated callers."""
+    if not user:
+        if user == "" or _auth_disabled():
+            return q
+        return q.filter(False)
+    return q.filter(Document.owner == user)
+
+
+
+def _slug(name: str) -> str:
+    """Filesystem-friendly version of a document title.
+
+    Whitespace becomes underscores; other unsafe punctuation is dropped.
+    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
+    """
+    import re as _re
+    s = (name or "").strip()
+    # Drop the trailing extension if the title happens to include one
+    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
+    s = _re.sub(r'\s+', '_', s)
+    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
+    s = _re.sub(r'_+', '_', s).strip('_')
+    return s or "form"
+
+
+# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
+_PDF_RENDER_SCALE = 2.0
+
+
+def _upload_path_inside(upload_dir: str, path: str) -> bool:
+    base = os.path.realpath(upload_dir)
+    p = os.path.realpath(path)
+    try:
+        return os.path.commonpath([base, p]) == base
+    except Exception:
+        return False
+
+
+def _resolve_user_upload_path(
+    upload_handler: Any,
+    upload_id: str,
+    owner: Optional[str],
+    auth_manager=None,
+) -> Optional[str]:
+    """Resolve an upload id to a filesystem path the caller may read."""
+    if upload_handler is None:
+        return None
+    resolved = upload_handler.resolve_upload(
+        upload_id,
+        owner=owner,
+        auth_manager=auth_manager,
+    )
+    if not isinstance(resolved, dict) or not resolved:
+        return None
+    path = resolved.get("path")
+    upload_dir = getattr(upload_handler, "upload_dir", None)
+    if path and upload_dir and not _upload_path_inside(upload_dir, path):
+        logger.warning("Upload path outside upload directory: %s", path)
+        return None
+    return path
+
+
+def _locate_upload(
+    upload_dir: str,
+    file_id: str,
+    owner: Optional[str] = None,
+    auth_manager=None,
+    upload_handler: Any = None,
+):
+    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
+    if upload_handler is None:
+        from src.upload_handler import UploadHandler
+
+        base_dir = os.path.dirname(os.path.abspath(upload_dir))
+        upload_handler = UploadHandler(base_dir, upload_dir)
+    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
+
+
+def _assert_pdf_marker_upload_owned(
+    request: Request,
+    content: str,
+    user: Optional[str],
+    upload_handler: Any,
+) -> None:
+    """Reject document content whose pdf_source marker points at another user's upload."""
+    if upload_handler is None:
+        return
+    from src.pdf_form_doc import find_source_upload_id
+
+    upload_id = find_source_upload_id(content or "")
+    if not upload_id:
+        return
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
+        raise HTTPException(
+            400,
+            "Document PDF marker references an upload you do not own",
+        )
+
+
+def _derive_title(content: str) -> str:
+    """Derive a title from document content."""
+    import re
+    if not isinstance(content, str):
+        return "Untitled"
+    text = content.strip()
+    if not text:
+        return "Untitled"
+
+    # Markdown header
+    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
+    if md:
+        title = md.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # HTML heading
+    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
+    if html:
+        title = html.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # First non-empty line (if short enough)
+    for line in text.split('\n'):
+        line = line.strip()
+        if line and 2 <= len(line) <= 60:
+            title = re.sub(r'[:#*`]+$', '', line).strip()
+            if title and len(title) > 50:
+                title = title[:48] + "…"
+            return title or "Untitled"
+
+    return "Untitled"

routes/document_helpers.py

@@ -1,243 +1,14 @@
-"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
 
-"""Document routes — CRUD for living documents with version history."""
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.document_helpers``, ``from routes.document_helpers import
+X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
+pattern used by test_security_regressions.py all operate on the *same* object.
+Keeps existing import paths working after slice 2m (#4082/#4071).
+"""
 
-import logging
-import os
-import re
-from typing import Any, Dict, Optional
+import sys as _sys
 
-from fastapi import HTTPException, Request
-from pydantic import BaseModel
+from routes.document import document_helpers as _canonical  # noqa: F401
 
-from core.database import Document, DocumentVersion
-from core.database import Session as DbSession
-from src.auth_helpers import _auth_disabled
-from src.upload_handler import UploadHandler
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class DocumentCreate(BaseModel):
-    session_id: Optional[str] = None
-    title: str = "Untitled"
-    language: Optional[str] = None
-    content: str = ""
-
-class DocumentUpdate(BaseModel):
-    content: str
-    summary: Optional[str] = None
-    force_version: bool = False
-
-class DocumentPatch(BaseModel):
-    title: Optional[str] = None
-    language: Optional[str] = None
-    session_id: Optional[str] = None  # link/unlink document to a session
-
-
-# ---- Helpers ----
-
-def _doc_to_dict(doc: Document) -> Dict[str, Any]:
-    return {
-        "id": doc.id,
-        "session_id": doc.session_id,
-        "title": doc.title,
-        "language": doc.language,
-        "current_content": doc.current_content,
-        "version_count": doc.version_count,
-        "is_active": doc.is_active,
-        "archived": bool(getattr(doc, "archived", False)),
-        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
-        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
-        # Source-email provenance (set when doc was created from an email
-        # attachment) — drives the "Send signed reply" menu item.
-        "source_email_uid":        getattr(doc, "source_email_uid", None),
-        "source_email_folder":     getattr(doc, "source_email_folder", None),
-        "source_email_account_id": getattr(doc, "source_email_account_id", None),
-        "source_email_message_id": getattr(doc, "source_email_message_id", None),
-    }
-
-def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
-    return {
-        "id": v.id,
-        "document_id": v.document_id,
-        "version_number": v.version_number,
-        "content": v.content,
-        "summary": v.summary,
-        "source": v.source,
-        "created_at": v.created_at.isoformat() if v.created_at else None,
-    }
-
-
-def _verify_doc_owner(db, doc: Document, user: str):
-    """Verify `user` owns this document. Raise 404 if not.
-
-    Documents now carry their own `owner` column, so a doc whose session
-    was deleted (session_id → NULL) can still prove ownership and stay
-    openable / cloneable. We trust that column first and only fall back to
-    the session join for any not-yet-backfilled legacy row.
-    """
-    if user is None:
-        if _auth_disabled():
-            return  # Single-user / no-auth mode: allow access
-        raise HTTPException(403, "Authentication required")
-    if doc.owner is not None:
-        if doc.owner != user:
-            raise HTTPException(404, "Document not found")
-        return
-    # Legacy fallback: derive ownership from the linked session.
-    if not doc.session_id:
-        raise HTTPException(404, "Document not found")
-    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
-    if not session or session.owner != user:
-        raise HTTPException(404, "Document not found")
-
-
-def _owner_session_filter(q, user):
-    """Restrict a documents query to those owned by `user`.
-
-    Documents now carry their own `owner` column (backfilled at boot from
-    the linked session, or assigned to the admin user for legacy/orphaned
-    docs). We filter on that directly rather than on a session join, so a
-    document whose session was deleted (session_id → NULL) still shows up
-    for its owner instead of silently vanishing from the Library + search.
-
-    The owner backfill runs in init_db before the app serves requests, so
-    by the time this filter is live there are no NULL-owner rows to leak;
-    we therefore match the owner strictly for authenticated callers."""
-    if not user:
-        if user == "" or _auth_disabled():
-            return q
-        return q.filter(False)
-    return q.filter(Document.owner == user)
-
-
-
-def _slug(name: str) -> str:
-    """Filesystem-friendly version of a document title.
-
-    Whitespace becomes underscores; other unsafe punctuation is dropped.
-    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
-    """
-    import re as _re
-    s = (name or "").strip()
-    # Drop the trailing extension if the title happens to include one
-    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
-    s = _re.sub(r'\s+', '_', s)
-    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
-    s = _re.sub(r'_+', '_', s).strip('_')
-    return s or "form"
-
-
-# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
-_PDF_RENDER_SCALE = 2.0
-
-
-def _upload_path_inside(upload_dir: str, path: str) -> bool:
-    base = os.path.realpath(upload_dir)
-    p = os.path.realpath(path)
-    try:
-        return os.path.commonpath([base, p]) == base
-    except Exception:
-        return False
-
-
-def _resolve_user_upload_path(
-    upload_handler: Any,
-    upload_id: str,
-    owner: Optional[str],
-    auth_manager=None,
-) -> Optional[str]:
-    """Resolve an upload id to a filesystem path the caller may read."""
-    if upload_handler is None:
-        return None
-    resolved = upload_handler.resolve_upload(
-        upload_id,
-        owner=owner,
-        auth_manager=auth_manager,
-    )
-    if not isinstance(resolved, dict) or not resolved:
-        return None
-    path = resolved.get("path")
-    upload_dir = getattr(upload_handler, "upload_dir", None)
-    if path and upload_dir and not _upload_path_inside(upload_dir, path):
-        logger.warning("Upload path outside upload directory: %s", path)
-        return None
-    return path
-
-
-def _locate_upload(
-    upload_dir: str,
-    file_id: str,
-    owner: Optional[str] = None,
-    auth_manager=None,
-    upload_handler: Any = None,
-):
-    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
-    if upload_handler is None:
-        from src.upload_handler import UploadHandler
-
-        base_dir = os.path.dirname(os.path.abspath(upload_dir))
-        upload_handler = UploadHandler(base_dir, upload_dir)
-    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
-
-
-def _assert_pdf_marker_upload_owned(
-    request: Request,
-    content: str,
-    user: Optional[str],
-    upload_handler: Any,
-) -> None:
-    """Reject document content whose pdf_source marker points at another user's upload."""
-    if upload_handler is None:
-        return
-    from src.pdf_form_doc import find_source_upload_id
-
-    upload_id = find_source_upload_id(content or "")
-    if not upload_id:
-        return
-    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
-    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
-        raise HTTPException(
-            400,
-            "Document PDF marker references an upload you do not own",
-        )
-
-
-def _derive_title(content: str) -> str:
-    """Derive a title from document content."""
-    import re
-    if not isinstance(content, str):
-        return "Untitled"
-    text = content.strip()
-    if not text:
-        return "Untitled"
-
-    # Markdown header
-    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
-    if md:
-        title = md.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # HTML heading
-    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
-    if html:
-        title = html.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # First non-empty line (if short enough)
-    for line in text.split('\n'):
-        line = line.strip()
-        if line and 2 <= len(line) <= 60:
-            title = re.sub(r'[:#*`]+$', '', line).strip()
-            if title and len(title) > 50:
-                title = title[:48] + "…"
-            return title or "Untitled"
-
-    return "Untitled"
+_sys.modules[__name__] = _canonical

routes/email_helpers.py

@@ -246,6 +246,8 @@ import re as _re_reply
 # serves replies and summaries (any fenced final-output block).
 _REPLY_OPEN_RE = _re_reply.compile(r"<<<\s*(?:REPLY|SUMMARY|OUTPUT)\s*>>+", _re_reply.I)
 _REPLY_CLOSE_RE = _re_reply.compile(r"<<<\s*END\s*>>+", _re_reply.I)
+_REPLY_ROLE_MARKER_RE = _re_reply.compile(r"</?\|(?:assistant|assistan|user|system|tool)\|>?|</\|end\|>?", _re_reply.I)
+_SUMMARY_BULLET_RE = _re_reply.compile(r"^(?:[-*\u2022]\s+|\d+[.)]\s+)")
 
 
 def _extract_reply(text: str) -> str:
@@ -272,9 +274,129 @@ def _extract_reply(text: str) -> str:
     # Drop any stray/duplicate marker tokens, then strip think markup.
     t = _REPLY_OPEN_RE.sub("", t)
     t = _REPLY_CLOSE_RE.sub("", t)
+    t = _REPLY_ROLE_MARKER_RE.sub("", t)
     return _strip_think(t).strip()
 
 
+def _build_email_summary_messages(sender: str, subject: str, body_for_llm: str) -> list[dict[str, str]]:
+    return [
+        {
+            "role": "system",
+            "content": (
+                "You are an email summarizer. Format: 1-3 short bullet points "
+                "(use '- '). Cover: main point, action items, deadlines. If the "
+                "email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR "
+                "CONTENTS - pull invoice totals, deadlines, key clauses, concrete "
+                "numbers/dates from PDFs/docs into the bullets. Be terse.\n\n"
+                "OUTPUT FORMAT: Put ONLY the bullet points between these exact "
+                "markers, each on its own line:\n"
+                "<<<SUMMARY>>>\n"
+                "- ...\n"
+                "<<<END>>>\n"
+                "Any reasoning must come BEFORE <<<SUMMARY>>> (ideally inside "
+                "<think>...</think>). Only the text between the markers is kept."
+            ),
+        },
+        {
+            "role": "user",
+            "content": (
+                f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}"
+                "\n\n---\n\nSummarize the email. Output the bullets between "
+                "<<<SUMMARY>>> and <<<END>>>."
+            ),
+        },
+    ]
+
+
+async def _generate_email_summary(
+    url: str,
+    model: str,
+    sender: str,
+    subject: str,
+    body_for_llm: str,
+    *,
+    headers: dict | None = None,
+    max_tokens: int = 8192,
+    timeout: int = 180,
+) -> str:
+    """Generate an interactive email summary through the shared LLM adapter."""
+    from src.llm_core import llm_call_async
+
+    raw = await llm_call_async(
+        url=url,
+        model=model,
+        messages=_build_email_summary_messages(sender, subject, body_for_llm),
+        temperature=0.3,
+        max_tokens=max_tokens,
+        headers=headers,
+        timeout=timeout,
+        workload="foreground",
+    )
+    return _normalize_email_summary(raw)
+
+
+async def _generate_scheduled_email_summary(
+    url: str,
+    model: str,
+    sender: str,
+    subject: str,
+    body_for_llm: str,
+    *,
+    headers: dict | None = None,
+    owner: str | None = None,
+    max_tokens: int = 8192,
+    timeout: int = 180,
+) -> str:
+    """Generate a scheduled summary through the background task candidate chain."""
+    from src.task_endpoint import task_llm_call_async
+
+    raw = await task_llm_call_async(
+        messages=_build_email_summary_messages(sender, subject, body_for_llm),
+        fallback_url=url,
+        fallback_model=model,
+        fallback_headers=headers,
+        owner=owner,
+        temperature=0.3,
+        max_tokens=max_tokens,
+        timeout=timeout,
+    )
+    return _normalize_email_summary(raw)
+
+
+def _normalize_email_summary(raw) -> str:
+    """Extract a stable cache/UI summary from provider output."""
+    raw_text = raw or ""
+    if _REPLY_OPEN_RE.search(raw_text):
+        summary = _extract_reply(raw_text)
+        if summary:
+            return summary
+
+    cleaned = _strip_think(raw_text).strip()
+    bullets = [
+        line.strip()
+        for line in cleaned.splitlines()
+        if _SUMMARY_BULLET_RE.match(line.strip())
+    ]
+    if bullets:
+        return "\n".join(bullets)
+    return cleaned.strip()
+
+
+EMAIL_SUMMARY_ERROR_CODE = "email_summary_unavailable"
+EMAIL_SUMMARY_ERROR_MESSAGE = "Failed to summarize"
+
+
+def _email_summary_failure_log_detail(exc: BaseException) -> str:
+    """Return useful provider-failure metadata without echoing exception text."""
+    detail = f"type={type(exc).__name__}"
+    status = getattr(exc, "status_code", None)
+    if status is None:
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+    if isinstance(status, int):
+        detail += f" status={status}"
+    return detail
+
+
 def _apply_email_style_mechanics(text: str) -> str:
     """Enforce deterministic writing-style mechanics that models often miss."""
     if not text:
@@ -1035,13 +1157,23 @@ def _coerce_imap_timeout_seconds(raw: str | None) -> int:
 _IMAP_TIMEOUT_SECONDS = _coerce_imap_timeout_seconds(os.environ.get("ODYSSEUS_IMAP_TIMEOUT_SECONDS"))
 
 
-def _open_imap_connection(host: str, port: int, *, starttls: bool, timeout: int = _IMAP_TIMEOUT_SECONDS):
+def _open_imap_connection(
+    host: str,
+    port: int,
+    *,
+    starttls: bool,
+    timeout: int = _IMAP_TIMEOUT_SECONDS,
+    ssl_context=None,
+):
     """Open an IMAP connection using the configured security mode."""
     port = int(port or 993)
     if starttls:
         conn = imaplib.IMAP4(host, port, timeout=timeout)
         try:
-            conn.starttls()
+            if ssl_context:
+                conn.starttls(ssl_context=ssl_context)
+            else:
+                conn.starttls()
         except Exception:
             # Don't leak the open plain socket if the STARTTLS upgrade is
             # rejected; close it before propagating. (#3174)
@@ -1051,7 +1183,8 @@ def _open_imap_connection(host: str, port: int, *, starttls: bool, timeout: int
                 pass
             raise
     elif port == 993:
-        conn = imaplib.IMAP4_SSL(host, port, timeout=timeout)
+        kwargs = {"ssl_context": ssl_context} if ssl_context else {}
+        conn = imaplib.IMAP4_SSL(host, port, timeout=timeout, **kwargs)
     else:
         conn = imaplib.IMAP4(host, port, timeout=timeout)
     try:

routes/email_pollers.py

@@ -40,6 +40,7 @@ from routes.email_helpers import (
     _pre_retrieve_context,
     _attach_compose_uploads, _cleanup_compose_uploads, _q,
     SCHEDULED_DB, _EMAIL_REPLY_SYS_PROMPT_BASE, _email_cache_owner_clause,
+    _generate_scheduled_email_summary, _email_summary_failure_log_detail,
 )
 
 logger = logging.getLogger(__name__)
@@ -100,6 +101,272 @@ def _owner_for_email_account(account_id: str | None) -> str:
         return ""
 
 
+def _email_date_only(value: str | None):
+    value = (value or "").strip()
+    if not value:
+        return None
+    try:
+        return datetime.strptime(value[:10], "%Y-%m-%d").date()
+    except Exception:
+        return None
+
+
+_AUTO_REPLY_KEYS = {
+    "email_auto_reply",
+    "email_auto_reply_start",
+    "email_auto_reply_end",
+    "email_auto_reply_subject",
+    "email_auto_reply_message",
+    "email_auto_reply_cooldown",
+    "email_auto_reply_scope",
+    "email_auto_reply_account_id",
+    "email_auto_reply_exclude_automated",
+    "email_auto_reply_pause_notifications",
+    "email_auto_reply_enabled_at",
+}
+
+
+def _effective_settings_for_email_account(settings: dict, account_id: str | None) -> dict:
+    """Overlay per-account auto-reply settings onto global settings.
+
+    Other automation toggles remain global. This lets each mailbox have its own
+    away reply while preserving existing installs that only have global keys.
+    """
+    effective = dict(settings or {})
+    key = str(account_id or "").strip()
+    by_account = effective.get("email_auto_reply_by_account") or {}
+    account_cfg = by_account.get(key) if key and isinstance(by_account, dict) else None
+    if isinstance(account_cfg, dict):
+        for k in _AUTO_REPLY_KEYS:
+            if k in account_cfg:
+                effective[k] = account_cfg[k]
+    return effective
+
+
+def _away_reply_active(settings: dict, account_id: str | None) -> bool:
+    if not settings.get("email_auto_reply", False):
+        return False
+
+    scope = str(settings.get("email_auto_reply_scope") or "all").strip().lower()
+    if scope == "account":
+        selected = str(settings.get("email_auto_reply_account_id") or "").strip()
+        if selected and selected != str(account_id or ""):
+            return False
+
+    today = datetime.utcnow().date()
+    start = _email_date_only(settings.get("email_auto_reply_start"))
+    end = _email_date_only(settings.get("email_auto_reply_end"))
+    if start and today < start:
+        return False
+    if end and today > end:
+        return False
+    return True
+
+
+def _message_after_away_enabled(settings: dict, msg) -> bool:
+    enabled_at = (settings.get("email_auto_reply_enabled_at") or "").strip()
+    if not enabled_at:
+        # Existing installs may already have the toggle on before this feature
+        # existed. Do not back-reply old mail until the user saves/toggles it.
+        return False
+    try:
+        enabled_dt = datetime.fromisoformat(enabled_at.replace("Z", "+00:00"))
+    except Exception:
+        return False
+    try:
+        msg_dt = email.utils.parsedate_to_datetime(msg.get("Date", ""))
+    except Exception:
+        return False
+    try:
+        if enabled_dt.tzinfo and not msg_dt.tzinfo:
+            msg_dt = msg_dt.replace(tzinfo=enabled_dt.tzinfo)
+        elif msg_dt.tzinfo and not enabled_dt.tzinfo:
+            enabled_dt = enabled_dt.replace(tzinfo=msg_dt.tzinfo)
+    except Exception:
+        pass
+    return msg_dt >= enabled_dt
+
+
+def _away_reply_period_key(settings: dict) -> str:
+    start = (settings.get("email_auto_reply_start") or "").strip()
+    end = (settings.get("email_auto_reply_end") or "").strip()
+    return f"{start or '*'}..{end or '*'}"
+
+
+def _away_reply_cooldown_seconds(settings: dict) -> int | None:
+    raw = str(settings.get("email_auto_reply_cooldown") or "period").strip().lower()
+    if raw == "1d":
+        return 24 * 60 * 60
+    if raw == "3d":
+        return 3 * 24 * 60 * 60
+    if raw == "7d":
+        return 7 * 24 * 60 * 60
+    return None
+
+
+def _ensure_away_reply_table():
+    import sqlite3 as _sql3
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS email_away_replies (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                owner TEXT DEFAULT '',
+                account_id TEXT DEFAULT '',
+                message_id TEXT DEFAULT '',
+                sender_addr TEXT DEFAULT '',
+                subject TEXT DEFAULT '',
+                period_key TEXT DEFAULT '',
+                sent_at TEXT DEFAULT ''
+            )
+        """)
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_email_away_msg ON email_away_replies(owner, account_id, message_id)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_email_away_sender ON email_away_replies(owner, account_id, sender_addr, sent_at)")
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _sender_is_automated(msg, sender_addr: str) -> bool:
+    auto_submitted = (msg.get("Auto-Submitted") or "").strip().lower()
+    if auto_submitted and auto_submitted != "no":
+        return True
+    precedence = (msg.get("Precedence") or "").strip().lower()
+    if precedence in {"bulk", "junk", "list"}:
+        return True
+    if msg.get("List-Id") or msg.get("List-Unsubscribe"):
+        return True
+    local = (sender_addr or "").split("@", 1)[0].lower()
+    return local in {
+        "no-reply", "noreply", "do-not-reply", "donotreply",
+        "notification", "notifications", "automated", "mailer-daemon",
+        "postmaster",
+    }
+
+
+def _away_reply_already_sent(settings: dict, account_owner: str, account_id: str | None,
+                             message_id: str, sender_addr: str) -> bool:
+    import sqlite3 as _sql3
+    _ensure_away_reply_table()
+    owner = account_owner or ""
+    aid = account_id or ""
+    sender = (sender_addr or "").strip().lower()
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        row = conn.execute(
+            "SELECT 1 FROM email_away_replies WHERE owner=? AND account_id=? AND message_id=? LIMIT 1",
+            (owner, aid, message_id),
+        ).fetchone()
+        if row:
+            return True
+
+        cooldown = _away_reply_cooldown_seconds(settings)
+        if cooldown is None:
+            period_key = _away_reply_period_key(settings)
+            row = conn.execute(
+                "SELECT 1 FROM email_away_replies WHERE owner=? AND account_id=? AND sender_addr=? AND period_key=? LIMIT 1",
+                (owner, aid, sender, period_key),
+            ).fetchone()
+            return bool(row)
+
+        since = datetime.utcnow().timestamp() - cooldown
+        rows = conn.execute(
+            "SELECT sent_at FROM email_away_replies WHERE owner=? AND account_id=? AND sender_addr=? ORDER BY sent_at DESC LIMIT 5",
+            (owner, aid, sender),
+        ).fetchall()
+        for (sent_at,) in rows:
+            try:
+                if datetime.fromisoformat(sent_at).timestamp() >= since:
+                    return True
+            except Exception:
+                continue
+        return False
+    finally:
+        conn.close()
+
+
+def _record_away_reply(settings: dict, account_owner: str, account_id: str | None,
+                       message_id: str, sender_addr: str, subject: str):
+    import sqlite3 as _sql3
+    _ensure_away_reply_table()
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        conn.execute(
+            """
+            INSERT INTO email_away_replies
+            (owner, account_id, message_id, sender_addr, subject, period_key, sent_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                account_owner or "",
+                account_id or "",
+                message_id,
+                (sender_addr or "").strip().lower(),
+                subject or "",
+                _away_reply_period_key(settings),
+                datetime.utcnow().isoformat(),
+            ),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _send_away_reply(settings: dict, account_owner: str, account_id: str | None,
+                     msg, message_id: str, sender: str, subject: str):
+    sender_name, sender_addr = email.utils.parseaddr(sender or "")
+    sender_addr = (sender_addr or "").strip()
+    if not sender_addr:
+        return False, "missing sender"
+
+    cfg = _get_email_config(account_id, owner=account_owner)
+    from_addr = (cfg.get("from_address") or cfg.get("smtp_user") or "").strip()
+    if not from_addr:
+        return False, "missing from address"
+    if sender_addr.lower() == from_addr.lower():
+        return False, "self mail"
+    if settings.get("email_auto_reply_exclude_automated", True) and _sender_is_automated(msg, sender_addr):
+        return False, "automated sender"
+    if _away_reply_already_sent(settings, account_owner, account_id, message_id, sender_addr):
+        return False, "already sent"
+
+    body = (settings.get("email_auto_reply_message") or "").strip()
+    if not body:
+        body = "Thanks for your email. I'm away and may be slower to reply."
+
+    subject_template = (settings.get("email_auto_reply_subject") or "(Away) {subject}").strip()
+    if subject_template:
+        original_subject = subject or ""
+        reply_subject = (
+            subject_template
+            .replace("{subject}", original_subject)
+            .replace("{original_subject}", original_subject)
+        ).strip() or "Re:"
+    else:
+        reply_subject = subject or ""
+        if not reply_subject.lower().lstrip().startswith("re:"):
+            reply_subject = f"Re: {reply_subject}" if reply_subject else "Re:"
+
+    outer = MIMEMultipart("alternative")
+    display = cfg.get("display_name") or ""
+    outer["From"] = email.utils.formataddr((display, from_addr)) if display else from_addr
+    outer["To"] = email.utils.formataddr((sender_name, sender_addr)) if sender_name else sender_addr
+    outer["Subject"] = reply_subject
+    outer["Date"] = email.utils.formatdate(localtime=False)
+    outer["Message-ID"] = email.utils.make_msgid()
+    outer["Auto-Submitted"] = "auto-replied"
+    outer["X-Auto-Response-Suppress"] = "All"
+    if message_id:
+        outer["In-Reply-To"] = message_id
+        refs = (msg.get("References") or "").strip()
+        outer["References"] = f"{refs} {message_id}".strip()
+    outer.attach(MIMEText(body, "plain", "utf-8"))
+
+    _send_smtp_message(cfg, from_addr, [sender_addr], outer.as_string())
+    _record_away_reply(settings, account_owner, account_id, message_id, sender_addr, subject)
+    return True, sender_addr
+
+
 # ── Routes ──
 
 async def _emit_progress(progress_cb, message: str):
@@ -125,9 +392,10 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
     settings = _load_settings()
     prev = {k: settings.get(k, False) for k in
             ("email_auto_summarize", "email_auto_reply", "email_auto_tag",
-             "email_auto_spam", "email_auto_calendar")}
+             "email_auto_spam", "email_auto_calendar", "_email_auto_reply_draft_only")}
     settings["email_auto_summarize"] = bool(do_summary)
     settings["email_auto_reply"] = bool(do_reply)
+    settings["_email_auto_reply_draft_only"] = bool(do_reply)
     settings["email_auto_tag"] = bool(do_tag)
     settings["email_auto_spam"] = bool(do_spam)
     settings["email_auto_calendar"] = bool(do_calendar)
@@ -142,7 +410,10 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
     finally:
         s2 = _load_settings()
         for k, v in prev.items():
-            s2[k] = v
+            if v is None and k.startswith("_"):
+                s2.pop(k, None)
+            else:
+                s2[k] = v
         _save_settings(s2)
 
 
@@ -176,7 +447,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
         return [], reconnect()
 
 
-async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None, away_only: bool = False) -> str:
     """Single pass of the auto-summarize/reply scan.
 
     When account_id is None, iterates over every enabled account in
@@ -208,6 +479,7 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
                 account_id=(ids[0] if ids else None),
                 max_process=max_process,
                 progress_cb=progress_cb,
+                away_only=away_only,
             )
         outs = []
         for idx, aid in enumerate(ids, start=1):
@@ -218,6 +490,7 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
                     account_id=aid,
                     max_process=max_process,
                     progress_cb=progress_cb,
+                    away_only=away_only,
                 )
                 outs.append(f"[{names.get(aid, aid[:8])}] {result}")
             except Exception as e:
@@ -229,23 +502,32 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
         account_id=account_id,
         max_process=max_process,
         progress_cb=progress_cb,
+        away_only=away_only,
     )
 
 
-async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
+async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None, away_only: bool = False) -> str:
     """Single pass of the auto-summarize/reply scan for ONE account.
     Reads current settings flags."""
     import asyncio
     import sqlite3 as _sql3
     from src.llm_core import _uses_max_completion_tokens
 
-    settings = _load_settings()
+    settings = _effective_settings_for_email_account(_load_settings(), account_id)
     auto_sum = settings.get("email_auto_summarize", False)
     auto_reply = settings.get("email_auto_reply", False)
+    auto_reply_draft = bool(auto_reply and settings.get("_email_auto_reply_draft_only", False))
+    auto_reply_away = bool(auto_reply and not auto_reply_draft and _away_reply_active(settings, account_id))
     auto_tag = settings.get("email_auto_tag", False)
     auto_spam = settings.get("email_auto_spam", False)
     auto_cal = settings.get("email_auto_calendar", False)
-    if not auto_sum and not auto_reply and not auto_tag and not auto_spam and not auto_cal:
+    if away_only:
+        auto_sum = False
+        auto_reply_draft = False
+        auto_tag = False
+        auto_spam = False
+        auto_cal = False
+    if not auto_sum and not auto_reply_draft and not auto_reply_away and not auto_tag and not auto_spam and not auto_cal:
         return "Nothing to do"
 
     # Owner of the account being processed. All calendar + mailbox reads/writes
@@ -304,11 +586,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
 
         _c = _sql3.connect(SCHEDULED_DB)
         _cache_owner_clause, _cache_owner_params = _email_cache_owner_clause(account_owner)
-        _sum_existing = {r[0] for r in _c.execute(
+        _sum_existing = set() if away_only else {r[0] for r in _c.execute(
             f"SELECT message_id FROM email_summaries WHERE {_cache_owner_clause}",
             _cache_owner_params,
         ).fetchall()}
-        _reply_existing = {r[0] for r in _c.execute(
+        _reply_existing = set() if away_only else {r[0] for r in _c.execute(
             f"SELECT message_id FROM email_ai_replies WHERE {_cache_owner_clause}",
             _cache_owner_params,
         ).fetchall()}
@@ -325,7 +607,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 ).fetchall()}
         else:
             _tag_existing = set()
-        _cal_existing = {r[0] for r in _c.execute(
+        _cal_existing = set() if away_only else {r[0] for r in _c.execute(
             f"SELECT message_id FROM email_calendar_extractions WHERE {_cache_owner_clause}",
             _cache_owner_params,
         ).fetchall()} if auto_cal else set()
@@ -351,27 +633,40 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         if auto_spam and not spam_folder:
             logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
 
-        task_candidates = resolve_task_candidates(owner=account_owner)
-        if not task_candidates:
-            return "No model configured"
-        url, model, headers = task_candidates[0]
+        needs_llm = bool(auto_sum or auto_reply_draft or auto_tag or auto_spam or auto_cal)
+        if needs_llm:
+            task_candidates = resolve_task_candidates(owner=account_owner)
+            if not task_candidates:
+                return "No model configured"
+            url, model, headers = task_candidates[0]
+        else:
+            url, model, headers = None, "", None
 
-        writing_style = settings.get("email_writing_style", "")
+        by_account_styles = settings.get("email_writing_styles_by_account") or {}
+        writing_style = ""
+        if account_id and isinstance(by_account_styles, dict):
+            writing_style = str(by_account_styles.get(str(account_id)) or "")
+        if not writing_style:
+            writing_style = settings.get("email_writing_style", "")
         processed = 0
         already_cached = 0
         too_short = 0
         no_msgid = 0
         examined = 0
         _summaries_created = 0
+        _summary_failed = 0
         _events_created = 0
         _replies_drafted = 0
         _reply_failed = 0
+        _away_replies_sent = 0
+        _away_replies_skipped = 0
+        _away_replies_failed = 0
         _detail_lines = []
         _current_folder = "INBOX"
         # Calendar extraction is sequential and each row can involve a model
         # call plus a calendar write. Keep the scheduled calendar-only pass
         # below the 5-minute action budget instead of timing out mid-run.
-        _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
+        _default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply_draft and not auto_reply_away and not auto_tag and not auto_spam) else 5
         try:
             _max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
         except Exception:
@@ -402,10 +697,6 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     seed = f"{_folder}|{uid_str}|{msg.get('From','')}|{msg.get('Date','')}|{msg.get('Subject','')}"
                     message_id = f"<synth-{_hl.sha256(seed.encode()).hexdigest()[:16]}@local>"
                     no_msgid += 1
-                need_sum = auto_sum and message_id not in _sum_existing
-                need_reply = auto_reply and message_id not in _reply_existing
-                need_class = (auto_tag or auto_spam) and message_id not in _tag_existing
-                need_cal = bool(settings.get("email_auto_calendar", False)) and message_id not in _cal_existing
                 # Only check urgency on INBOX (received mail), not Sent
                 # Skip messages that are themselves urgency alerts, or that
                 # we sent to ourselves — otherwise the alert loop re-flags
@@ -422,17 +713,45 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 except Exception:
                     _from_addr_only = ""
                 _is_self_mail = bool(_self_self_addr) and _from_addr_only.lower() == _self_self_addr
+                need_sum = auto_sum and message_id not in _sum_existing
+                need_reply = auto_reply_draft and message_id not in _reply_existing
+                need_away_reply = bool(
+                    auto_reply_away
+                    and _folder.upper() == "INBOX"
+                    and not _is_self_mail
+                    and (away_only or _message_after_away_enabled(settings, msg))
+                    and not _away_reply_already_sent(settings, account_owner, account_id, message_id, _from_addr_only)
+                )
+                need_class = (auto_tag or auto_spam) and message_id not in _tag_existing
+                need_cal = bool(settings.get("email_auto_calendar", False)) and message_id not in _cal_existing
                 need_urgent = (auto_urgent and message_id not in _urgent_existing
                                and not _folder.lower().startswith("sent")
                                and "sent" not in _folder.lower()
                                and not _is_alert_echo
                                and not _is_self_mail)
-                if not need_sum and not need_reply and not need_class and not need_cal and not need_urgent:
+                if not need_sum and not need_reply and not need_away_reply and not need_class and not need_cal and not need_urgent:
                     already_cached += 1
                     await _emit_progress(progress_cb, f"Checked {examined}/{len(uid_list)} · {already_cached} already cached")
                     continue
                 subject = _decode_header(msg.get("Subject", ""))
                 sender = _decode_header(msg.get("From", ""))
+                if need_away_reply:
+                    try:
+                        sent_away, away_detail = _send_away_reply(
+                            settings, account_owner, account_id, msg, message_id, sender, subject
+                        )
+                        if sent_away:
+                            _away_replies_sent += 1
+                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                            _detail_lines.append(f"away reply · {_folder}#{_uid_text} · {subject or '(no subject)'} — {away_detail}")
+                        else:
+                            _away_replies_skipped += 1
+                            logger.info(f"Away reply skipped for uid={uid}: {away_detail}")
+                    except Exception as e:
+                        _away_replies_failed += 1
+                        _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                        _detail_lines.append(f"away reply failed · {_folder}#{_uid_text} · {subject or '(no subject)'}")
+                        logger.warning(f"Away reply {uid} failed: {e}")
                 body = _extract_text(msg)
                 # Pull text out of any PDFs / text attachments and append to
                 # the body so summaries / replies can actually reason about
@@ -454,7 +773,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                 elif need_reply:
                     if not body:
                         body = subject
-                elif (not body or len(body) < 100) and not att_text:
+                elif not need_away_reply and (not body or len(body) < 100) and not att_text:
                     too_short += 1
                     continue
                 # Augmented body sent to the LLM: original body + attachment text.
@@ -468,16 +787,17 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
 
                 if need_sum:
                     try:
-                        summary = await task_llm_call_async(
-                            messages=[
-                                {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
-                                {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
-                            ],
-                            fallback_url=url, fallback_model=model, fallback_headers=headers,
+                        summary = await _generate_scheduled_email_summary(
+                            url=url,
+                            model=model,
+                            sender=sender,
+                            subject=subject,
+                            body_for_llm=body_for_llm,
+                            headers=req_headers,
                             owner=account_owner or None,
-                            temperature=0.3, max_tokens=16384, timeout=240,
+                            max_tokens=16384,
+                            timeout=240,
                         )
-                        summary = _extract_reply((summary or "").strip())
                         if summary:
                             _c = _sql3.connect(SCHEDULED_DB)
                             _c.execute("""
@@ -491,10 +811,19 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             _summaries_created += 1
                             _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                             _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
+                        else:
+                            _summary_failed += 1
+                            _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
+                            _detail_lines.append(f"summary empty · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
                     except Exception as e:
+                        _summary_failed += 1
                         _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
                         _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
-                        logger.warning(f"Auto-summary {uid} failed: {e}")
+                        logger.warning(
+                            "Auto-summary uid=%s failed %s",
+                            _uid_text,
+                            _email_summary_failure_log_detail(e),
+                        )
 
                 if need_reply:
                     await _emit_progress(progress_cb, f"Drafting reply {processed + 1}/{_max_process} · checked {examined}/{len(uid_list)}")
@@ -993,7 +1322,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
         # Build a clear status message
         ops = []
         if auto_sum: ops.append("summary")
-        if auto_reply: ops.append("reply")
+        if auto_reply_draft: ops.append("reply")
+        if auto_reply_away: ops.append("away")
         if auto_tag: ops.append("tag")
         if auto_spam: ops.append("spam")
         ops_label = "/".join(ops) or "none"
@@ -1002,10 +1332,16 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
             parts.append(f"processed {processed} new")
         if auto_sum:
             parts.append(f"summarized {_summaries_created}")
-        if auto_reply:
+            if _summary_failed:
+                parts.append(f"{_summary_failed} summary failed")
+        if auto_reply_draft:
             parts.append(f"drafted {_replies_drafted} repl" + ("y" if _replies_drafted == 1 else "ies"))
             if _reply_failed:
                 parts.append(f"{_reply_failed} reply failed")
+        if auto_reply_away:
+            parts.append(f"sent {_away_replies_sent} away repl" + ("y" if _away_replies_sent == 1 else "ies"))
+            if _away_replies_failed:
+                parts.append(f"{_away_replies_failed} away failed")
         if already_cached:
             parts.append(f"{already_cached} already cached")
         if too_short:
@@ -1032,12 +1368,13 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
 
 
 async def _auto_summarize_poller():
-    """Background loop kept for backward compatibility — calls _auto_summarize_pass every 60s.
+    """Background loop kept for backward compatibility — calls _auto_summarize_pass periodically.
     Newer setups should use scheduled tasks instead (summarize_emails, draft_email_replies)."""
     import asyncio as _asyncio
     while True:
         try:
-            await _asyncio.sleep(1800)
+            settings = _load_settings()
+            await _asyncio.sleep(60 if settings.get("email_auto_reply", False) else 1800)
             await _auto_summarize_pass()
         except Exception as e:
             logger.error(f"Auto-summarize poller crash: {e}")
@@ -1068,6 +1405,28 @@ def _scheduled_poll_once() -> dict:
         for r in rows:
             sid = r[0]
             try:
+                # Atomically claim this row before doing any work. Two
+                # pollers can race here (the in-process asyncio task and an
+                # externally cron-driven `odysseus-mail poll-scheduled`, or
+                # an admin running the CLI manually alongside the in-process
+                # one despite the ODYSSEUS_INPROCESS_POLLERS=0 guidance) -
+                # both can SELECT the same 'pending' row before either has
+                # updated its status. The UPDATE...WHERE status='pending' is
+                # the atomicity boundary: only the poller whose UPDATE
+                # actually changes a row (rowcount == 1) proceeds to send;
+                # a loser sees rowcount == 0 and skips it instead of sending
+                # a duplicate.
+                claim_conn = sqlite3.connect(SCHEDULED_DB)
+                claim_cur = claim_conn.execute(
+                    "UPDATE scheduled_emails SET status='sending' WHERE id=? AND status='pending'",
+                    (sid,),
+                )
+                claim_conn.commit()
+                claimed = claim_cur.rowcount == 1
+                claim_conn.close()
+                if not claimed:
+                    continue
+
                 attachments = json.loads(r[8] or "[]")
                 row_account_id = r[9] if len(r) > 9 else None
                 odysseus_kind = r[10] if len(r) > 10 else "scheduled"

routes/gallery/gallery_routes.py

@@ -1,7 +1,9 @@
 """Gallery routes — browsable library for photos and AI-generated images."""
 
 import os
+import base64
 import hashlib
+import io
 import logging
 import re
 import uuid
@@ -27,6 +29,181 @@ from routes.gallery.gallery_helpers import (
 
 logger = logging.getLogger(__name__)
 
+_SAM_STATE: Dict[str, Any] = {}
+_GROUNDING_STATE: Dict[str, Any] = {}
+
+
+def _b64_to_pil_image(image_b64: str, *, mode: str = "RGBA"):
+    if not image_b64:
+        raise HTTPException(400, "Missing image")
+    if "," in image_b64 and image_b64.split(",", 1)[0].startswith("data:"):
+        image_b64 = image_b64.split(",", 1)[1]
+    try:
+        from PIL import Image
+
+        raw = base64.b64decode(image_b64)
+        return Image.open(io.BytesIO(raw)).convert(mode)
+    except HTTPException:
+        raise
+    except Exception as exc:
+        raise HTTPException(400, "Invalid image") from exc
+
+
+def _pil_image_to_b64(img, *, fmt: str = "PNG") -> str:
+    buf = io.BytesIO()
+    img.save(buf, format=fmt)
+    return base64.b64encode(buf.getvalue()).decode("ascii")
+
+
+def _load_sam_backend():
+    model_id = os.getenv("ODYSSEUS_SAM_MODEL", "facebook/sam-vit-base")
+    cached = _SAM_STATE.get(model_id)
+    if cached:
+        return cached
+    try:
+        import torch
+        from transformers import SamModel, SamProcessor
+    except Exception as exc:
+        raise HTTPException(
+            501,
+            "SAM mask tools are not installed. Install Cookbook Dependencies -> SAM mask tools.",
+        ) from exc
+
+    device = "cpu"
+    try:
+        if torch.cuda.is_available():
+            device = "cuda"
+        elif getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
+            device = "mps"
+    except Exception:
+        device = "cpu"
+
+    try:
+        processor = SamProcessor.from_pretrained(model_id)
+        model = SamModel.from_pretrained(model_id)
+        model.to(device)
+        model.eval()
+    except Exception as exc:
+        raise HTTPException(500, f"Failed to load SAM model {model_id}: {exc}") from exc
+
+    cached = {"torch": torch, "processor": processor, "model": model, "device": device, "model_id": model_id}
+    _SAM_STATE[model_id] = cached
+    return cached
+
+
+def _load_grounding_backend():
+    model_id = os.getenv("ODYSSEUS_GROUNDING_MODEL", "google/owlvit-base-patch32")
+    cached = _GROUNDING_STATE.get(model_id)
+    if cached:
+        return cached
+    try:
+        import torch
+        from transformers import OwlViTForObjectDetection, OwlViTProcessor
+    except Exception as exc:
+        raise HTTPException(
+            501,
+            "Object mask tools are not installed. Install Cookbook Dependencies -> SAM mask tools.",
+        ) from exc
+
+    device = "cpu"
+    try:
+        if torch.cuda.is_available():
+            device = "cuda"
+        elif getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
+            device = "mps"
+    except Exception:
+        device = "cpu"
+
+    try:
+        processor = OwlViTProcessor.from_pretrained(model_id)
+        model = OwlViTForObjectDetection.from_pretrained(model_id)
+        model.to(device)
+        model.eval()
+    except Exception as exc:
+        raise HTTPException(500, f"Failed to load object mask model {model_id}: {exc}") from exc
+
+    cached = {"torch": torch, "processor": processor, "model": model, "device": device, "model_id": model_id}
+    _GROUNDING_STATE[model_id] = cached
+    return cached
+
+
+def _model_input_to_device(value, device: str, torch):
+    if not hasattr(value, "to"):
+        return value
+    if (
+        device == "mps"
+        and hasattr(torch, "float64")
+        and getattr(value, "dtype", None) == torch.float64
+    ):
+        return value.to(device=device, dtype=torch.float32)
+    return value.to(device)
+
+
+def _model_inputs_to_device(inputs, device: str, torch) -> Dict[str, Any]:
+    return {
+        key: _model_input_to_device(value, device, torch)
+        for key, value in inputs.items()
+    }
+
+
+def _ground_text_to_box(image, text: str, *, threshold: float = 0.05):
+    query = (text or "").strip()
+    if not query:
+        raise HTTPException(400, "Missing object text")
+    backend = _load_grounding_backend()
+    torch = backend["torch"]
+    processor = backend["processor"]
+    model = backend["model"]
+    device = backend["device"]
+
+    labels = [query]
+    if not query.lower().startswith(("a ", "an ", "the ")):
+        labels.append(f"a photo of {query}")
+    try:
+        inputs = processor(text=[labels], images=image, return_tensors="pt")
+        model_inputs = _model_inputs_to_device(inputs, device, torch)
+        with torch.no_grad():
+            outputs = model(**model_inputs)
+        target_sizes = torch.tensor([[image.height, image.width]])
+        if hasattr(processor, "post_process_object_detection"):
+            results = processor.post_process_object_detection(
+                outputs=outputs,
+                target_sizes=target_sizes,
+                threshold=float(threshold),
+            )
+        elif hasattr(processor, "post_process_grounded_object_detection"):
+            results = processor.post_process_grounded_object_detection(
+                outputs=outputs,
+                target_sizes=target_sizes,
+                threshold=float(threshold),
+                text_labels=[labels],
+            )
+        else:
+            raise HTTPException(500, "Installed Transformers does not expose OWL-ViT object detection post-processing")
+        boxes = results[0].get("boxes")
+        scores = results[0].get("scores")
+        labels_idx = results[0].get("labels")
+        text_labels = results[0].get("text_labels") or results[0].get("labels_text")
+        if boxes is None or scores is None or len(boxes) == 0:
+            raise HTTPException(404, f"No visible object matched '{query}'")
+        idx = int(torch.argmax(scores).item())
+        box = [float(v) for v in boxes[idx].detach().cpu().tolist()]
+        label_idx = int(labels_idx[idx].detach().cpu().item()) if labels_idx is not None and len(labels_idx) else 0
+        label = labels[min(label_idx, len(labels) - 1)]
+        if text_labels and len(text_labels) > idx:
+            label = str(text_labels[idx])
+        return {
+            "box": box,
+            "score": float(scores[idx].detach().cpu().item()),
+            "label": label,
+            "model": backend["model_id"],
+        }
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.exception("ground_text_to_box failed")
+        raise HTTPException(500, f"Object mask failed: {exc}") from exc
+
 
 def _current_user_is_admin(request: Request, user: str | None) -> bool:
     if not user:
@@ -1240,20 +1417,89 @@ def setup_gallery_routes() -> APIRouter:
             except httpx.TimeoutException:
                 raise HTTPException(504, "OpenAI inpaint timed out (120s)")
 
-        # Self-hosted diffusion server path
+        # Self-hosted diffusion server path. Newer Odysseus image
+        # wrappers expose the OpenAI-compatible /v1/images/edits
+        # multipart route even when they are local/self-hosted. Older
+        # diffusion_server.py exposes /v1/images/inpaint as JSON. Try the
+        # OpenAI-compatible local route first, then fall back.
         try:
             # Forward chosen_model so the diffusion server can route if it ever
             # supports multiple models per process. Harmless if ignored.
             if chosen_model:
                 body["model"] = chosen_model
-            async with httpx.AsyncClient(timeout=120) as client:
+            async with httpx.AsyncClient(timeout=240) as client:
+                try:
+                    import base64, io
+                    from PIL import Image
+
+                    img_bytes = base64.b64decode(body["image"])
+                    mask_bytes = base64.b64decode(body["mask"])
+                    # Normalize both inputs to PNG bytes. Local MLX and
+                    # Diffusers wrappers expect white mask pixels to mean
+                    # "edit this region", which matches the editor's mask.
+                    source_png = Image.open(io.BytesIO(img_bytes)).convert("RGBA")
+                    mask_png = Image.open(io.BytesIO(mask_bytes)).convert("L")
+                    src_buf = io.BytesIO()
+                    source_png.save(src_buf, format="PNG")
+                    mask_buf = io.BytesIO()
+                    mask_png.save(mask_buf, format="PNG")
+                    files = {
+                        "image": ("source.png", src_buf.getvalue(), "image/png"),
+                        "mask": ("mask.png", mask_buf.getvalue(), "image/png"),
+                    }
+                    data = {
+                        "model": chosen_model or body.get("model") or "",
+                        "prompt": body.get("prompt", ""),
+                        "size": f"{int(body.get('width') or source_png.width)}x{int(body.get('height') or source_png.height)}",
+                        "n": "1",
+                    }
+                    r = await client.post(_join_checked_gallery_endpoint(base, "/images/edits"), data=data, files=files)
+                    if r.status_code == 200:
+                        result = r.json()
+                        if isinstance(result, dict) and result.get("data"):
+                            item = result["data"][0]
+                            if item.get("b64_json"):
+                                return {"image": item["b64_json"]}
+                            if item.get("url"):
+                                raw_b64 = await _fetch_result_image_b64(item["url"])
+                                if raw_b64:
+                                    return {"image": raw_b64}
+                        if isinstance(result, dict) and result.get("image"):
+                            return {"image": result["image"]}
+                        raise HTTPException(502, "Image edit endpoint returned no image")
+                    if r.status_code not in (404, 405):
+                        logger.warning("inpaint_proxy self-hosted edits: status %s", r.status_code)
+                        detail = "Image edit request failed"
+                        try:
+                            err = r.json()
+                            detail = err.get("detail") or err.get("error") or detail
+                        except Exception:
+                            pass
+                        # A plain SD/SDXL checkpoint often exposes
+                        # generation only at /images/edits.
+                        # That does not mean the endpoint cannot inpaint:
+                        # Odysseus diffusion_server.py has a dedicated
+                        # /images/inpaint route that can derive/fallback to
+                        # inpaint, img2img crop+composite, or txt2img
+                        # crop+composite. Fall through to that route instead
+                        # of surfacing "does not support image edits".
+                        if r.status_code == 400 and "does not support image edits" in str(detail).lower():
+                            logger.info("inpaint_proxy self-hosted edits unsupported; falling back to /images/inpaint")
+                        else:
+                            raise HTTPException(r.status_code, detail)
+                except HTTPException:
+                    raise
+                except Exception:
+                    logger.exception("inpaint_proxy: failed to prepare self-hosted edit request")
+                    raise HTTPException(400, "Failed to prepare inpaint request")
+
                 r = await client.post(_join_checked_gallery_endpoint(base, "/images/inpaint"), json=body)
                 if r.status_code != 200:
                     logger.error("inpaint_proxy diffusion: status %s", r.status_code)
                     raise HTTPException(r.status_code, "Inpaint request failed")
                 return r.json()
         except httpx.TimeoutException:
-            raise HTTPException(504, "Inpaint request timed out (120s)")
+            raise HTTPException(504, "Inpaint request timed out (240s)")
         except HTTPException:
             raise
         except Exception:
@@ -1588,6 +1834,132 @@ def setup_gallery_routes() -> APIRouter:
             return {"error": "AI upscale failed"}
 
     # ---- POST /api/image/remove-bg ----
+    @router.post("/api/image/mask")
+    async def smart_mask(request: Request):
+        """Create a neutral segmentation mask from user-provided points or a box.
+
+        This endpoint intentionally does not inspect edit prompts. It only
+        turns explicit visual selection hints into a binary mask that the
+        editor can reuse for wand/layer-mask/inpaint workflows.
+        """
+        require_privilege(request, "can_generate_images")
+        body = await request.json()
+        image = _b64_to_pil_image(body.get("image") or "", mode="RGB")
+        points = body.get("points") or []
+        box = body.get("box")
+        text = (body.get("text") or body.get("query") or "").strip()
+        grounded = None
+
+        if not points and not box and text:
+            grounded = _ground_text_to_box(image, text)
+            box = grounded["box"]
+
+        if not points and not box:
+            raise HTTPException(400, "Provide at least one point, box, or object text")
+
+        backend = _load_sam_backend()
+        torch = backend["torch"]
+        processor = backend["processor"]
+        model = backend["model"]
+        device = backend["device"]
+
+        kwargs: Dict[str, Any] = {"return_tensors": "pt"}
+        input_points = []
+        if points:
+            input_labels = []
+            for p in points:
+                try:
+                    input_points.append([float(p["x"]), float(p["y"])])
+                    input_labels.append(int(p.get("label", 1)))
+                except Exception as exc:
+                    raise HTTPException(400, "Invalid point format") from exc
+            kwargs["input_points"] = [input_points]
+            kwargs["input_labels"] = [input_labels]
+        if box:
+            if not isinstance(box, list) or len(box) != 4:
+                raise HTTPException(400, "Box must be [x1, y1, x2, y2]")
+            try:
+                kwargs["input_boxes"] = [[[float(v) for v in box]]]
+            except Exception as exc:
+                raise HTTPException(400, "Invalid box format") from exc
+
+        try:
+            inputs = processor(image, **kwargs)
+            model_inputs = _model_inputs_to_device(inputs, device, torch)
+            with torch.no_grad():
+                outputs = model(**model_inputs)
+            masks = processor.image_processor.post_process_masks(
+                outputs.pred_masks.detach().cpu(),
+                inputs["original_sizes"].detach().cpu(),
+                inputs["reshaped_input_sizes"].detach().cpu(),
+            )
+            mask_tensor = masks[0]
+            while getattr(mask_tensor, "ndim", 0) > 3:
+                mask_tensor = mask_tensor[0]
+            if getattr(mask_tensor, "ndim", 0) == 3:
+                scores = outputs.iou_scores.detach().cpu()[0]
+                while getattr(scores, "ndim", 0) > 1:
+                    scores = scores[0]
+                # SAM commonly returns multiple candidates for a click. The
+                # highest-IoU candidate can be the entire image, which is
+                # useless as an editor selection. Prefer a candidate that
+                # contains the clicked point while keeping area reasonable.
+                point_xy = None
+                if input_points:
+                    try:
+                        point_xy = (
+                            int(round(float(input_points[0][0]))),
+                            int(round(float(input_points[0][1]))),
+                        )
+                    except Exception:
+                        point_xy = None
+                best_idx = 0
+                best_rank = None
+                total_px = max(1, int(mask_tensor.shape[-1]) * int(mask_tensor.shape[-2]))
+                for i in range(int(mask_tensor.shape[0])):
+                    candidate = mask_tensor[i]
+                    area_ratio = float(candidate.sum().item()) / float(total_px)
+                    if area_ratio >= 0.985:
+                        continue
+                    contains_click = True
+                    if point_xy:
+                        px = max(0, min(int(candidate.shape[-1]) - 1, point_xy[0]))
+                        py = max(0, min(int(candidate.shape[-2]) - 1, point_xy[1]))
+                        contains_click = bool(candidate[py, px].item())
+                    if not contains_click:
+                        continue
+                    score = float(scores[min(i, len(scores) - 1)].item()) if len(scores) else 0.0
+                    # Strongly penalize broad masks; a click-selection should
+                    # usually be local unless the user gives a box.
+                    rank = score - (area_ratio * 0.35)
+                    if best_rank is None or rank > best_rank:
+                        best_rank = rank
+                        best_idx = i
+                if best_rank is None and len(scores):
+                    best_idx = int(torch.argmax(scores).item())
+                mask_tensor = mask_tensor[min(best_idx, mask_tensor.shape[0] - 1)]
+            mask_array = (mask_tensor.numpy() > 0).astype("uint8") * 255
+            from PIL import Image
+
+            mask_img = Image.fromarray(mask_array, mode="L")
+            if mask_img.size != image.size:
+                mask_img = mask_img.resize(image.size, Image.NEAREST)
+            bbox = mask_img.getbbox()
+            result = {
+                "mask": _pil_image_to_b64(mask_img),
+                "bbox": list(bbox) if bbox else None,
+                "model": backend["model_id"],
+                "device": device,
+            }
+            if grounded:
+                result["grounding"] = grounded
+            return result
+        except HTTPException:
+            raise
+        except Exception as exc:
+            logger.exception("smart_mask failed")
+            raise HTTPException(500, f"SAM mask failed: {exc}") from exc
+
     @router.post("/api/image/remove-bg")
     async def remove_background(request: Request):
         """Remove background from an image. If the client passes a `hint_mask`

routes/history/history_routes.py

@@ -10,7 +10,9 @@ from fastapi import APIRouter, Request, HTTPException
 
 from core.models import ChatMessage
 from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
+from src.auth_helpers import effective_user
 from src.topic_analyzer import analyze_topics
+from src.upload_handler import reserve_message_upload_references
 from routes.session_routes import (
     _message_role,
     _message_text,
@@ -98,9 +100,29 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
     return to_delete
 
 
-def setup_history_routes(session_manager) -> APIRouter:
+def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
     router = APIRouter(tags=["history"])
 
+    def _reserve_message_uploads(
+        request: Request,
+        content: Any,
+        metadata: Any = None,
+    ) -> None:
+        try:
+            missing_id = reserve_message_upload_references(
+                upload_handler,
+                effective_user(request),
+                content,
+                metadata,
+            )
+        except (TypeError, ValueError) as exc:
+            raise HTTPException(400, "Invalid message attachment metadata") from exc
+        if missing_id:
+            raise HTTPException(
+                409,
+                f"Referenced upload is no longer available: {missing_id}",
+            )
+
     def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
         entry = {"role": m.role, "content": _history_display_content(m.content)}
         meta = {}
@@ -138,6 +160,8 @@ def setup_history_routes(session_manager) -> APIRouter:
                 )
                 page_offset = int(offset) if offset is not None else max(total - page_limit, 0)
                 page_offset = max(0, min(page_offset, total))
+                # Keep display pagination page-scoped. ``get_session`` is the
+                # full model-context hydration seam and must not be entered here.
                 rows = (
                     db.query(DbChatMessage)
                     .filter(DbChatMessage.session_id == session_id)
@@ -190,7 +214,10 @@ def setup_history_routes(session_manager) -> APIRouter:
                     entry["metadata"] = msg["metadata"]
                 history_dict.append(entry)
 
-        # Fallback: load from DB if in-memory is empty
+        # Fallback: load from DB if in-memory renders empty. Display only —
+        # get_session above is the hydration seam, so nothing here writes back
+        # into session.history — rebuilding it from raw rows would overwrite
+        # parsed multimodal content and the _db_id edit/delete keys it just set.
         if not history_dict:
             db = SessionLocal()
             try:
@@ -200,20 +227,10 @@ def setup_history_routes(session_manager) -> APIRouter:
                     .order_by(DbChatMessage.timestamp)
                     .all()
                 )
-                db_history = []
-                for m in db_messages:
-                    db_history.append(_db_history_entry(m))
-                if db_history:
-                    # Rebuild in-memory history from the full set so hidden
-                    # messages (e.g. compaction summaries) are kept for AI context.
-                    session.history = [
-                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
-                        for m in db_history
-                    ]
                 # Response excludes hidden messages, matching the in-memory path.
                 history_dict = [
-                    m for m in db_history
-                    if not (m.get("metadata") or {}).get("hidden")
+                    entry for entry in (_db_history_entry(m) for m in db_messages)
+                    if not (entry.get("metadata") or {}).get("hidden")
                 ]
             except Exception as e:
                 logger.error(f"DB fallback failed for {session_id}: {e}")
@@ -251,7 +268,9 @@ def setup_history_routes(session_manager) -> APIRouter:
             content = body.get("content", "")
             if not content:
                 raise HTTPException(400, "content is required")
-            msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
+            metadata = body.get("metadata")
+            _reserve_message_uploads(request, content, metadata)
+            msg = ChatMessage(role=role, content=content, metadata=metadata)
             session_manager.add_message(session_id, msg)
             return {"status": "ok"}
         except KeyError:
@@ -331,6 +350,8 @@ def setup_history_routes(session_manager) -> APIRouter:
             if not msg_id or content is None:
                 raise HTTPException(400, "msg_id and content are required")
 
+            _reserve_message_uploads(request, content)
+
             session = session_manager.get_session(session_id)
             db = SessionLocal()
             try:
@@ -576,8 +597,14 @@ def setup_history_routes(session_manager) -> APIRouter:
             body = await request.json()
             keep_count = body.get("keep_count", 0)
 
-            # Get the source session
-            source = session_manager.sessions.get(session_id)
+            # Get the source session. keep_count indexes into source.history,
+            # so this must go through get_session — reading the cache directly
+            # forks an empty transcript out of a metadata-only session after a
+            # restart (display pagination no longer hydrates it).
+            try:
+                source = session_manager.get_session(session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
             if not source:
                 raise HTTPException(404, "Session not found")
 
@@ -630,6 +657,55 @@ def setup_history_routes(session_manager) -> APIRouter:
         except Exception as e:
             raise HTTPException(500, f"Topic analysis failed: {e}")
 
+    @router.get("/api/session/{session_id}/context")
+    async def get_session_context_usage(request: Request, session_id: str) -> Dict[str, Any]:
+        """Return an estimated whole-chat context usage for the session's model.
+
+        Streaming footers report the prompt size for the last request. This
+        endpoint estimates the persisted session context so the header can show
+        when the whole chat is approaching compaction.
+        """
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            from src.model_context import estimate_tokens, get_context_length
+
+            messages = session.get_context_messages()
+            used = int(estimate_tokens(messages))
+            ctx_len = int(get_context_length(session.endpoint_url, session.model) or 0)
+            pct = round((used / ctx_len) * 100, 1) if ctx_len else 0.0
+            pct = max(0.0, min(100.0, pct))
+            visible_messages = sum(
+                1 for m in session.history
+                if not (getattr(m, "metadata", None) or {}).get("hidden")
+            )
+            compacted_messages = sum(
+                1 for m in session.history
+                if (getattr(m, "metadata", None) or {}).get("compacted")
+            )
+            can_compact = used > 0
+            return {
+                "session_id": session_id,
+                "model": session.model,
+                "endpoint_url": session.endpoint_url,
+                "used_tokens": used,
+                "context_length": ctx_len,
+                "context_percent": pct,
+                "messages": visible_messages,
+                "context_messages": len(messages),
+                "compacted_messages": compacted_messages,
+                "can_compact": can_compact,
+                "should_compact": pct >= 70,
+                "auto_compact_threshold": 85,
+            }
+        except Exception as e:
+            logger.error(f"Context usage error {session_id}: {e}")
+            raise HTTPException(500, str(e))
+
     @router.post("/api/session/{session_id}/compact")
     async def compact_session(request: Request, session_id: str):
         """Manually trigger context compaction for a session."""
@@ -674,7 +750,7 @@ def setup_history_routes(session_manager) -> APIRouter:
             compact_model = util_model or session.model
             compact_headers = util_headers if util_url else session.headers
 
-            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
+            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT, normalize_compaction_summary
             compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
             sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
             summary = await llm_call_async(
@@ -686,6 +762,7 @@ def setup_history_routes(session_manager) -> APIRouter:
                 temperature=0.2, max_tokens=1024,
                 headers=compact_headers, timeout=30,
             )
+            summary = normalize_compaction_summary(summary)
 
             # Replace session history: summary as system message + recent messages
             # System message holds the full summary for AI context

routes/hwfit_routes.py

@@ -429,11 +429,27 @@ def setup_hwfit_routes():
             system["available_ram_gb"] = 0
             system["total_ram_gb"] = 0
         system = _apply_manual_hardware(system, manual_mode, manual_gpu_count, manual_vram_gb, manual_ram_gb, manual_backend)
-        # Image models use a single GPU — always use per-GPU VRAM
-        gpu_vrams = [float(g.get("vram_gb") or 0) for g in (system.get("gpus") or []) if isinstance(g, dict)]
-        single_vram = max(gpu_vrams) if gpu_vrams else ((system.get("gpu_vram_gb") or 0) / max(system.get("gpu_count") or 1, 1))
-        system["gpu_vram_gb"] = single_vram
-        system["gpu_count"] = 1 if single_vram > 0 else 0
+        try:
+            requested_gpu_count = int(gpu_count) if gpu_count != "" else None
+        except ValueError:
+            requested_gpu_count = None
+        if requested_gpu_count == 0:
+            # Respect the UI's RAM toggle. Before this route always rewrote the
+            # system to best-single-GPU VRAM, so image rows never changed when
+            # switching RAM/GPU.
+            system["has_gpu"] = False
+            system["gpu_vram_gb"] = 0
+            system["gpu_count"] = 0
+            system["gpu_only"] = False
+        else:
+            # Image diffusion backends generally use one device per pipeline,
+            # so rank GPU mode against the best single GPU rather than total
+            # multi-GPU VRAM.
+            gpu_vrams = [float(g.get("vram_gb") or 0) for g in (system.get("gpus") or []) if isinstance(g, dict)]
+            single_vram = max(gpu_vrams) if gpu_vrams else ((system.get("gpu_vram_gb") or 0) / max(system.get("gpu_count") or 1, 1))
+            system["gpu_vram_gb"] = single_vram
+            system["gpu_count"] = 1 if single_vram > 0 else 0
+            system["gpu_only"] = True if single_vram > 0 else False
         results = rank_image_models(system, search=search or None, sort=sort)
         return {"system": system, "models": results}
 

routes/mcp/__init__.py

@@ -0,0 +1,5 @@
+"""MCP route domain package (slice 2o, #4082/#4071).
+
+Contains mcp_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/mcp_routes.py re-exports from here.
+"""

routes/mcp/mcp_routes.py

@@ -0,0 +1,703 @@
+# routes/mcp_routes.py
+"""MCP (Model Context Protocol) server management routes."""
+import json
+import os
+import uuid
+import urllib.parse
+import html
+from pathlib import Path
+from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi.responses import RedirectResponse, HTMLResponse
+import logging
+import httpx
+
+from core.database import McpServer, SessionLocal
+from core.middleware import require_admin
+from src.constants import DATA_DIR, MCP_OAUTH_DIR
+from src.mcp_manager import McpManager
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/mcp", tags=["mcp"])
+
+
+def _mcp_oauth_base_dir() -> Path:
+    """Directory that may contain OAuth files managed by Odysseus."""
+    return Path(MCP_OAUTH_DIR).resolve(strict=False)
+
+
+def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str:
+    """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth."""
+    raw = str(raw_path or "").strip()
+    if not raw:
+        return ""
+
+    base = _mcp_oauth_base_dir()
+    path = Path(os.path.expanduser(raw))
+    if not path.is_absolute():
+        path = base / path
+    resolved = path.resolve(strict=False)
+
+    try:
+        resolved.relative_to(base)
+    except ValueError as exc:
+        raise HTTPException(
+            400,
+            f"Invalid OAuth {field_name}: path must stay under {base}",
+        ) from exc
+    return str(resolved)
+
+
+def _sanitize_mcp_oauth_config(oauth_cfg):
+    """Return an OAuth config copy with file paths confined to mcp_oauth."""
+    if not oauth_cfg:
+        return oauth_cfg
+    if not isinstance(oauth_cfg, dict):
+        return {}
+    sanitized = dict(oauth_cfg)
+    for field_name in ("keys_file", "token_file"):
+        if sanitized.get(field_name):
+            sanitized[field_name] = _resolve_mcp_oauth_path(
+                sanitized[field_name],
+                field_name,
+            )
+    return sanitized
+
+
+def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool:
+    """Check token existence without letting legacy bad paths break listing."""
+    if not isinstance(oauth_cfg, dict):
+        return False
+    try:
+        token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file")
+    except HTTPException:
+        if strict:
+            raise
+        logger.warning("Ignoring MCP OAuth config with unsafe token_file")
+        return True
+    return bool(token_file and not os.path.exists(token_file))
+
+
+def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None:
+    """Pass sanitized Gmail package paths to MCP servers that honor them."""
+    if not oauth_cfg or not isinstance(env, dict):
+        return
+    keys_file = oauth_cfg.get("keys_file")
+    token_file = oauth_cfg.get("token_file")
+    if keys_file:
+        env["GMAIL_OAUTH_PATH"] = keys_file
+    if token_file:
+        env["GMAIL_CREDENTIALS_PATH"] = token_file
+
+
+def _load_disabled_map():
+    """Load per-server disabled tool sets from DB."""
+    db = SessionLocal()
+    try:
+        disabled_map = {}
+        for srv in db.query(McpServer).all():
+            if srv.disabled_tools:
+                try:
+                    names = json.loads(srv.disabled_tools)
+                    if names:
+                        disabled_map[srv.id] = set(names)
+                except (json.JSONDecodeError, TypeError):
+                    pass
+        return disabled_map
+    finally:
+        db.close()
+
+
+def _mcp_oauth_redirect_uri() -> str:
+    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
+    from src.mcp_oauth import REDIRECT_URI
+    return REDIRECT_URI
+
+
+def setup_mcp_routes(mcp_manager: McpManager):
+    """Setup MCP routes with the provided manager."""
+
+    @router.get("/servers")
+    def list_servers(request: Request):
+        """List all configured MCP servers with connection status."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            servers = db.query(McpServer).all()
+            result = []
+            for srv in servers:
+                status = mcp_manager.get_server_status(srv.id)
+                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
+                needs_oauth = False
+                if oauth_cfg:
+                    needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False)
+                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+                total_tools = status.get("tool_count", 0)
+                result.append({
+                    "id": srv.id,
+                    "name": srv.name,
+                    "transport": srv.transport,
+                    "command": srv.command,
+                    "args": json.loads(srv.args) if srv.args else [],
+                    "env": json.loads(srv.env) if srv.env else {},
+                    "url": srv.url,
+                    "is_enabled": srv.is_enabled,
+                    "status": status.get("status", "disconnected"),
+                    "tool_count": total_tools,
+                    "disabled_tool_count": len(disabled_list),
+                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
+                    "error": status.get("error"),
+                    "auth_url": status.get("auth_url"),
+                    "has_oauth": oauth_cfg is not None,
+                    "needs_oauth": needs_oauth,
+                })
+            return result
+        finally:
+            db.close()
+
+    @router.post("/servers")
+    async def add_server(
+        request: Request,
+        name: str = Form(...),
+        transport: str = Form("stdio"),
+        command: str = Form(None),
+        args: str = Form("[]"),
+        env: str = Form("{}"),
+        url: str = Form(None),
+        oauth_file: str = Form(None),
+        oauth_config: str = Form(None),
+    ):
+        """Add a new MCP server config and attempt connection. Admin-only:
+        registering a stdio server is equivalent to executing arbitrary
+        binaries on the host."""
+        require_admin(request)
+        server_id = str(uuid.uuid4())[:8]
+
+        # Validate
+        if transport == "stdio" and not command:
+            raise HTTPException(400, "command is required for stdio transport")
+        if transport == "sse" and not url:
+            raise HTTPException(400, "url is required for SSE transport")
+        if transport == "http" and not url:
+            raise HTTPException(400, "url is required for HTTP transport")
+
+        # Parse JSON fields
+        try:
+            parsed_args = json.loads(args) if args else []
+        except json.JSONDecodeError:
+            parsed_args = []
+        try:
+            parsed_env = json.loads(env) if env else {}
+        except json.JSONDecodeError:
+            parsed_env = {}
+        if not isinstance(parsed_env, dict):
+            parsed_env = {}
+
+        # Parse OAuth config
+        parsed_oauth_config = None
+        if oauth_config:
+            try:
+                parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config))
+            except json.JSONDecodeError:
+                pass
+        _apply_mcp_oauth_env(parsed_env, parsed_oauth_config)
+
+        # Write OAuth credentials file if provided (for Google MCP servers)
+        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
+        if oauth_file:
+            try:
+                oauth_data = json.loads(oauth_file)
+                oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir")
+                oauth_filename = oauth_data.get("filename", "")
+                client_id = oauth_data.get("client_id", "")
+                client_secret = oauth_data.get("client_secret", "")
+                if oauth_dir and oauth_filename and client_id and client_secret:
+                    filepath = _resolve_mcp_oauth_path(
+                        Path(oauth_dir) / str(oauth_filename),
+                        "filename",
+                    )
+                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+                    creds = {
+                        "installed": {
+                            "client_id": client_id,
+                            "client_secret": client_secret,
+                            "redirect_uris": ["http://localhost"],
+                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                            "token_uri": "https://accounts.google.com/o/oauth2/token",
+                        }
+                    }
+                    with open(filepath, "w", encoding="utf-8") as f:
+                        json.dump(creds, f, indent=2)
+                    logger.info(f"Wrote OAuth credentials to {filepath}")
+                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
+                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
+            except (json.JSONDecodeError, OSError) as e:
+                logger.warning(f"Failed to write OAuth file: {e}")
+
+        # Save to DB
+        db = SessionLocal()
+        try:
+            srv = McpServer(
+                id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=json.dumps(parsed_args),
+                env=json.dumps(parsed_env),
+                url=url,
+                is_enabled=True,
+                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
+            )
+            db.add(srv)
+            db.commit()
+        finally:
+            db.close()
+
+        # Check if OAuth token already exists — skip connection attempt if not
+        needs_oauth = False
+        if parsed_oauth_config:
+            needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config)
+
+        connected = False
+        if not needs_oauth:
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=parsed_args,
+                env=parsed_env,
+                url=url,
+            )
+
+        status = mcp_manager.get_server_status(server_id)
+        needs_auth = status.get("status") == "needs_auth"
+        return {
+            "id": server_id,
+            "name": name,
+            "connected": connected,
+            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
+            "tool_count": status.get("tool_count", 0),
+            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
+            "needs_oauth": needs_oauth,
+            "needs_auth": needs_auth,
+            "auth_url": status.get("auth_url"),
+        }
+
+    @router.post("/servers/{server_id}/reconnect")
+    async def reconnect_server(server_id: str, request: Request):
+        """Reconnect to an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            status = mcp_manager.get_server_status(server_id)
+            return {
+                "connected": connected,
+                "status": status.get("status", "disconnected"),
+                "tool_count": status.get("tool_count", 0),
+                "error": status.get("error"),
+                "auth_url": status.get("auth_url"),
+                "needs_auth": status.get("status") == "needs_auth",
+            }
+        finally:
+            db.close()
+
+    @router.patch("/servers/{server_id}")
+    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
+        """Enable or disable an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            enabled = str(is_enabled).lower() == "true"
+            srv.is_enabled = enabled
+            db.commit()
+
+            if enabled:
+                args = json.loads(srv.args) if srv.args else []
+                env = json.loads(srv.env) if srv.env else {}
+                await mcp_manager.connect_server(
+                    server_id=server_id,
+                    name=srv.name,
+                    transport=srv.transport,
+                    command=srv.command,
+                    args=args,
+                    env=env,
+                    url=srv.url,
+                )
+            else:
+                await mcp_manager.disconnect_server(server_id)
+
+            return {"id": server_id, "is_enabled": enabled}
+        finally:
+            db.close()
+
+    @router.delete("/servers/{server_id}")
+    async def delete_server(server_id: str, request: Request):
+        """Remove an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            db.delete(srv)
+            db.commit()
+            return {"status": "deleted"}
+        finally:
+            db.close()
+
+    @router.get("/tools")
+    def list_tools(request: Request):
+        """List all discovered MCP tools across all connected servers."""
+        require_admin(request)
+        disabled_map = _load_disabled_map()
+        return mcp_manager.get_all_tools(disabled_map)
+
+    @router.get("/servers/{server_id}/tools")
+    def list_server_tools(server_id: str, request: Request):
+        """List all tools for a specific MCP server with enabled/disabled state."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+            disabled_set = set(disabled_list)
+        finally:
+            db.close()
+
+        all_tools = mcp_manager.get_all_tools()
+        server_tools = [t for t in all_tools if t["server_id"] == server_id]
+        for t in server_tools:
+            t["is_disabled"] = t["name"] in disabled_set
+        return server_tools
+
+    @router.patch("/servers/{server_id}/tools")
+    async def update_disabled_tools(server_id: str, request: Request):
+        """Bulk update disabled tools list for a server.
+
+        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
+        """
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            body = await request.json()
+            disabled = body.get("disabled", [])
+            if not isinstance(disabled, list):
+                raise HTTPException(400, "disabled must be a list of tool names")
+
+            srv.disabled_tools = json.dumps(disabled) if disabled else None
+            db.commit()
+
+            return {"id": server_id, "disabled_count": len(disabled)}
+        finally:
+            db.close()
+
+    # ── OAuth flow for Google MCP servers ──────────────────────────
+
+    @router.get("/oauth/authorize/{server_id}")
+    def oauth_authorize(server_id: str, request: Request):
+        """Show OAuth authorization page with Google sign-in link."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            if not srv.oauth_config:
+                raise HTTPException(400, "Server has no OAuth config")
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            if not keys_file or not os.path.exists(keys_file):
+                raise HTTPException(400, "OAuth keys file not found")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            if not keys:
+                raise HTTPException(400, "Invalid OAuth keys file format")
+
+            client_id = keys["client_id"]
+            scopes = oauth_cfg.get("scopes", [])
+
+            # For Desktop App creds, default to localhost — the user will
+            # paste the resulting URL back if they're on a different device.
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            params = {
+                "client_id": client_id,
+                "redirect_uri": redirect_uri,
+                "response_type": "code",
+                "scope": " ".join(scopes),
+                "access_type": "offline",
+                "prompt": "consent",
+                "state": server_id,
+            }
+            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
+
+            # Determine if user is accessing from the same machine
+            host = request.headers.get("host", "")
+            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
+
+            if is_local:
+                # Same machine — just redirect, callback will work directly
+                return RedirectResponse(auth_url)
+            else:
+                # Remote device — show paste-back page
+                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, redirect_uri))
+        finally:
+            db.close()
+
+    @router.get("/oauth/callback")
+    async def oauth_callback(code: str, state: str, request: Request):
+        """Handle OAuth callback. Generic MCP OAuth flows resolve via the
+        pending-state registry; Google flows fall through to the legacy path."""
+        require_admin(request)
+        from src.mcp_oauth import resolve_pending
+        if resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+        # Legacy Google path: state is the server_id
+        return await _exchange_and_connect(state, code, request)
+
+    @router.post("/oauth/exchange/{server_id}")
+    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
+        """Manual code exchange — user pastes the callback URL from their browser."""
+        require_admin(request)
+        try:
+            parsed = urllib.parse.urlparse(callback_url)
+            params = urllib.parse.parse_qs(parsed.query)
+            code = params.get("code", [None])[0]
+            if not code:
+                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
+        except Exception:
+            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
+
+        # Generic MCP OAuth: if the pasted URL carries a state we are waiting on,
+        # resolve it directly (the background connect finishes the handshake).
+        state = params.get("state", [None])[0]
+        from src.mcp_oauth import resolve_pending
+        if state and resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+
+        return await _exchange_and_connect(server_id, code, request)
+
+    async def _exchange_and_connect(server_id: str, code: str, request: Request):
+        """Exchange auth code for tokens and connect the MCP server."""
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
+            if not srv.oauth_config:
+                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            token_file = oauth_cfg.get("token_file", "")
+            if not keys_file or not token_file:
+                raise HTTPException(400, "OAuth keys/token file not configured")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            client_id = keys["client_id"]
+            client_secret = keys["client_secret"]
+
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    "https://oauth2.googleapis.com/token",
+                    data={
+                        "code": code,
+                        "client_id": client_id,
+                        "client_secret": client_secret,
+                        "redirect_uri": redirect_uri,
+                        "grant_type": "authorization_code",
+                    },
+                )
+
+            if resp.status_code != 200:
+                err = resp.text
+                logger.error(f"OAuth token exchange failed: {err}")
+                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
+
+            tokens = resp.json()
+            logger.info(f"OAuth tokens received for server {server_id}")
+
+            # Save tokens to the file the MCP package expects
+            os.makedirs(os.path.dirname(token_file), exist_ok=True)
+            with open(token_file, "w", encoding="utf-8") as f:
+                json.dump(tokens, f, indent=2)
+            logger.info(f"Saved OAuth tokens to {token_file}")
+
+            # Attempt to connect the MCP server now
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            if connected:
+                status = mcp_manager.get_server_status(server_id)
+                tool_count = status.get("tool_count", 0)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorization Successful",
+                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
+                    success=True,
+                ))
+            else:
+                status = mcp_manager.get_server_status(server_id)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorized but Connection Failed",
+                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
+                ))
+        except HTTPException as e:
+            logger.warning(f"OAuth callback rejected: {e.detail}")
+            return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code)
+        except Exception as e:
+            logger.exception(f"OAuth callback error: {e}")
+            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
+        finally:
+            db.close()
+
+    return router
+
+
+def _oauth_authorize_page(
+    auth_url: str,
+    server_id: str,
+    redirect_uri: str,
+) -> str:
+    """Page with Google sign-in link and URL paste-back form for remote access."""
+    # Escape values interpolated into the page: `server_id` comes from the OAuth
+    # state and is not trusted.
+    auth_url = html.escape(auth_url, quote=True)
+    server_id = html.escape(server_id, quote=True)
+    redirect_uri = html.escape(redirect_uri, quote=True)
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>Authorize — Odysseus</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 480px; text-align: center; }}
+  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
+  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
+  .step b {{ color: #e06c75; }}
+  a.auth-link {{
+    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
+    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
+    font-weight: 600; font-size: 0.9rem;
+  }}
+  a.auth-link:hover {{ background: #c55; }}
+  input[type=text] {{
+    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
+    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
+    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
+  }}
+  input:focus {{ outline: none; border-color: #e06c75; }}
+  button {{
+    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
+    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
+    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
+  }}
+  button:hover {{ background: #c55; }}
+  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
+</style></head>
+<body><div class="card">
+  <h2>Authorize Google Account</h2>
+  <div class="step">
+    <b>1.</b> Click the button below to sign in with Google<br>
+    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
+    <b>3.</b> Copy the full URL from your browser's address bar<br>
+    <b>4.</b> Paste it below and click Connect
+  </div>
+  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
+  <div class="divider"></div>
+  <!-- Relative action: the browser resolves it against the origin this page was
+       served from, so the form follows the user through any proxy without the
+       app having to know the scheme or the host. An absolute http:// action is
+       blocked as mixed content on exactly the HTTPS deployments that need
+       paste-back, and request.url.scheme cannot be trusted to spot them —
+       uvicorn only honours X-Forwarded-Proto from a peer in
+       --forwarded-allow-ips, which defaults to 127.0.0.1 and excludes a proxy
+       arriving over the Docker bridge. -->
+  <form method="POST" action="/api/mcp/oauth/exchange/{server_id}">
+    <p>Paste the URL from your browser after signing in:</p>
+    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
+    <br><button type="submit">Connect</button>
+  </form>
+</div></body></html>"""
+
+
+def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
+    """Generate a simple HTML page for the OAuth result."""
+    safe_title = html.escape(title)
+    safe_message = html.escape(message)
+    color = "#00661a" if success else "#e06c75"
+    icon = "&#10003;" if success else "&#10007;"
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>{safe_title}</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 420px; text-align: center; }}
+  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
+  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
+</style></head>
+<body><div class="card">
+  <div class="icon">{icon}</div>
+  <h2>{safe_title}</h2>
+  <p>{safe_message}</p>
+</div></body></html>"""

routes/mcp_routes.py

@@ -1,697 +1,18 @@
-# routes/mcp_routes.py
-"""MCP (Model Context Protocol) server management routes."""
-import json
-import os
-import uuid
-import urllib.parse
-import html
-from pathlib import Path
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-import logging
-import httpx
+"""Backward-compat shim — canonical location is routes/mcp/mcp_routes.py.
 
-from core.database import McpServer, SessionLocal
-from core.middleware import require_admin
-from src.constants import DATA_DIR, MCP_OAUTH_DIR
-from src.mcp_manager import McpManager
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.mcp_routes``, ``from routes.mcp_routes import X``,
+``importlib.import_module("routes.mcp_routes")``, the
+``sys.modules.pop("routes.mcp_routes")`` + re-import pattern in
+test_security_regressions.py, and the ``monkeypatch.setattr(mcp_routes,
+"MCP_OAUTH_DIR", ...)`` pattern all operate on the *same* object. This also
+makes ``mcp_routes.__file__`` resolve to the canonical file (which the
+source-introspection at line 839 reads). Keeps existing import paths working
+after slice 2o (#4082/#4071).
+"""
 
-logger = logging.getLogger(__name__)
+import sys as _sys
 
-router = APIRouter(prefix="/api/mcp", tags=["mcp"])
+from routes.mcp import mcp_routes as _canonical  # noqa: F401
 
-
-def _mcp_oauth_base_dir() -> Path:
-    """Directory that may contain OAuth files managed by Odysseus."""
-    return Path(MCP_OAUTH_DIR).resolve(strict=False)
-
-
-def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str:
-    """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth."""
-    raw = str(raw_path or "").strip()
-    if not raw:
-        return ""
-
-    base = _mcp_oauth_base_dir()
-    path = Path(os.path.expanduser(raw))
-    if not path.is_absolute():
-        path = base / path
-    resolved = path.resolve(strict=False)
-
-    try:
-        resolved.relative_to(base)
-    except ValueError as exc:
-        raise HTTPException(
-            400,
-            f"Invalid OAuth {field_name}: path must stay under {base}",
-        ) from exc
-    return str(resolved)
-
-
-def _sanitize_mcp_oauth_config(oauth_cfg):
-    """Return an OAuth config copy with file paths confined to mcp_oauth."""
-    if not oauth_cfg:
-        return oauth_cfg
-    if not isinstance(oauth_cfg, dict):
-        return {}
-    sanitized = dict(oauth_cfg)
-    for field_name in ("keys_file", "token_file"):
-        if sanitized.get(field_name):
-            sanitized[field_name] = _resolve_mcp_oauth_path(
-                sanitized[field_name],
-                field_name,
-            )
-    return sanitized
-
-
-def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool:
-    """Check token existence without letting legacy bad paths break listing."""
-    if not isinstance(oauth_cfg, dict):
-        return False
-    try:
-        token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file")
-    except HTTPException:
-        if strict:
-            raise
-        logger.warning("Ignoring MCP OAuth config with unsafe token_file")
-        return True
-    return bool(token_file and not os.path.exists(token_file))
-
-
-def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None:
-    """Pass sanitized Gmail package paths to MCP servers that honor them."""
-    if not oauth_cfg or not isinstance(env, dict):
-        return
-    keys_file = oauth_cfg.get("keys_file")
-    token_file = oauth_cfg.get("token_file")
-    if keys_file:
-        env["GMAIL_OAUTH_PATH"] = keys_file
-    if token_file:
-        env["GMAIL_CREDENTIALS_PATH"] = token_file
-
-
-def _load_disabled_map():
-    """Load per-server disabled tool sets from DB."""
-    db = SessionLocal()
-    try:
-        disabled_map = {}
-        for srv in db.query(McpServer).all():
-            if srv.disabled_tools:
-                try:
-                    names = json.loads(srv.disabled_tools)
-                    if names:
-                        disabled_map[srv.id] = set(names)
-                except (json.JSONDecodeError, TypeError):
-                    pass
-        return disabled_map
-    finally:
-        db.close()
-
-
-def _mcp_oauth_redirect_uri() -> str:
-    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
-    from src.mcp_oauth import REDIRECT_URI
-    return REDIRECT_URI
-
-
-def setup_mcp_routes(mcp_manager: McpManager):
-    """Setup MCP routes with the provided manager."""
-
-    @router.get("/servers")
-    def list_servers(request: Request):
-        """List all configured MCP servers with connection status."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            servers = db.query(McpServer).all()
-            result = []
-            for srv in servers:
-                status = mcp_manager.get_server_status(srv.id)
-                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
-                needs_oauth = False
-                if oauth_cfg:
-                    needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False)
-                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-                total_tools = status.get("tool_count", 0)
-                result.append({
-                    "id": srv.id,
-                    "name": srv.name,
-                    "transport": srv.transport,
-                    "command": srv.command,
-                    "args": json.loads(srv.args) if srv.args else [],
-                    "env": json.loads(srv.env) if srv.env else {},
-                    "url": srv.url,
-                    "is_enabled": srv.is_enabled,
-                    "status": status.get("status", "disconnected"),
-                    "tool_count": total_tools,
-                    "disabled_tool_count": len(disabled_list),
-                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
-                    "error": status.get("error"),
-                    "auth_url": status.get("auth_url"),
-                    "has_oauth": oauth_cfg is not None,
-                    "needs_oauth": needs_oauth,
-                })
-            return result
-        finally:
-            db.close()
-
-    @router.post("/servers")
-    async def add_server(
-        request: Request,
-        name: str = Form(...),
-        transport: str = Form("stdio"),
-        command: str = Form(None),
-        args: str = Form("[]"),
-        env: str = Form("{}"),
-        url: str = Form(None),
-        oauth_file: str = Form(None),
-        oauth_config: str = Form(None),
-    ):
-        """Add a new MCP server config and attempt connection. Admin-only:
-        registering a stdio server is equivalent to executing arbitrary
-        binaries on the host."""
-        require_admin(request)
-        server_id = str(uuid.uuid4())[:8]
-
-        # Validate
-        if transport == "stdio" and not command:
-            raise HTTPException(400, "command is required for stdio transport")
-        if transport == "sse" and not url:
-            raise HTTPException(400, "url is required for SSE transport")
-        if transport == "http" and not url:
-            raise HTTPException(400, "url is required for HTTP transport")
-
-        # Parse JSON fields
-        try:
-            parsed_args = json.loads(args) if args else []
-        except json.JSONDecodeError:
-            parsed_args = []
-        try:
-            parsed_env = json.loads(env) if env else {}
-        except json.JSONDecodeError:
-            parsed_env = {}
-        if not isinstance(parsed_env, dict):
-            parsed_env = {}
-
-        # Parse OAuth config
-        parsed_oauth_config = None
-        if oauth_config:
-            try:
-                parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config))
-            except json.JSONDecodeError:
-                pass
-        _apply_mcp_oauth_env(parsed_env, parsed_oauth_config)
-
-        # Write OAuth credentials file if provided (for Google MCP servers)
-        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
-        if oauth_file:
-            try:
-                oauth_data = json.loads(oauth_file)
-                oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir")
-                oauth_filename = oauth_data.get("filename", "")
-                client_id = oauth_data.get("client_id", "")
-                client_secret = oauth_data.get("client_secret", "")
-                if oauth_dir and oauth_filename and client_id and client_secret:
-                    filepath = _resolve_mcp_oauth_path(
-                        Path(oauth_dir) / str(oauth_filename),
-                        "filename",
-                    )
-                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
-                    creds = {
-                        "installed": {
-                            "client_id": client_id,
-                            "client_secret": client_secret,
-                            "redirect_uris": ["http://localhost"],
-                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                            "token_uri": "https://accounts.google.com/o/oauth2/token",
-                        }
-                    }
-                    with open(filepath, "w", encoding="utf-8") as f:
-                        json.dump(creds, f, indent=2)
-                    logger.info(f"Wrote OAuth credentials to {filepath}")
-                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
-                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
-            except (json.JSONDecodeError, OSError) as e:
-                logger.warning(f"Failed to write OAuth file: {e}")
-
-        # Save to DB
-        db = SessionLocal()
-        try:
-            srv = McpServer(
-                id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=json.dumps(parsed_args),
-                env=json.dumps(parsed_env),
-                url=url,
-                is_enabled=True,
-                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
-            )
-            db.add(srv)
-            db.commit()
-        finally:
-            db.close()
-
-        # Check if OAuth token already exists — skip connection attempt if not
-        needs_oauth = False
-        if parsed_oauth_config:
-            needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config)
-
-        connected = False
-        if not needs_oauth:
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=parsed_args,
-                env=parsed_env,
-                url=url,
-            )
-
-        status = mcp_manager.get_server_status(server_id)
-        needs_auth = status.get("status") == "needs_auth"
-        return {
-            "id": server_id,
-            "name": name,
-            "connected": connected,
-            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
-            "tool_count": status.get("tool_count", 0),
-            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
-            "needs_oauth": needs_oauth,
-            "needs_auth": needs_auth,
-            "auth_url": status.get("auth_url"),
-        }
-
-    @router.post("/servers/{server_id}/reconnect")
-    async def reconnect_server(server_id: str, request: Request):
-        """Reconnect to an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            status = mcp_manager.get_server_status(server_id)
-            return {
-                "connected": connected,
-                "status": status.get("status", "disconnected"),
-                "tool_count": status.get("tool_count", 0),
-                "error": status.get("error"),
-                "auth_url": status.get("auth_url"),
-                "needs_auth": status.get("status") == "needs_auth",
-            }
-        finally:
-            db.close()
-
-    @router.patch("/servers/{server_id}")
-    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
-        """Enable or disable an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            enabled = str(is_enabled).lower() == "true"
-            srv.is_enabled = enabled
-            db.commit()
-
-            if enabled:
-                args = json.loads(srv.args) if srv.args else []
-                env = json.loads(srv.env) if srv.env else {}
-                await mcp_manager.connect_server(
-                    server_id=server_id,
-                    name=srv.name,
-                    transport=srv.transport,
-                    command=srv.command,
-                    args=args,
-                    env=env,
-                    url=srv.url,
-                )
-            else:
-                await mcp_manager.disconnect_server(server_id)
-
-            return {"id": server_id, "is_enabled": enabled}
-        finally:
-            db.close()
-
-    @router.delete("/servers/{server_id}")
-    async def delete_server(server_id: str, request: Request):
-        """Remove an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            db.delete(srv)
-            db.commit()
-            return {"status": "deleted"}
-        finally:
-            db.close()
-
-    @router.get("/tools")
-    def list_tools(request: Request):
-        """List all discovered MCP tools across all connected servers."""
-        require_admin(request)
-        disabled_map = _load_disabled_map()
-        return mcp_manager.get_all_tools(disabled_map)
-
-    @router.get("/servers/{server_id}/tools")
-    def list_server_tools(server_id: str, request: Request):
-        """List all tools for a specific MCP server with enabled/disabled state."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-            disabled_set = set(disabled_list)
-        finally:
-            db.close()
-
-        all_tools = mcp_manager.get_all_tools()
-        server_tools = [t for t in all_tools if t["server_id"] == server_id]
-        for t in server_tools:
-            t["is_disabled"] = t["name"] in disabled_set
-        return server_tools
-
-    @router.patch("/servers/{server_id}/tools")
-    async def update_disabled_tools(server_id: str, request: Request):
-        """Bulk update disabled tools list for a server.
-
-        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
-        """
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            body = await request.json()
-            disabled = body.get("disabled", [])
-            if not isinstance(disabled, list):
-                raise HTTPException(400, "disabled must be a list of tool names")
-
-            srv.disabled_tools = json.dumps(disabled) if disabled else None
-            db.commit()
-
-            return {"id": server_id, "disabled_count": len(disabled)}
-        finally:
-            db.close()
-
-    # ── OAuth flow for Google MCP servers ──────────────────────────
-
-    @router.get("/oauth/authorize/{server_id}")
-    def oauth_authorize(server_id: str, request: Request):
-        """Show OAuth authorization page with Google sign-in link."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            if not srv.oauth_config:
-                raise HTTPException(400, "Server has no OAuth config")
-
-            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
-            keys_file = oauth_cfg.get("keys_file", "")
-            if not keys_file or not os.path.exists(keys_file):
-                raise HTTPException(400, "OAuth keys file not found")
-
-            with open(keys_file, encoding="utf-8") as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            if not keys:
-                raise HTTPException(400, "Invalid OAuth keys file format")
-
-            client_id = keys["client_id"]
-            scopes = oauth_cfg.get("scopes", [])
-
-            # For Desktop App creds, default to localhost — the user will
-            # paste the resulting URL back if they're on a different device.
-            redirect_uri = _mcp_oauth_redirect_uri()
-
-            params = {
-                "client_id": client_id,
-                "redirect_uri": redirect_uri,
-                "response_type": "code",
-                "scope": " ".join(scopes),
-                "access_type": "offline",
-                "prompt": "consent",
-                "state": server_id,
-            }
-            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
-
-            # Determine if user is accessing from the same machine
-            host = request.headers.get("host", "")
-            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
-
-            if is_local:
-                # Same machine — just redirect, callback will work directly
-                return RedirectResponse(auth_url)
-            else:
-                # Remote device — show paste-back page
-                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri))
-        finally:
-            db.close()
-
-    @router.get("/oauth/callback")
-    async def oauth_callback(code: str, state: str, request: Request):
-        """Handle OAuth callback. Generic MCP OAuth flows resolve via the
-        pending-state registry; Google flows fall through to the legacy path."""
-        require_admin(request)
-        from src.mcp_oauth import resolve_pending
-        if resolve_pending(state, code):
-            return HTMLResponse(_oauth_result_page(
-                "Authorization Successful",
-                "The MCP server is connecting. You can close this window and return to Odysseus.",
-                success=True,
-            ))
-        # Legacy Google path: state is the server_id
-        return await _exchange_and_connect(state, code, request)
-
-    @router.post("/oauth/exchange/{server_id}")
-    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
-        """Manual code exchange — user pastes the callback URL from their browser."""
-        require_admin(request)
-        try:
-            parsed = urllib.parse.urlparse(callback_url)
-            params = urllib.parse.parse_qs(parsed.query)
-            code = params.get("code", [None])[0]
-            if not code:
-                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
-        except Exception:
-            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
-
-        # Generic MCP OAuth: if the pasted URL carries a state we are waiting on,
-        # resolve it directly (the background connect finishes the handshake).
-        state = params.get("state", [None])[0]
-        from src.mcp_oauth import resolve_pending
-        if state and resolve_pending(state, code):
-            return HTMLResponse(_oauth_result_page(
-                "Authorization Successful",
-                "The MCP server is connecting. You can close this window and return to Odysseus.",
-                success=True,
-            ))
-
-        return await _exchange_and_connect(server_id, code, request)
-
-    async def _exchange_and_connect(server_id: str, code: str, request: Request):
-        """Exchange auth code for tokens and connect the MCP server."""
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
-            if not srv.oauth_config:
-                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
-
-            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
-            keys_file = oauth_cfg.get("keys_file", "")
-            token_file = oauth_cfg.get("token_file", "")
-            if not keys_file or not token_file:
-                raise HTTPException(400, "OAuth keys/token file not configured")
-
-            with open(keys_file, encoding="utf-8") as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            client_id = keys["client_id"]
-            client_secret = keys["client_secret"]
-
-            redirect_uri = _mcp_oauth_redirect_uri()
-
-            async with httpx.AsyncClient() as client:
-                resp = await client.post(
-                    "https://oauth2.googleapis.com/token",
-                    data={
-                        "code": code,
-                        "client_id": client_id,
-                        "client_secret": client_secret,
-                        "redirect_uri": redirect_uri,
-                        "grant_type": "authorization_code",
-                    },
-                )
-
-            if resp.status_code != 200:
-                err = resp.text
-                logger.error(f"OAuth token exchange failed: {err}")
-                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
-
-            tokens = resp.json()
-            logger.info(f"OAuth tokens received for server {server_id}")
-
-            # Save tokens to the file the MCP package expects
-            os.makedirs(os.path.dirname(token_file), exist_ok=True)
-            with open(token_file, "w", encoding="utf-8") as f:
-                json.dump(tokens, f, indent=2)
-            logger.info(f"Saved OAuth tokens to {token_file}")
-
-            # Attempt to connect the MCP server now
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            if connected:
-                status = mcp_manager.get_server_status(server_id)
-                tool_count = status.get("tool_count", 0)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorization Successful",
-                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
-                    success=True,
-                ))
-            else:
-                status = mcp_manager.get_server_status(server_id)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorized but Connection Failed",
-                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
-                ))
-        except HTTPException as e:
-            logger.warning(f"OAuth callback rejected: {e.detail}")
-            return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code)
-        except Exception as e:
-            logger.exception(f"OAuth callback error: {e}")
-            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
-        finally:
-            db.close()
-
-    return router
-
-
-def _oauth_authorize_page(
-    auth_url: str,
-    server_id: str,
-    host: str,
-    redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback",
-) -> str:
-    """Page with Google sign-in link and URL paste-back form for remote access."""
-    # Escape values interpolated into the page: `host` comes from the request
-    # Host header and `server_id` from the OAuth state — neither is trusted.
-    auth_url = html.escape(auth_url, quote=True)
-    server_id = html.escape(server_id, quote=True)
-    host = html.escape(host, quote=True)
-    redirect_uri = html.escape(redirect_uri, quote=True)
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>Authorize — Odysseus</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 480px; text-align: center; }}
-  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
-  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
-  .step b {{ color: #e06c75; }}
-  a.auth-link {{
-    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
-    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
-    font-weight: 600; font-size: 0.9rem;
-  }}
-  a.auth-link:hover {{ background: #c55; }}
-  input[type=text] {{
-    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
-    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
-    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
-  }}
-  input:focus {{ outline: none; border-color: #e06c75; }}
-  button {{
-    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
-    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
-    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
-  }}
-  button:hover {{ background: #c55; }}
-  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
-</style></head>
-<body><div class="card">
-  <h2>Authorize Google Account</h2>
-  <div class="step">
-    <b>1.</b> Click the button below to sign in with Google<br>
-    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
-    <b>3.</b> Copy the full URL from your browser's address bar<br>
-    <b>4.</b> Paste it below and click Connect
-  </div>
-  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
-  <div class="divider"></div>
-  <form method="POST" action="http://{host}/api/mcp/oauth/exchange/{server_id}">
-    <p>Paste the URL from your browser after signing in:</p>
-    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
-    <br><button type="submit">Connect</button>
-  </form>
-</div></body></html>"""
-
-
-def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
-    """Generate a simple HTML page for the OAuth result."""
-    safe_title = html.escape(title)
-    safe_message = html.escape(message)
-    color = "#00661a" if success else "#e06c75"
-    icon = "&#10003;" if success else "&#10007;"
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>{safe_title}</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 420px; text-align: center; }}
-  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
-  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
-</style></head>
-<body><div class="card">
-  <div class="icon">{icon}</div>
-  <h2>{safe_title}</h2>
-  <p>{safe_message}</p>
-</div></body></html>"""
+_sys.modules[__name__] = _canonical

routes/memory/memory_routes.py

@@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str:
         return text
     return _LIST_PREFIX_RE.sub("", text, count=1).strip()
 
-from services.memory import MemoryManager
+from services.memory import MemoryManager, MemoryStoreUnreadable
 from core.session_manager import SessionManager
 from src.request_models import MemoryAddRequest
 from core.database import SessionLocal
@@ -35,6 +35,22 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
 logger = logging.getLogger(__name__)
 
 
+def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
+    """Load the whole store for a read-modify-write cycle.
+
+    A transient read failure must not look like an empty store: the caller
+    would append to ``[]`` and save that back, atomically destroying every
+    existing memory (issue #5673). Surface it as a 503 and change nothing.
+    """
+    try:
+        return memory_manager.load_all_for_update()
+    except MemoryStoreUnreadable as e:
+        logger.error("Refusing to rewrite the memory store: %s", e)
+        raise HTTPException(
+            503, "Memory store is temporarily unreadable — no changes were made."
+        )
+
+
 def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
     """Set up memory-related routes."""
     router = APIRouter(prefix="/api/memory", tags=["memory"])
@@ -116,7 +132,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
         new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
         if memory_data.session_id:
             new_entry["session_id"] = memory_data.session_id
-        all_mem = memory_manager.load_all()
+        all_mem = _load_for_update(memory_manager)
         all_mem.append(new_entry)
         memory_manager.save(all_mem)
         # Sync vector index
@@ -487,7 +503,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
         """Pin or unpin a memory. Pinned memories are always included in context."""
         user = _owner(request)
-        all_mem = memory_manager.load_all()
+        all_mem = _load_for_update(memory_manager)
         for i, memory in enumerate(all_mem):
             if memory["id"] == memory_id:
                 _verify_memory_owner(memory, user)
@@ -512,7 +528,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
         """Update an existing memory item with new text and optional category."""
         user = _owner(request)
-        all_mem = memory_manager.load_all()
+        all_mem = _load_for_update(memory_manager)
         for i, memory in enumerate(all_mem):
             if memory["id"] == memory_id:
                 _verify_memory_owner(memory, user)
@@ -534,7 +550,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM
     def delete_memory(request: Request, memory_id: str):
         """Delete a memory item by its ID."""
         user = _owner(request)
-        all_mem = memory_manager.load_all()
+        all_mem = _load_for_update(memory_manager)
 
         # Find and verify ownership before deleting
         target = next((m for m in all_mem if m["id"] == memory_id), None)

routes/model_routes.py

@@ -46,10 +46,12 @@ _ENDPOINT_SETTING_FIELDS = {
 }
 
 _ENDPOINT_FALLBACK_FIELDS = {
-    "default_model_fallbacks": "Default Model Fallbacks",
+    "foreground_model_fallbacks": "Foreground Model Fallbacks",
     "utility_model_fallbacks": "Utility Model Fallbacks",
     "vision_model_fallbacks":  "Vision Model Fallbacks",
 }
+# `default_model_fallbacks` is intentionally absent. The legacy data remains
+# stored as-is even when an endpoint is removed, but no longer affects routing.
 
 
 def _speech_settings_using_endpoint(settings: dict, ep_id: str) -> list:
@@ -179,7 +181,12 @@ def _clear_user_pref_endpoint_refs(all_prefs: dict, ep_id: str) -> int:
     if not isinstance(all_prefs, dict):
         return 0
     users = all_prefs.get("_users")
-    pref_sets = users.values() if isinstance(users, dict) else [all_prefs]
+    # A mixed store can contain auth-disabled foreground policy at the root
+    # alongside named-owner preferences. Both are active namespaces; legacy
+    # `default_model_fallbacks` remains untouched by the field allowlist.
+    pref_sets = [all_prefs]
+    if isinstance(users, dict):
+        pref_sets.extend(users.values())
     cleared_users = 0
     for prefs in pref_sets:
         if isinstance(prefs, dict) and _clear_endpoint_settings_for_endpoint(prefs, ep_id):
@@ -472,7 +479,11 @@ def _endpoint_kind(ep: Any) -> str:
 
 
 def _endpoint_refresh_mode(ep: Any, endpoint_kind: str | None = None) -> str:
-    return _normalize_refresh_mode(getattr(ep, "model_refresh_mode", None), endpoint_kind or _endpoint_kind(ep))
+    return _normalize_endpoint_refresh_mode(
+        getattr(ep, "model_refresh_mode", None),
+        endpoint_kind or _endpoint_kind(ep),
+        getattr(ep, "base_url", ""),
+    )
 
 
 def _endpoint_refresh_interval(ep: Any, category: str) -> float:
@@ -851,6 +862,99 @@ def _ollama_model_names(data: Any) -> List[str]:
     return out
 
 
+def _is_google_api_base(base_url: str) -> bool:
+    try:
+        return (urlparse(base_url).hostname or "").lower() == "generativelanguage.googleapis.com"
+    except Exception:
+        return False
+
+
+def _normalize_endpoint_refresh_mode(value: Any, endpoint_kind: str = "auto", base_url: str = "") -> str:
+    if not str(value or "").strip() and _is_google_api_base(base_url):
+        return "manual"
+    return _normalize_refresh_mode(value, endpoint_kind)
+
+
+def _google_native_root(base_url: str) -> str:
+    """Return the Gemini native API root for a Google endpoint.
+
+    Chat calls may be configured against Google's OpenAI-compatible
+    `/openai` path, but model catalog reads should use the native Models API
+    so we get Google's current Model resource shape.
+    """
+    try:
+        parsed = urlparse(base_url)
+    except Exception:
+        return "https://generativelanguage.googleapis.com/v1beta"
+    path = (parsed.path or "").rstrip("/")
+    if path.endswith("/openai"):
+        path = path[: -len("/openai")].rstrip("/")
+    if not path:
+        path = "/v1beta"
+    return urlunparse(parsed._replace(path=path, query="", fragment="")).rstrip("/")
+
+
+def _google_native_models_url(base_url: str) -> str:
+    return _google_native_root(base_url) + "/models"
+
+
+def _google_model_id_from_item(item: Any) -> str:
+    if not isinstance(item, dict):
+        return ""
+    value = item.get("baseModelId") or item.get("name") or item.get("model") or ""
+    return str(value or "").strip().removeprefix("models/")
+
+
+def _google_model_supports_chat(item: Any) -> bool:
+    """Return whether a native Google Model resource supports chat generation."""
+    if not isinstance(item, dict):
+        return False
+    methods = item.get("supportedGenerationMethods")
+    if not isinstance(methods, list):
+        return False
+    chat_methods = {"generateContent", "generateMessage", "generateText", "generateAnswer"}
+    return any(method in chat_methods for method in methods)
+
+
+def _probe_google_models(base_url: str, api_key: str = None, timeout: int = 5, page_size: int = 1000) -> List[str]:
+    """Read Google's native paginated Models API.
+
+    This intentionally returns only provider-reported model IDs. Capability
+    mapping is handled by the model capability reader and must not infer from
+    names here.
+    """
+    url = _google_native_models_url(base_url)
+    try:
+        page_size = min(max(int(page_size or 1000), 1), 1000)
+    except Exception:
+        page_size = 1000
+    headers = {"Accept": "application/json"}
+    if api_key:
+        headers["x-goog-api-key"] = api_key
+    params: Dict[str, Any] = {"pageSize": page_size}
+    models: List[str] = []
+    seen = set()
+    page_token = ""
+    for _ in range(20):
+        request_params = dict(params)
+        if page_token:
+            request_params["pageToken"] = page_token
+        r = httpx.get(url, headers=headers, params=request_params, timeout=timeout, verify=llm_verify())
+        r.raise_for_status()
+        data = r.json()
+        for item in data.get("models") or []:
+            if not _google_model_supports_chat(item):
+                continue
+            model_id = _google_model_id_from_item(item)
+            if model_id and model_id not in seen:
+                seen.add(model_id)
+                models.append(model_id)
+        page_token = str(data.get("nextPageToken") or "").strip()
+        if not page_token:
+            break
+    return models
+
+
 def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> List[str]:
     """Probe a base URL's /models endpoint and return list of model IDs.
     For Anthropic, queries their /v1/models API, falling back to hardcoded list."""
@@ -863,6 +967,17 @@ def _probe_endpoint(base_url: str, api_key: str = None, timeout: int = 5) -> Lis
         if api_key:
             return fetch_available_models(api_key, timeout=timeout)
         return []
+    if _is_google_api_base(base):
+        try:
+            models = _probe_google_models(base, api_key, timeout=timeout)
+            if models:
+                return models
+        except httpx.HTTPStatusError as e:
+            status = e.response.status_code if e.response is not None else "unknown"
+            logger.warning(f"Google native models probe failed: HTTP {status}")
+        except Exception as e:
+            logger.warning(f"Google native models probe failed: {e}")
+        return []
     if provider == "anthropic":
         # Try Anthropic's /v1/models endpoint first
         url = _safe_build_models_url(base)
@@ -1202,6 +1317,56 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
     return [m for m in merged if m not in hidden]
 
 
+def _picker_requires_pinning(base_url: str, kind: str) -> bool:
+    return _classify_endpoint(base_url, kind) == "api"
+
+
+def _has_explicit_pinned_models(ep) -> bool:
+    """Whether pinned_models was deliberately written for this endpoint.
+
+    API endpoints use pinned_models as an allow-list. An explicit empty JSON
+    list means "show no models"; it must not fall back to the old hidden-list
+    migration behavior.
+    """
+    raw = getattr(ep, "pinned_models", None)
+    return raw is not None and str(raw).strip() != ""
+
+
+def _legacy_visible_api_models(ep) -> List[str]:
+    """Return API models selected under the old hidden-list picker.
+
+    Before API endpoints switched to an explicit allow-list, selected models
+    were represented as cached_models minus hidden_models. Existing OpenRouter
+    rows can therefore have many checked models and an empty pinned_models
+    field. Treat that old state as the initial pinned list so settings and chat
+    agree after upgrade.
+    """
+    return _visible_models(
+        _cached_model_ids(ep),
+        getattr(ep, "hidden_models", None),
+        None,
+    )
+
+
+def _picker_models_for_endpoint(ep, base_url: str, kind: str):
+    """Return model IDs that should appear in the picker for an endpoint.
+
+    API providers expose remote inventory from /v1/models. Default to that
+    visible inventory until an explicit pinned-model allow-list is saved.
+    Local/self-hosted endpoints keep the older hide-list behavior.
+    """
+    pinned = _normalize_model_ids(getattr(ep, "pinned_models", None))
+    if _picker_requires_pinning(base_url, kind):
+        if not _has_explicit_pinned_models(ep):
+            pinned = _legacy_visible_api_models(ep)
+        return pinned, pinned
+    return _visible_models(
+        _cached_model_ids(ep),
+        getattr(ep, "hidden_models", None),
+        pinned,
+    ), pinned
+
+
 def _api_key_fingerprint(api_key: Optional[str]) -> str:
     """Stable, non-secret label for distinguishing same-URL credentials."""
     key = (api_key or "").strip()
@@ -1400,24 +1565,18 @@ def setup_model_routes(model_discovery):
         for ep in endpoints:
             base = _normalize_base(ep.base_url)
             provider = _safe_detect_provider(base)
-            # Merge cached + pinned models, then filter out hidden ones
             ep_model_type = getattr(ep, "model_type", None) or "llm"
-            model_ids = _visible_models(
-                _cached_model_ids(ep),
-                ep.hidden_models,
-                getattr(ep, "pinned_models", None),
-            )
             # Build correct URL based on provider
             chat_url = build_chat_url(base)
             kind = _effective_endpoint_kind(ep, base)
             category = _classify_endpoint(base, kind)
+            model_ids, pinned = _picker_models_for_endpoint(ep, base, kind)
 
             if model_ids:
                 curated_key = _match_provider_curated(base, None)
                 curated, extra = _curate_models(model_ids, curated_key)
                 # Pinned models are admin-selected — they always belong in the
                 # primary curated list, not buried in extras.
-                pinned = _normalize_model_ids(getattr(ep, "pinned_models", None))
                 for m in pinned:
                     if m not in curated:
                         curated.append(m)
@@ -1779,18 +1938,24 @@ def setup_model_routes(model_discovery):
                 _invalidate_models_cache()
             rows = db.query(ModelEndpoint).order_by(ModelEndpoint.created_at).all()
             results = []
+            upgraded_legacy_pins = False
             for r in rows:
                 all_models = _cached_model_ids(r)
                 hidden = _hidden_model_ids(r)
                 pinned = _normalize_model_ids(getattr(r, "pinned_models", None))
-                visible = _visible_models(all_models, r.hidden_models, pinned)
                 # Keep the list route cache-only. It feeds Settings →
                 # Added Models and must render immediately; explicit
                 # Refresh/Probe endpoints do the network work.
-                status = "online" if (all_models or pinned) else ("empty" if r.is_enabled else "offline")
                 ping = None
                 base = _normalize_base(r.base_url)
                 kind = _effective_endpoint_kind(r, base)
+                visible, pinned = _picker_models_for_endpoint(r, base, kind)
+                if _picker_requires_pinning(base, kind) and pinned and not _has_explicit_pinned_models(r):
+                    r.pinned_models = json.dumps(pinned)
+                    upgraded_legacy_pins = True
+                model_inventory_count = len(_merge_model_ids(all_models, pinned))
+                picker_requires_pinning = _picker_requires_pinning(base, kind)
+                status = "online" if (all_models or visible or pinned) else ("empty" if r.is_enabled else "offline")
                 results.append({
                     "id": r.id,
                     "name": r.name,
@@ -1799,6 +1964,8 @@ def setup_model_routes(model_discovery):
                     "api_key_fingerprint": _api_key_fingerprint(r.api_key),
                     "is_enabled": r.is_enabled,
                     "models": visible,
+                    "model_count": model_inventory_count,
+                    "picker_requires_pinning": picker_requires_pinning,
                     "pinned_models": pinned,
                     "hidden_count": len(hidden),
                     "online": status != "offline",
@@ -1812,6 +1979,9 @@ def setup_model_routes(model_discovery):
                     "model_refresh_interval": getattr(r, "model_refresh_interval", None),
                     "model_refresh_timeout": getattr(r, "model_refresh_timeout", None),
                 })
+            if upgraded_legacy_pins:
+                db.commit()
+                _invalidate_models_cache()
             return results
         finally:
             db.close()
@@ -1854,7 +2024,7 @@ def setup_model_routes(model_discovery):
             name = base_url.replace("http://", "").replace("https://", "").split("/")[0]
 
         requested_kind = _normalize_endpoint_kind(endpoint_kind)
-        refresh_mode = _normalize_refresh_mode(model_refresh_mode, requested_kind)
+        refresh_mode = _normalize_endpoint_refresh_mode(model_refresh_mode, requested_kind, base_url)
         refresh_interval = _parse_positive_int(model_refresh_interval, minimum=30, maximum=86400)
         refresh_timeout = _parse_positive_int(model_refresh_timeout, minimum=1, maximum=60)
         require_model_list = _truthy(require_models)
@@ -1915,6 +2085,10 @@ def setup_model_routes(model_discovery):
                 if refresh_timeout is not None:
                     existing.model_refresh_timeout = refresh_timeout
                     changed = True
+                incoming_model_type = (model_type or "").strip() or "llm"
+                if incoming_model_type and (getattr(existing, "model_type", None) or "llm") != incoming_model_type:
+                    existing.model_type = incoming_model_type
+                    changed = True
                 if api_key.strip() and not existing.api_key:
                     existing.api_key = api_key.strip()
                     changed = True
@@ -2147,9 +2321,10 @@ def setup_model_routes(model_discovery):
                 raise HTTPException(404, "Endpoint not found")
             hidden = _hidden_model_ids(ep)
             all_models = _cached_model_ids(ep)
+            base = _normalize_base(ep.base_url)
+            kind = _effective_endpoint_kind(ep, base)
+            picker_requires_pinning = _picker_requires_pinning(base, kind)
             if refresh:
-                base = _normalize_base(ep.base_url)
-                kind = _effective_endpoint_kind(ep, base)
                 category = _classify_endpoint(base, kind)
                 timeout = _manual_refresh_timeout(ep, category, refresh_timeout)
                 try:
@@ -2167,7 +2342,7 @@ def setup_model_routes(model_discovery):
                 else:
                     response.headers["X-Model-Refresh-Status"] = "failed"
                     response.headers["X-Model-Refresh-Warning"] = "Model refresh failed or returned no models; kept cached models."
-            pinned = _normalize_model_ids(getattr(ep, "pinned_models", None))
+            _, pinned = _picker_models_for_endpoint(ep, base, kind)
             pinned_set = set(pinned)
             return [
                 {
@@ -2175,6 +2350,7 @@ def setup_model_routes(model_discovery):
                     "display": m.split("/")[-1],
                     "is_hidden": m in hidden,
                     "is_pinned": m in pinned_set,
+                    "picker_requires_pinning": picker_requires_pinning,
                 }
                 for m in _merge_model_ids(all_models, pinned)
             ]
@@ -2203,11 +2379,28 @@ def setup_model_routes(model_discovery):
                 hidden = body.get("hidden")
                 if not isinstance(hidden, list):
                     raise HTTPException(400, "hidden must be a list of model IDs")
-                ep.hidden_models = json.dumps(hidden) if hidden else None
+                base = _normalize_base(ep.base_url)
+                kind = _effective_endpoint_kind(ep, base)
+                if _picker_requires_pinning(base, kind):
+                    # Compatibility for older/admin UI paths that still submit
+                    # the previous hide-list shape. API pickers are allow-lists:
+                    # convert "unchecked models" into an explicit pinned list so
+                    # Settings summary, /api/models, and chat agree.
+                    selected = _visible_models(_cached_model_ids(ep), hidden, None)
+                    ep.pinned_models = json.dumps(selected)
+                    ep.hidden_models = None
+                else:
+                    ep.hidden_models = json.dumps(hidden) if hidden else None
             # Accept either "pinned" or "pinned_models" for the manual IDs list.
             if "pinned_models" in body or "pinned" in body:
                 pinned = _normalize_model_ids(body.get("pinned_models", body.get("pinned")))
-                ep.pinned_models = json.dumps(pinned) if pinned else None
+                base = _normalize_base(ep.base_url)
+                kind = _effective_endpoint_kind(ep, base)
+                if _picker_requires_pinning(base, kind):
+                    ep.pinned_models = json.dumps(pinned)
+                    ep.hidden_models = None
+                else:
+                    ep.pinned_models = json.dumps(pinned) if pinned else None
             db.commit()
             _invalidate_models_cache()
             hidden_count = len(json.loads(ep.hidden_models)) if ep.hidden_models else 0
@@ -2249,7 +2442,6 @@ def setup_model_routes(model_discovery):
             _user_prefs = _load_for_user(_user) or {}
             ep_id = (_user_prefs.get("default_endpoint_id") or "").strip()
             model = (_user_prefs.get("default_model") or "").strip()
-            _fallbacks = _user_prefs.get("default_model_fallbacks") or []
             # If user has no personal default, fall back to global default
             # But only based on the "share_defaults_with_users" flag
             # (only if share_defaults_with_users is enabled)
@@ -2258,12 +2450,9 @@ def setup_model_routes(model_discovery):
                     ep_id = settings.get("default_endpoint_id", "")
                 if not model:
                     model = settings.get("default_model", "")
-                if not _fallbacks:
-                    _fallbacks = settings.get("default_model_fallbacks") or []
         else:
             ep_id = settings.get("default_endpoint_id", "")
             model = settings.get("default_model", "")
-            _fallbacks = settings.get("default_model_fallbacks") or []
         db = SessionLocal()
         try:
             ep = None
@@ -2278,33 +2467,6 @@ def setup_model_routes(model_discovery):
                 if _user and not _is_admin:
                     ep_q = owner_filter(ep_q, ModelEndpoint, _user)
                 ep = ep_q.first()
-            # Configured fallback chain — when the chosen default endpoint is
-            # gone/disabled, honor the user's configured `default_model_fallbacks`
-            # in order BEFORE arbitrarily grabbing the first enabled endpoint.
-            # (Previously this jumped straight to "first enabled", which is why
-            # deleting/changing the main endpoint silently reassigned the default
-            # chat to some unrelated endpoint instead of the fallback.)
-            if not ep:
-                for entry in _fallbacks:
-                    if not isinstance(entry, dict):
-                        continue
-                    fid = (entry.get("endpoint_id") or "").strip()
-                    if not fid:
-                        continue
-                    cand_q = db.query(ModelEndpoint).filter(
-                        ModelEndpoint.id == fid, ModelEndpoint.is_enabled == True
-                    )
-                    if _user and not _is_admin:
-                        cand_q = owner_filter(cand_q, ModelEndpoint, _user)
-                    cand = cand_q.first()
-                    if cand:
-                        ep = cand
-                        # Use the fallback entry's model. Reset even when empty
-                        # so we don't carry the prior endpoint's stale model onto
-                        # this fallback — the cached-models lookup below then
-                        # fills it from the fallback endpoint.
-                        model = (entry.get("model") or "").strip()
-                        break
             # Last resort: first enabled endpoint owned by THIS user. Do not
             # include null-owner/shared endpoints here: a brand-new user with
             # no explicit default should not auto-open a pending chat using an
@@ -2360,11 +2522,21 @@ def setup_model_routes(model_discovery):
                     ep.model_type = body["model_type"].strip() or ep.model_type
                 if "pinned_models" in body:
                     _pinned = _normalize_model_ids(body["pinned_models"])
-                    ep.pinned_models = json.dumps(_pinned) if _pinned else None
+                    _base_for_pins = _normalize_base(ep.base_url)
+                    _kind_for_pins = _effective_endpoint_kind(ep, _base_for_pins)
+                    if _picker_requires_pinning(_base_for_pins, _kind_for_pins):
+                        ep.pinned_models = json.dumps(_pinned)
+                        ep.hidden_models = None
+                    else:
+                        ep.pinned_models = json.dumps(_pinned) if _pinned else None
                 if "endpoint_kind" in body:
                     ep.endpoint_kind = _normalize_endpoint_kind(body.get("endpoint_kind"))
                 if "model_refresh_mode" in body:
-                    ep.model_refresh_mode = _normalize_refresh_mode(body.get("model_refresh_mode"), _endpoint_kind(ep))
+                    ep.model_refresh_mode = _normalize_endpoint_refresh_mode(
+                        body.get("model_refresh_mode"),
+                        _endpoint_kind(ep),
+                        ep.base_url,
+                    )
                 if "model_refresh_interval" in body:
                     interval = _parse_positive_int(body.get("model_refresh_interval"), minimum=30, maximum=86400)
                     ep.model_refresh_interval = interval

routes/note/__init__.py

@@ -0,0 +1,5 @@
+"""Note route domain package (slice 2f, #4082/#4071).
+
+Contains note_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/note_routes.py re-exports from here.
+"""

routes/note/note_routes.py

@@ -0,0 +1,937 @@
+# routes/note_routes.py
+"""Google Keep-style notes / checklists API."""
+
+import json
+import uuid
+import logging
+from typing import Dict, Any, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from core.database import SessionLocal, Note
+from core.middleware import INTERNAL_TOOL_USER
+from src.auth_helpers import require_user
+from src.constants import DATA_DIR
+from src.upload_handler import reserve_upload_references
+from sqlalchemy.orm.attributes import flag_modified
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Request models
+# ---------------------------------------------------------------------------
+
+class NoteCreate(BaseModel):
+    title: str = ""
+    content: Optional[str] = None
+    items: Optional[list] = None
+    note_type: str = "note"
+    color: Optional[str] = None
+    label: Optional[str] = None
+    pinned: bool = False
+    due_date: Optional[str] = None
+    source: str = "user"
+    session_id: Optional[str] = None
+    image_url: Optional[str] = None
+    repeat: Optional[str] = "none"
+    sort_order: Optional[int] = None
+
+
+class NoteUpdate(BaseModel):
+    title: Optional[str] = None
+    content: Optional[str] = None
+    items: Optional[list] = None
+    note_type: Optional[str] = None
+    color: Optional[str] = None
+    label: Optional[str] = None
+    pinned: Optional[bool] = None
+    archived: Optional[bool] = None
+    due_date: Optional[str] = None
+    image_url: Optional[str] = None
+    repeat: Optional[str] = None
+    sort_order: Optional[int] = None
+    agent_session_id: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _note_to_dict(note: Note) -> Dict[str, Any]:
+    items = None
+    if note.items:
+        try:
+            items = json.loads(note.items)
+        except (json.JSONDecodeError, TypeError):
+            items = None
+    ai_cls = None
+    raw_ai = getattr(note, "ai_classification", None)
+    if raw_ai:
+        try:
+            ai_cls = json.loads(raw_ai)
+        except (json.JSONDecodeError, TypeError):
+            ai_cls = None
+    return {
+        "id": note.id,
+        "owner": note.owner,
+        "title": note.title,
+        "content": note.content,
+        "items": items,
+        "note_type": note.note_type,
+        "color": note.color,
+        "label": note.label,
+        "pinned": note.pinned,
+        "archived": note.archived,
+        "due_date": note.due_date,
+        "source": note.source,
+        "session_id": note.session_id,
+        "sort_order": note.sort_order or 0,
+        "image_url": note.image_url,
+        "repeat": note.repeat or "none",
+        "ai_classification": ai_cls,
+        "ai_content_hash": getattr(note, "ai_content_hash", None),
+        "agent_session_id": getattr(note, "agent_session_id", None),
+        "created_at": note.created_at.isoformat() if note.created_at else None,
+        "updated_at": note.updated_at.isoformat() if note.updated_at else None,
+    }
+
+
+def _reminder_text_from_note(note: Note) -> tuple[str, str]:
+    """Return the reminder title/body from a stored note row."""
+    title = (note.title or "Note reminder").strip() or "Note reminder"
+    if note.items:
+        try:
+            items = json.loads(note.items)
+        except (json.JSONDecodeError, TypeError):
+            items = None
+        if isinstance(items, list):
+            pending: list[str] = []
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                if item.get("done") or item.get("checked"):
+                    continue
+                text = str(item.get("text") or "").strip()
+                if text:
+                    pending.append(text)
+            if pending:
+                shown = "\n".join(f"- {text}" for text in pending[:8])
+                extra = f"\n...and {len(pending) - 8} more" if len(pending) > 8 else ""
+                return title, f"Pending ({len(pending)}):\n{shown}{extra}"
+            return title, f"{len(items)} item{'s' if len(items) != 1 else ''}"
+    return title, (note.content or "").strip()[:400]
+
+
+
+# ---------------------------------------------------------------------------
+# Reminder dispatch — module-level so background tasks (built-in actions)
+# can call it directly without an HTTP roundtrip + auth cookie. The route
+# version below is a thin wrapper that pulls `owner` from the request.
+# ---------------------------------------------------------------------------
+
+# Scheduler reference — set by setup_note_routes() so dispatch_reminder can
+# push a parallel in-app notification (frontend polls the scheduler's queue
+# and fires real browser Notification(...) popups). Optional; works without it.
+_scheduler_ref = None
+
+
+async def dispatch_reminder(
+    title: str,
+    note_body: str,
+    note_id: str,
+    owner: str = "",
+    queue_browser: bool = True,
+    settings_override: dict | None = None,
+) -> dict:
+    """Fire a reminder via the configured channel (browser/email/ntfy/webhook).
+
+    Args:
+        title: short headline shown to the user
+        note_body: longer body text
+        note_id: stable id (used as tag/dedupe in browser notifications)
+        owner: the user this reminder belongs to — scopes SMTP config to
+               their account so we don't cross-leak credentials
+
+    Returns: {synthesis, email_sent, ntfy_sent}. Browser channel is wired via
+    the in-memory notification queue picked up by the frontend poller, so
+    nothing is "sent" synchronously for it — the channel just routes there.
+    """
+    from src.settings import load_settings
+    settings = {**load_settings(), **(settings_override or {})}
+    channel = settings.get("reminder_channel", "browser")
+    llm_on = bool(settings.get("reminder_llm_synthesis", False))
+    title = (title or "").strip()
+    note_body = (note_body or "").strip()
+    cache_key = str(note_id) if note_id else ""
+    cache = {}
+    cache_path = None
+    if cache_key:
+        try:
+            import json as _json
+            from datetime import datetime as _dt, timezone as _tz, timedelta as _td
+            from pathlib import Path as _P
+            _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
+            cache_path = _P(DATA_DIR) / f"note_pings_{_slug}.json"
+            if cache_path.exists():
+                cache = _json.loads(cache_path.read_text(encoding="utf-8"))
+            last = cache.get(cache_key)
+            if last:
+                last_channel = None
+                if isinstance(last, dict):
+                    last_channel = last.get("channel")
+                    last = last.get("at")
+                last_dt = _dt.fromisoformat(str(last))
+                if last_dt.tzinfo is None:
+                    last_dt = last_dt.replace(tzinfo=_tz.utc)
+                # Legacy cache values were plain timestamps and could be
+                # written by the frontend even when the email/ntfy send failed.
+                # Treat those as browser-only dedupe so email reminders can be
+                # retried by the backend scanner after a failed frontend path.
+                should_skip = last_dt >= _dt.now(_tz.utc) - _td(minutes=25)
+                if should_skip and channel in ("email", "ntfy", "webhook"):
+                    should_skip = last_channel == channel
+                if should_skip:
+                    return {
+                        "synthesis": None,
+                        "email_sent": False,
+                        "ntfy_sent": False,
+                        "webhook_sent": False,
+                        "browser_sent": True,
+                        "skipped": True,
+                    }
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: cache read failed: {_e}")
+
+    synthesis = None
+    _SYNTH_FAILED_TAG = "[utility model unavailable — no summary generated]"
+    if llm_on:
+        try:
+            from src.endpoint_resolver import resolve_endpoint
+            from src.llm_core import llm_call_async
+            from src.reminder_personas import synthesis_system_prompt
+            url, model, headers = resolve_endpoint("utility", owner=owner or None)
+            if not url:
+                url, model, headers = resolve_endpoint("default", owner=owner or None)
+            if url and model:
+                persona_id = (settings.get("reminder_llm_persona") or "").strip()
+                sys_prompt = synthesis_system_prompt(persona_id)
+                raw = await llm_call_async(
+                    url=url, model=model,
+                    messages=[
+                        {"role": "system", "content": sys_prompt},
+                        {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()},
+                    ],
+                    temperature=0.7, max_tokens=200, headers=headers, timeout=30,
+                )
+                from src.text_helpers import strip_think as _strip_think
+                # prose=True strips untagged "The user wants me to…" chain-of-thought.
+                # prompt_echo=True strips Qwen-style "Thinking Process:" / leaked
+                # prompt prefixes. Both are safe here because this is a
+                # one-sentence LLM-only output, not user-pasted content.
+                synthesis = _strip_think(raw or "", prose=True, prompt_echo=True)
+                # Reminder synthesis is supposed to be ONE sentence. Strip-think's
+                # paragraph-based heuristic misses cases where the model puts
+                # reasoning + answer on consecutive lines inside one paragraph
+                # (e.g. "I should write... [\n] You have one task waiting...").
+                # Walk lines, drop reasoning/prompt-echo lines, then keep the
+                # last surviving line — that's the actual warm sentence.
+                if synthesis:
+                    import re as _re
+                    # Tightened: target ACTUAL self-talk (model narrating what
+                    # it'll do) rather than any first-person sentence. The old
+                    # pattern killed legit warm sentences like "I'll see you
+                    # tomorrow" or "I should be done by then". New rules:
+                    #  • "I (need|should|have|'ll|will) (write|draft|reply|…)"
+                    #    only matches when followed by a TASK verb taking an
+                    #    OBJECT (so first-person + intransitive verb passes).
+                    #  • Self-instructional patterns the model emits verbatim:
+                    #    "I should write something that reminds them…",
+                    #    "I need to draft…", "Let me think…".
+                    #  • Explicit instructions echoed back from the prompt:
+                    #    "Keep it under 25 words", "No greetings".
+                    _reasoning = _re.compile(
+                        r"^\s*(?:"
+                        # "I should write/draft/compose…" with a task-object follow
+                        r"i (?:need|should|have|'ll|will|am going|am)\s+to\s+"
+                        r"(?:write|draft|compose|craft|generate|produce|create|"
+                        r"summarize|answer|provide|note|address|remind|output)"
+                        r"\s+(?:a |an |the |something|this|that|here|them|him|her|"
+                        r"you|user|reply|response|sentence|message|line|warm)|"
+                        # The model literally narrating about the user
+                        r"the user (?:wants|is asking|asks|needs|wrote|said|requested) (?:me )?(?:to|for|that|about|something)|"
+                        # "Let me [think/write/draft/…] (about/for/the …)"
+                        r"let me (?:think|write|draft|consider|note|see|check)\b\s+(?:about|for|the|this|that|if|whether)|"
+                        # "Looking at the/this/that …"
+                        r"looking at (?:the|this|that)\b|"
+                        # "Based on the/this/what …"
+                        r"based on (?:the|this|what|context|that)\b|"
+                        # Prompt-echo of length / style instructions
+                        r"keep it under \d+ words\b|"
+                        r"(?:no greetings|no preamble|no hashtags|just output the)\b"
+                        r").*",
+                        _re.IGNORECASE,
+                    )
+                    # Echo of the prompt's "Pending:" / "<N> pending" tail.
+                    _echo = _re.compile(
+                        r"^\s*(?:pending\s*[:.]|(?:\d+|one|two|three|four|five)\s+pending\b)",
+                        _re.IGNORECASE,
+                    )
+                    lines = [ln for ln in synthesis.splitlines() if ln.strip()]
+                    cleaned = [ln for ln in lines if not _reasoning.match(ln) and not _echo.match(ln)]
+                    if cleaned:
+                        # The model's actual answer is normally the LAST surviving
+                        # line — reasoning leads, answer trails.
+                        synthesis = cleaned[-1].strip()
+            else:
+                synthesis = _SYNTH_FAILED_TAG
+        except Exception as e:
+            logger.warning(f"Reminder LLM synthesis failed: {e}")
+            synthesis = _SYNTH_FAILED_TAG
+        if synthesis:
+            _s = synthesis.strip(); _low = _s.lower()
+            if (not _s or _low.startswith("error:") or _low.startswith("[error")
+                    or "operation failed" in _low
+                    or ("upstream" in _low and "failed" in _low)) and synthesis != _SYNTH_FAILED_TAG:
+                logger.warning(f"Reminder synthesis looked like an error, replacing: {_s[:120]!r}")
+                synthesis = _SYNTH_FAILED_TAG
+
+    email_sent = False
+    email_error = ""
+    if channel == "email":
+        try:
+            from routes.email_routes import _get_email_config, _smtp_ready
+            from email.mime.text import MIMEText
+            from email.mime.multipart import MIMEMultipart
+            from datetime import datetime as _dt
+            # `reminder_email_account_id` lets the user pick WHICH email
+            # account to send reminders from (when they have several
+            # configured in Integrations). Falls back to the default
+            # account when no explicit choice is saved.
+            _acc_id = (settings.get("reminder_email_account_id") or "").strip() or None
+            cfg = _get_email_config(account_id=_acc_id, owner=owner or "")
+            if not _smtp_ready(cfg):
+                try:
+                    from core.database import SessionLocal as _SL, EmailAccount as _EA
+                    from sqlalchemy import and_, or_
+                    db = _SL()
+                    try:
+                        q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
+                        if owner:
+                            unowned = or_(_EA.owner == None, _EA.owner == "")  # noqa: E711
+                            same_mailbox = or_(_EA.imap_user == owner, _EA.from_address == owner)
+                            q = q.filter(or_(_EA.owner == owner, and_(unowned, same_mailbox)))
+                        for row in q.order_by(_EA.is_default.desc(), _EA.created_at.asc()).all():
+                            trial = _get_email_config(account_id=row.id, owner=owner or "")
+                            if _smtp_ready(trial):
+                                cfg = trial
+                                break
+                    finally:
+                        db.close()
+                except Exception as _fallback_error:
+                    logger.debug(f"Reminder SMTP fallback lookup failed: {_fallback_error}")
+            from_addr = (cfg.get("from_address") or cfg.get("smtp_user") or "").strip()
+            recipient = (settings.get("reminder_email_to") or "").strip() or from_addr
+            # Loud diagnostic so we can see WHY a reminder didn't send (the
+            # previous "silently no-op when cfg has no smtp_host" was invisible).
+            logger.info(
+                "dispatch_reminder[email] note_id=%s owner=%r "
+                "has_smtp_host=%s has_smtp_user=%s has_from=%s has_recipient=%s",
+                note_id, owner,
+                bool(cfg.get("smtp_host")), bool(cfg.get("smtp_user")),
+                bool(from_addr), bool(recipient),
+            )
+            missing = []
+            if not cfg.get("smtp_host"):
+                missing.append("SMTP host")
+            if not cfg.get("smtp_user"):
+                missing.append("SMTP user")
+            if not (cfg.get("smtp_password") or cfg.get("oauth_provider")):
+                missing.append("SMTP credentials")
+            if not from_addr:
+                missing.append("from address")
+            if not recipient:
+                missing.append("recipient")
+            if missing:
+                email_error = "Missing " + ", ".join(missing)
+                logger.warning(
+                    "Reminder email not sent for note_id=%s account=%r: %s",
+                    note_id, cfg.get("account_name"), email_error,
+                )
+            else:
+                msg = MIMEMultipart("alternative")
+                msg["From"] = from_addr
+                msg["To"] = recipient
+                _t = title or 'Note'
+                _t = _t[len('Reminder:'):].strip() if _t.lower().startswith('reminder:') else _t
+                msg["Subject"] = f"Reminder (Odysseus): {_t}"
+                msg["Date"] = _dt.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+                msg["X-Odysseus-Origin"] = "odysseus-ui"
+                msg["X-Odysseus-Kind"] = "reminder"
+                msg["X-Odysseus-Ref"] = str(note_id)
+                # Body shape: synthesis (warm sentence) → blank line → bold
+                # title header → note details. The title was previously only
+                # in the subject line, so the email read like a faceless
+                # to-do list with no anchor to which note triggered it.
+                _body_chunks = []
+                if synthesis:
+                    _body_chunks.append(synthesis)
+                if _t:
+                    _body_chunks.append(_t)
+                if note_body:
+                    _body_chunks.append(note_body)
+                plain = "\n\n".join(_body_chunks) if _body_chunks else title
+                msg.attach(MIMEText(plain, "plain", "utf-8"))
+
+                def _smtp_send():
+                    from routes.email_helpers import _send_smtp_message
+                    _send_smtp_message(cfg, from_addr, [recipient], msg.as_string())
+
+                import asyncio as _aio
+                await _aio.to_thread(_smtp_send)
+                email_sent = True
+        except Exception as e:
+            email_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder email send failed: {e}")
+
+    webhook_sent = False
+    webhook_error = ""
+    if channel == "webhook":
+        try:
+            import httpx
+            import json as _wjson
+            from src.integrations import load_integrations
+            # Built-in payload defaults for known presets so users don't have
+            # to configure a template just to use a standard service.
+            _PRESET_TEMPLATE_DEFAULTS = {
+                "discord_webhook": '{"embeds": [{"title": "{{title}}", "description": "{{message}}", "color": 5793266}]}',
+            }
+            intg_id = settings.get("reminder_webhook_integration_id", "").strip()
+            template = settings.get("reminder_webhook_payload_template", "").strip()
+            if not intg_id:
+                webhook_error = "No webhook integration selected"
+            else:
+                intg = next(
+                    (i for i in load_integrations()
+                     if i.get("id") == intg_id and i.get("base_url")),
+                    None,
+                )
+                if not intg:
+                    webhook_error = f"Integration {intg_id!r} not found or missing base URL"
+                else:
+                    # Fall back to a built-in default for known presets so
+                    # users don't have to configure a template for standard
+                    # services like Discord.
+                    if not template:
+                        template = _PRESET_TEMPLATE_DEFAULTS.get(intg.get("preset", ""), "")
+                    if not template:
+                        webhook_error = "No payload template configured"
+                    else:
+                        # Render template: JSON-escape the values so the result
+                        # is always valid JSON regardless of special characters.
+                        # dumps() returns `"value"` — strip outer quotes.
+                        msg = (synthesis or note_body or title or "Reminder")[:4000]
+                        _t = _wjson.dumps(title or "Reminder")[1:-1]
+                        _m = _wjson.dumps(msg)[1:-1]
+                        rendered = template.replace("{{title}}", _t).replace("{{message}}", _m)
+                        hdrs = {"Content-Type": "application/json"}
+                        api_key = intg.get("api_key", "")
+                        auth_type = (intg.get("auth_type") or "none").lower()
+                        if api_key:
+                            if auth_type == "bearer":
+                                hdrs["Authorization"] = f"Bearer {api_key}"
+                            elif auth_type == "header":
+                                hdrs[intg.get("auth_header") or "Authorization"] = api_key
+                        url = intg["base_url"].rstrip("/")
+                        # SSRF guard — matches the pattern used by webhook_routes,
+                        # CalDAV, search, and embeddings. Blocks link-local / metadata
+                        # addresses (169.254.x.x) by default; set
+                        # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true to also block
+                        # RFC-1918 ranges for locked-down deployments.
+                        import os as _os
+                        from src.url_safety import check_outbound_url as _chk
+                        _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
+                        _ok, _reason = _chk(url, block_private=_block)
+                        if not _ok:
+                            webhook_error = f"Webhook URL rejected: {_reason}"
+                        else:
+                            async with httpx.AsyncClient(timeout=10.0) as client:
+                                resp = await client.post(url, content=rendered.encode(), headers=hdrs)
+                                webhook_sent = resp.is_success
+                                if not webhook_sent:
+                                    webhook_error = f"Webhook returned HTTP {resp.status_code}"
+        except Exception as e:
+            webhook_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder webhook send failed: {e}")
+
+    ntfy_sent = False
+    ntfy_error = ""
+    if channel == "ntfy":
+        try:
+            from src.integrations import load_integrations
+            import httpx
+            intg = next(
+                (i for i in load_integrations()
+                 if i.get("preset") == "ntfy" and i.get("enabled", True) and i.get("base_url")),
+                None,
+            )
+            if intg:
+                base = intg["base_url"].rstrip("/")
+                topic = settings.get("reminder_ntfy_topic") or "reminders"
+                ntfy_body = synthesis or note_body or title
+                # ntfy Title is an ASCII HTTP header; sanitize Unicode and cap its length.
+                _clean_title = (title or "Reminder").encode("ascii", "replace").decode("ascii")[:200]
+                hdrs = {"Title": _clean_title, "Priority": "high", "Tags": "bell"}
+                api_key = intg.get("api_key", "")
+                if api_key:
+                    hdrs["Authorization"] = f"Bearer {api_key}"
+                # SSRF guard — same check (and env knob) as the webhook branch
+                # above: link-local / metadata addresses are always rejected;
+                # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
+                # so a ntfy base_url can't be pointed at internal services.
+                import os as _os
+                from src.url_safety import check_outbound_url as _chk
+                _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
+                _ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
+                if not _ok:
+                    ntfy_error = f"ntfy URL rejected: {_reason}"
+                else:
+                    async with httpx.AsyncClient(timeout=10.0) as client:
+                        resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
+                        ntfy_sent = resp.is_success
+                        if not ntfy_sent:
+                            ntfy_error = f"ntfy returned HTTP {resp.status_code}"
+            else:
+                ntfy_error = "No enabled ntfy integration"
+        except Exception as e:
+            ntfy_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder ntfy send failed: {e}")
+
+    # In-app browser notification ALWAYS fires (regardless of channel). The
+    # frontend polls `/api/tasks/notifications` and turns any entry with a
+    # `body` into a real `Notification(...)` — same surface as task-success
+    # popups. Lets the user see reminders inside the app even when the
+    # primary channel is email/ntfy and the tab is open.
+    browser_sent = False
+    local_browser_sent = (not queue_browser and channel == "browser")
+    if queue_browser and _scheduler_ref is not None:
+        try:
+            _scheduler_ref.add_notification(
+                task_name=title or "Reminder",
+                status="success",
+                task_id=f"reminder-{note_id}",
+                owner=owner or None,
+                body=(synthesis or note_body or title or "").strip()[:500] or "Reminder",
+            )
+            browser_sent = True
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: in-app notif push failed: {_e}")
+
+    # Dedupe across paths: write to the same cache file `action_ping_notes`
+    # reads, so the background scanner's REPING_MIN window suppresses a
+    # second send for the same note within 25 min. Without this, a note
+    # whose due_date fires while the user has the app open got TWO emails
+    # (frontend-fired here + background-fired by ping_notes 0–5 min later).
+    if (email_sent or ntfy_sent or webhook_sent or browser_sent or local_browser_sent) and note_id:
+        try:
+            import json as _json
+            from datetime import datetime as _dt, timezone as _tz
+            from pathlib import Path as _P
+            # Per-owner cache so the scanner's prune step on user A's run
+            # doesn't drop user B's just-fired entry (review C4).
+            _STATE = cache_path
+            if _STATE is None:
+                _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
+                _STATE = _P(DATA_DIR) / f"note_pings_{_slug}.json"
+            _STATE.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                _cache = cache or (_json.loads(_STATE.read_text(encoding="utf-8")) if _STATE.exists() else {})
+            except Exception:
+                _cache = {}
+            sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "webhook" if webhook_sent else "browser"
+            _cache[cache_key or str(note_id)] = {
+                "at": _dt.now(_tz.utc).isoformat(),
+                "channel": sent_channel,
+            }
+            _STATE.write_text(_json.dumps(_cache), encoding="utf-8")
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: cache write failed: {_e}")
+
+    return {
+        "channel": channel,
+        "synthesis": synthesis,
+        "email_sent": email_sent,
+        "email_error": email_error,
+        "ntfy_sent": ntfy_sent,
+        "ntfy_error": ntfy_error,
+        "webhook_sent": webhook_sent,
+        "webhook_error": webhook_error,
+        "browser_sent": browser_sent or local_browser_sent,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Router factory
+# ---------------------------------------------------------------------------
+
+def setup_note_routes(task_scheduler=None, upload_handler=None):
+    # Expose the scheduler to module-level `dispatch_reminder` so reminders
+    # can also push to the in-app notification queue (the polling system
+    # turns each entry into a real browser Notification + the existing
+    # tasks-tab badge / dot system).
+    global _scheduler_ref
+    _scheduler_ref = task_scheduler
+
+    router = APIRouter(prefix="/api/notes", tags=["notes"])
+
+    def _owner(request: Request) -> Optional[str]:
+        # require_user, not bare get_current_user: a request that reaches
+        # these owner-scoped routes with NO identity (auth-middleware
+        # regression, SSRF from a sibling service) must fail closed (401)
+        # when auth is configured — not be treated as the single-user mode
+        # and handed blanket access to every account's notes. The documented
+        # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback,
+        # unconfigured first-run) still resolve to None, the single-user
+        # path. fire_reminder below already gated this way; the CRUD routes
+        # did not.
+        return require_user(request) or None
+
+    def _reserve_note_uploads(owner: Optional[str], *values) -> None:
+        missing_id = reserve_upload_references(upload_handler, owner, *values)
+        if missing_id:
+            raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
+
+    def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
+        if user == INTERNAL_TOOL_USER:
+            return True
+        if not user:
+            # require_user() already admitted this request, which only happens
+            # for auth-disabled, loopback-bypass, or unconfigured single-user
+            # modes. There is no separate non-admin account boundary there.
+            return True
+        try:
+            from core.auth import AuthManager
+            auth_mgr = getattr(request.app.state, "auth_manager", None) or AuthManager()
+            if not getattr(auth_mgr, "is_configured", True):
+                return True
+            return bool(auth_mgr.is_admin(user))
+        except Exception:
+            return False
+
+    # --- LIST ---
+    @router.get("")
+    def list_notes(
+        request: Request,
+        archived: Optional[bool] = None,
+        label: Optional[str] = None,
+    ):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            q = db.query(Note)
+            if user is not None:
+                q = q.filter(Note.owner == user)
+            if archived is not None:
+                q = q.filter(Note.archived == archived)
+            else:
+                q = q.filter(Note.archived == False)
+            if label:
+                q = q.filter(Note.label == label)
+            # Archived view: most recently archived first. Active view: pin + manual order.
+            if archived is True:
+                notes = q.order_by(Note.updated_at.desc()).all()
+            else:
+                notes = q.order_by(Note.pinned.desc(), Note.sort_order.asc(), Note.updated_at.desc()).all()
+            return {"notes": [_note_to_dict(n) for n in notes]}
+        finally:
+            db.close()
+
+    # --- CREATE ---
+    @router.post("")
+    def create_note(request: Request, body: NoteCreate):
+        user = _owner(request)
+        _reserve_note_uploads(
+            user,
+            body.image_url,
+            body.color,
+            body.content,
+            json.dumps(body.items) if body.items is not None else None,
+        )
+        db = SessionLocal()
+        try:
+            note = Note(
+                id=str(uuid.uuid4()),
+                owner=user,
+                title=body.title,
+                content=body.content,
+                items=json.dumps(body.items) if body.items is not None else None,
+                note_type=body.note_type,
+                color=body.color,
+                label=body.label,
+                pinned=body.pinned,
+                due_date=body.due_date,
+                source=body.source,
+                session_id=body.session_id,
+                image_url=body.image_url,
+                repeat=body.repeat or "none",
+                sort_order=body.sort_order if body.sort_order is not None else 0,
+            )
+            db.add(note)
+            db.commit()
+            db.refresh(note)
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- GET ONE ---
+    @router.get("/{note_id}")
+    def get_note(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- UPDATE ---
+    @router.put("/{note_id}")
+    def update_note(request: Request, note_id: str, body: NoteUpdate):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+
+            _reserve_note_uploads(
+                user,
+                body.image_url,
+                body.color,
+                body.content,
+                json.dumps(body.items) if body.items is not None else None,
+            )
+            if body.title is not None:
+                note.title = body.title
+            if body.content is not None:
+                note.content = body.content
+            if body.items is not None:
+                note.items = json.dumps(body.items)
+                flag_modified(note, "items")
+            if body.note_type is not None:
+                note.note_type = body.note_type
+            if body.color is not None:
+                note.color = body.color
+            if body.label is not None:
+                note.label = body.label
+            if body.pinned is not None:
+                note.pinned = body.pinned
+            if body.archived is not None:
+                note.archived = body.archived
+            if body.due_date is not None:
+                note.due_date = body.due_date
+            if body.image_url is not None:
+                note.image_url = body.image_url
+            if body.repeat is not None:
+                note.repeat = body.repeat
+            if body.sort_order is not None:
+                note.sort_order = body.sort_order
+            if body.agent_session_id is not None:
+                note.agent_session_id = body.agent_session_id
+
+            db.commit()
+            db.refresh(note)
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- DELETE ---
+    @router.delete("/{note_id}")
+    def delete_note(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            db.delete(note)
+            db.commit()
+            return {"ok": True}
+        finally:
+            db.close()
+
+    # --- TOGGLE PIN ---
+    @router.post("/{note_id}/pin")
+    def toggle_pin(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            note.pinned = not note.pinned
+            db.commit()
+            return {"ok": True, "pinned": note.pinned}
+        finally:
+            db.close()
+
+    # --- TOGGLE ARCHIVE ---
+    @router.post("/{note_id}/archive")
+    def toggle_archive(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            note.archived = not note.archived
+            db.commit()
+            return {"ok": True, "archived": note.archived}
+        finally:
+            db.close()
+
+    # --- TOGGLE CHECKLIST ITEM ---
+    @router.post("/{note_id}/items/{index}/toggle")
+    def toggle_item(request: Request, note_id: str, index: int):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            if not note.items:
+                raise HTTPException(400, "Note has no checklist items")
+            items = json.loads(note.items)
+            if index < 0 or index >= len(items):
+                raise HTTPException(400, f"Item index {index} out of range")
+            items[index]["done"] = not items[index].get("done", False)
+            note.items = json.dumps(items)
+            flag_modified(note, "items")
+            db.commit()
+            return {"ok": True, "items": items}
+        finally:
+            db.close()
+
+    # --- FIRE REMINDER ---
+    @router.post("/fire-reminder")
+    async def fire_reminder(request: Request):
+        """Dispatch a reminder according to user settings.
+
+        Called by the frontend when a reminder fires. Optionally generates an
+        LLM synthesis line and/or sends an email through configured SMTP.
+        Returns {synthesis, email_sent}.
+        """
+        # Gate against anonymous callers — LLM synthesis can burn tokens.
+        user = require_user(request)
+        body = await request.json()
+        note_id = str(body.get("note_id") or "").strip()
+        if not note_id:
+            raise HTTPException(400, "note_id required")
+
+        caller = _owner(request)
+        is_test = note_id.startswith("test-")
+        is_admin = _is_admin_or_single_user(request, user or caller)
+        _override: dict = {}
+        if is_test:
+            if not is_admin:
+                raise HTTPException(403, "Admin only")
+            title = (body.get("title") or "Test Reminder").strip() or "Test Reminder"
+            note_body = (body.get("body") or "").strip()
+            # Optional overrides let the admin settings test button pass the
+            # current UI values directly so it never races a pending save.
+            if body.get("channel"):
+                _override["reminder_channel"] = body["channel"]
+            if body.get("webhook_integration_id"):
+                _override["reminder_webhook_integration_id"] = body["webhook_integration_id"]
+            if body.get("webhook_payload_template"):
+                _override["reminder_webhook_payload_template"] = body["webhook_payload_template"]
+            # Mirror the in-UI AI Synthesis toggle + persona so the test
+            # actually exercises the synthesis path before/without a Save.
+            if "llm_synthesis" in body:
+                _override["reminder_llm_synthesis"] = bool(body["llm_synthesis"])
+            if "llm_persona" in body:
+                _override["reminder_llm_persona"] = str(body["llm_persona"] or "")
+        else:
+            db = SessionLocal()
+            try:
+                note = db.query(Note).filter(Note.id == note_id).first()
+                if not note:
+                    raise HTTPException(404, "Note not found")
+                if caller is not None and note.owner != caller:
+                    raise HTTPException(404, "Note not found")
+                title, note_body = _reminder_text_from_note(note)
+            finally:
+                db.close()
+
+        return await dispatch_reminder(
+            title=title, note_body=note_body, note_id=note_id,
+            owner=caller or "",
+            queue_browser=False,
+            settings_override=_override or None,
+        )
+
+    # --- REORDER NOTES ---
+    @router.post("/reorder")
+    async def reorder_notes(request: Request):
+        """Update sort_order for a list of note IDs in the order provided."""
+        user = _owner(request)
+        body = await request.json()
+        ids = body.get("ids", [])
+        if not isinstance(ids, list):
+            raise HTTPException(400, "ids must be a list")
+        # v2 review HIGH-12: drop the legacy `(owner == user) | (owner ==
+        # None)` OR which let an authenticated user silently reorder
+        # every legacy-null-owner note belonging to other accounts. In
+        # an unconfigured (single-user) auth deploy the OR is still safe
+        # because there's no second user to attack; we keep that branch
+        # explicit and gated on AuthManager.is_configured.
+        try:
+            from core.auth import AuthManager
+            _allow_null = not AuthManager().is_configured
+        except Exception:
+            _allow_null = False
+        db = SessionLocal()
+        try:
+            for i, nid in enumerate(ids):
+                q = db.query(Note).filter(Note.id == nid)
+                if user is not None:
+                    if _allow_null:
+                        q = q.filter((Note.owner == user) | (Note.owner == None))  # noqa: E711
+                    else:
+                        q = q.filter(Note.owner == user)
+                note = q.first()
+                if note:
+                    note.sort_order = i
+            db.commit()
+            return {"ok": True, "count": len(ids)}
+        finally:
+            db.close()
+
+    return router

routes/note_routes.py

@@ -1,915 +1,18 @@
-# routes/note_routes.py
-"""Google Keep-style notes / checklists API."""
+"""Backward-compat shim — canonical location is routes/note/note_routes.py.
 
-import json
-import uuid
-import logging
-from typing import Dict, Any, Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.note_routes``, ``from routes.note_routes import X``,
+``importlib.import_module("routes.note_routes")``, and the
+``import ... as note_routes`` + ``monkeypatch.setattr(note_routes, "SessionLocal",
+...)`` pattern used by test_note_reminder_fire_scope.py /
+test_notes_fail_closed_auth.py all operate on the *same* object the
+application actually uses. Keeps existing import paths working after
+slice 2f (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Request
-from pydantic import BaseModel
+import sys as _sys
 
-from core.database import SessionLocal, Note
-from core.middleware import INTERNAL_TOOL_USER
-from src.auth_helpers import require_user
-from src.constants import DATA_DIR
-from sqlalchemy.orm.attributes import flag_modified
+from routes.note import note_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Request models
-# ---------------------------------------------------------------------------
-
-class NoteCreate(BaseModel):
-    title: str = ""
-    content: Optional[str] = None
-    items: Optional[list] = None
-    note_type: str = "note"
-    color: Optional[str] = None
-    label: Optional[str] = None
-    pinned: bool = False
-    due_date: Optional[str] = None
-    source: str = "user"
-    session_id: Optional[str] = None
-    image_url: Optional[str] = None
-    repeat: Optional[str] = "none"
-    sort_order: Optional[int] = None
-
-
-class NoteUpdate(BaseModel):
-    title: Optional[str] = None
-    content: Optional[str] = None
-    items: Optional[list] = None
-    note_type: Optional[str] = None
-    color: Optional[str] = None
-    label: Optional[str] = None
-    pinned: Optional[bool] = None
-    archived: Optional[bool] = None
-    due_date: Optional[str] = None
-    image_url: Optional[str] = None
-    repeat: Optional[str] = None
-    sort_order: Optional[int] = None
-    agent_session_id: Optional[str] = None
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _note_to_dict(note: Note) -> Dict[str, Any]:
-    items = None
-    if note.items:
-        try:
-            items = json.loads(note.items)
-        except (json.JSONDecodeError, TypeError):
-            items = None
-    ai_cls = None
-    raw_ai = getattr(note, "ai_classification", None)
-    if raw_ai:
-        try:
-            ai_cls = json.loads(raw_ai)
-        except (json.JSONDecodeError, TypeError):
-            ai_cls = None
-    return {
-        "id": note.id,
-        "owner": note.owner,
-        "title": note.title,
-        "content": note.content,
-        "items": items,
-        "note_type": note.note_type,
-        "color": note.color,
-        "label": note.label,
-        "pinned": note.pinned,
-        "archived": note.archived,
-        "due_date": note.due_date,
-        "source": note.source,
-        "session_id": note.session_id,
-        "sort_order": note.sort_order or 0,
-        "image_url": note.image_url,
-        "repeat": note.repeat or "none",
-        "ai_classification": ai_cls,
-        "ai_content_hash": getattr(note, "ai_content_hash", None),
-        "agent_session_id": getattr(note, "agent_session_id", None),
-        "created_at": note.created_at.isoformat() if note.created_at else None,
-        "updated_at": note.updated_at.isoformat() if note.updated_at else None,
-    }
-
-
-def _reminder_text_from_note(note: Note) -> tuple[str, str]:
-    """Return the reminder title/body from a stored note row."""
-    title = (note.title or "Note reminder").strip() or "Note reminder"
-    if note.items:
-        try:
-            items = json.loads(note.items)
-        except (json.JSONDecodeError, TypeError):
-            items = None
-        if isinstance(items, list):
-            pending: list[str] = []
-            for item in items:
-                if not isinstance(item, dict):
-                    continue
-                if item.get("done") or item.get("checked"):
-                    continue
-                text = str(item.get("text") or "").strip()
-                if text:
-                    pending.append(text)
-            if pending:
-                shown = "\n".join(f"- {text}" for text in pending[:8])
-                extra = f"\n...and {len(pending) - 8} more" if len(pending) > 8 else ""
-                return title, f"Pending ({len(pending)}):\n{shown}{extra}"
-            return title, f"{len(items)} item{'s' if len(items) != 1 else ''}"
-    return title, (note.content or "").strip()[:400]
-
-
-
-# ---------------------------------------------------------------------------
-# Reminder dispatch — module-level so background tasks (built-in actions)
-# can call it directly without an HTTP roundtrip + auth cookie. The route
-# version below is a thin wrapper that pulls `owner` from the request.
-# ---------------------------------------------------------------------------
-
-# Scheduler reference — set by setup_note_routes() so dispatch_reminder can
-# push a parallel in-app notification (frontend polls the scheduler's queue
-# and fires real browser Notification(...) popups). Optional; works without it.
-_scheduler_ref = None
-
-
-async def dispatch_reminder(
-    title: str,
-    note_body: str,
-    note_id: str,
-    owner: str = "",
-    queue_browser: bool = True,
-    settings_override: dict | None = None,
-) -> dict:
-    """Fire a reminder via the configured channel (browser/email/ntfy/webhook).
-
-    Args:
-        title: short headline shown to the user
-        note_body: longer body text
-        note_id: stable id (used as tag/dedupe in browser notifications)
-        owner: the user this reminder belongs to — scopes SMTP config to
-               their account so we don't cross-leak credentials
-
-    Returns: {synthesis, email_sent, ntfy_sent}. Browser channel is wired via
-    the in-memory notification queue picked up by the frontend poller, so
-    nothing is "sent" synchronously for it — the channel just routes there.
-    """
-    from src.settings import load_settings
-    settings = {**load_settings(), **(settings_override or {})}
-    channel = settings.get("reminder_channel", "browser")
-    llm_on = bool(settings.get("reminder_llm_synthesis", False))
-    title = (title or "").strip()
-    note_body = (note_body or "").strip()
-    cache_key = str(note_id) if note_id else ""
-    cache = {}
-    cache_path = None
-    if cache_key:
-        try:
-            import json as _json
-            from datetime import datetime as _dt, timezone as _tz, timedelta as _td
-            from pathlib import Path as _P
-            _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
-            cache_path = _P(DATA_DIR) / f"note_pings_{_slug}.json"
-            if cache_path.exists():
-                cache = _json.loads(cache_path.read_text(encoding="utf-8"))
-            last = cache.get(cache_key)
-            if last:
-                last_channel = None
-                if isinstance(last, dict):
-                    last_channel = last.get("channel")
-                    last = last.get("at")
-                last_dt = _dt.fromisoformat(str(last))
-                if last_dt.tzinfo is None:
-                    last_dt = last_dt.replace(tzinfo=_tz.utc)
-                # Legacy cache values were plain timestamps and could be
-                # written by the frontend even when the email/ntfy send failed.
-                # Treat those as browser-only dedupe so email reminders can be
-                # retried by the backend scanner after a failed frontend path.
-                should_skip = last_dt >= _dt.now(_tz.utc) - _td(minutes=25)
-                if should_skip and channel in ("email", "ntfy", "webhook"):
-                    should_skip = last_channel == channel
-                if should_skip:
-                    return {
-                        "synthesis": None,
-                        "email_sent": False,
-                        "ntfy_sent": False,
-                        "webhook_sent": False,
-                        "browser_sent": True,
-                        "skipped": True,
-                    }
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: cache read failed: {_e}")
-
-    synthesis = None
-    _SYNTH_FAILED_TAG = "[utility model unavailable — no summary generated]"
-    if llm_on:
-        try:
-            from src.endpoint_resolver import resolve_endpoint
-            from src.llm_core import llm_call_async
-            from src.reminder_personas import synthesis_system_prompt
-            url, model, headers = resolve_endpoint("utility", owner=owner or None)
-            if not url:
-                url, model, headers = resolve_endpoint("default", owner=owner or None)
-            if url and model:
-                persona_id = (settings.get("reminder_llm_persona") or "").strip()
-                sys_prompt = synthesis_system_prompt(persona_id)
-                raw = await llm_call_async(
-                    url=url, model=model,
-                    messages=[
-                        {"role": "system", "content": sys_prompt},
-                        {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()},
-                    ],
-                    temperature=0.7, max_tokens=200, headers=headers, timeout=30,
-                )
-                from src.text_helpers import strip_think as _strip_think
-                # prose=True strips untagged "The user wants me to…" chain-of-thought.
-                # prompt_echo=True strips Qwen-style "Thinking Process:" / leaked
-                # prompt prefixes. Both are safe here because this is a
-                # one-sentence LLM-only output, not user-pasted content.
-                synthesis = _strip_think(raw or "", prose=True, prompt_echo=True)
-                # Reminder synthesis is supposed to be ONE sentence. Strip-think's
-                # paragraph-based heuristic misses cases where the model puts
-                # reasoning + answer on consecutive lines inside one paragraph
-                # (e.g. "I should write... [\n] You have one task waiting...").
-                # Walk lines, drop reasoning/prompt-echo lines, then keep the
-                # last surviving line — that's the actual warm sentence.
-                if synthesis:
-                    import re as _re
-                    # Tightened: target ACTUAL self-talk (model narrating what
-                    # it'll do) rather than any first-person sentence. The old
-                    # pattern killed legit warm sentences like "I'll see you
-                    # tomorrow" or "I should be done by then". New rules:
-                    #  • "I (need|should|have|'ll|will) (write|draft|reply|…)"
-                    #    only matches when followed by a TASK verb taking an
-                    #    OBJECT (so first-person + intransitive verb passes).
-                    #  • Self-instructional patterns the model emits verbatim:
-                    #    "I should write something that reminds them…",
-                    #    "I need to draft…", "Let me think…".
-                    #  • Explicit instructions echoed back from the prompt:
-                    #    "Keep it under 25 words", "No greetings".
-                    _reasoning = _re.compile(
-                        r"^\s*(?:"
-                        # "I should write/draft/compose…" with a task-object follow
-                        r"i (?:need|should|have|'ll|will|am going|am)\s+to\s+"
-                        r"(?:write|draft|compose|craft|generate|produce|create|"
-                        r"summarize|answer|provide|note|address|remind|output)"
-                        r"\s+(?:a |an |the |something|this|that|here|them|him|her|"
-                        r"you|user|reply|response|sentence|message|line|warm)|"
-                        # The model literally narrating about the user
-                        r"the user (?:wants|is asking|asks|needs|wrote|said|requested) (?:me )?(?:to|for|that|about|something)|"
-                        # "Let me [think/write/draft/…] (about/for/the …)"
-                        r"let me (?:think|write|draft|consider|note|see|check)\b\s+(?:about|for|the|this|that|if|whether)|"
-                        # "Looking at the/this/that …"
-                        r"looking at (?:the|this|that)\b|"
-                        # "Based on the/this/what …"
-                        r"based on (?:the|this|what|context|that)\b|"
-                        # Prompt-echo of length / style instructions
-                        r"keep it under \d+ words\b|"
-                        r"(?:no greetings|no preamble|no hashtags|just output the)\b"
-                        r").*",
-                        _re.IGNORECASE,
-                    )
-                    # Echo of the prompt's "Pending:" / "<N> pending" tail.
-                    _echo = _re.compile(
-                        r"^\s*(?:pending\s*[:.]|(?:\d+|one|two|three|four|five)\s+pending\b)",
-                        _re.IGNORECASE,
-                    )
-                    lines = [ln for ln in synthesis.splitlines() if ln.strip()]
-                    cleaned = [ln for ln in lines if not _reasoning.match(ln) and not _echo.match(ln)]
-                    if cleaned:
-                        # The model's actual answer is normally the LAST surviving
-                        # line — reasoning leads, answer trails.
-                        synthesis = cleaned[-1].strip()
-            else:
-                synthesis = _SYNTH_FAILED_TAG
-        except Exception as e:
-            logger.warning(f"Reminder LLM synthesis failed: {e}")
-            synthesis = _SYNTH_FAILED_TAG
-        if synthesis:
-            _s = synthesis.strip(); _low = _s.lower()
-            if (not _s or _low.startswith("error:") or _low.startswith("[error")
-                    or "operation failed" in _low
-                    or ("upstream" in _low and "failed" in _low)) and synthesis != _SYNTH_FAILED_TAG:
-                logger.warning(f"Reminder synthesis looked like an error, replacing: {_s[:120]!r}")
-                synthesis = _SYNTH_FAILED_TAG
-
-    email_sent = False
-    email_error = ""
-    if channel == "email":
-        try:
-            from routes.email_routes import _get_email_config
-            from email.mime.text import MIMEText
-            from email.mime.multipart import MIMEMultipart
-            from datetime import datetime as _dt
-            # `reminder_email_account_id` lets the user pick WHICH email
-            # account to send reminders from (when they have several
-            # configured in Integrations). Falls back to the default
-            # account when no explicit choice is saved.
-            _acc_id = (settings.get("reminder_email_account_id") or "").strip() or None
-            cfg = _get_email_config(account_id=_acc_id, owner=owner or "")
-            if not (cfg.get("smtp_host") and cfg.get("smtp_user") and cfg.get("smtp_password")):
-                try:
-                    from core.database import SessionLocal as _SL, EmailAccount as _EA
-                    from sqlalchemy import and_, or_
-                    db = _SL()
-                    try:
-                        q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
-                        if owner:
-                            unowned = or_(_EA.owner == None, _EA.owner == "")  # noqa: E711
-                            same_mailbox = or_(_EA.imap_user == owner, _EA.from_address == owner)
-                            q = q.filter(or_(_EA.owner == owner, and_(unowned, same_mailbox)))
-                        for row in q.order_by(_EA.is_default.desc(), _EA.created_at.asc()).all():
-                            trial = _get_email_config(account_id=row.id, owner=owner or "")
-                            if trial.get("smtp_host") and trial.get("smtp_user") and trial.get("smtp_password"):
-                                cfg = trial
-                                break
-                    finally:
-                        db.close()
-                except Exception as _fallback_error:
-                    logger.debug(f"Reminder SMTP fallback lookup failed: {_fallback_error}")
-            from_addr = (cfg.get("from_address") or cfg.get("smtp_user") or "").strip()
-            recipient = (settings.get("reminder_email_to") or "").strip() or from_addr
-            # Loud diagnostic so we can see WHY a reminder didn't send (the
-            # previous "silently no-op when cfg has no smtp_host" was invisible).
-            logger.info(
-                "dispatch_reminder[email] note_id=%s owner=%r "
-                "has_smtp_host=%s has_smtp_user=%s has_from=%s has_recipient=%s",
-                note_id, owner,
-                bool(cfg.get("smtp_host")), bool(cfg.get("smtp_user")),
-                bool(from_addr), bool(recipient),
-            )
-            missing = []
-            if not cfg.get("smtp_host"):
-                missing.append("SMTP host")
-            if not cfg.get("smtp_user"):
-                missing.append("SMTP user")
-            if not cfg.get("smtp_password"):
-                missing.append("SMTP password")
-            if not from_addr:
-                missing.append("from address")
-            if not recipient:
-                missing.append("recipient")
-            if missing:
-                email_error = "Missing " + ", ".join(missing)
-                logger.warning(
-                    "Reminder email not sent for note_id=%s account=%r: %s",
-                    note_id, cfg.get("account_name"), email_error,
-                )
-            else:
-                msg = MIMEMultipart("alternative")
-                msg["From"] = from_addr
-                msg["To"] = recipient
-                _t = title or 'Note'
-                _t = _t[len('Reminder:'):].strip() if _t.lower().startswith('reminder:') else _t
-                msg["Subject"] = f"Reminder (Odysseus): {_t}"
-                msg["Date"] = _dt.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-                msg["X-Odysseus-Origin"] = "odysseus-ui"
-                msg["X-Odysseus-Kind"] = "reminder"
-                msg["X-Odysseus-Ref"] = str(note_id)
-                # Body shape: synthesis (warm sentence) → blank line → bold
-                # title header → note details. The title was previously only
-                # in the subject line, so the email read like a faceless
-                # to-do list with no anchor to which note triggered it.
-                _body_chunks = []
-                if synthesis:
-                    _body_chunks.append(synthesis)
-                if _t:
-                    _body_chunks.append(_t)
-                if note_body:
-                    _body_chunks.append(note_body)
-                plain = "\n\n".join(_body_chunks) if _body_chunks else title
-                msg.attach(MIMEText(plain, "plain", "utf-8"))
-
-                def _smtp_send():
-                    from routes.email_helpers import _send_smtp_message
-                    _send_smtp_message(cfg, from_addr, [recipient], msg.as_string())
-
-                import asyncio as _aio
-                await _aio.to_thread(_smtp_send)
-                email_sent = True
-        except Exception as e:
-            email_error = str(e) or e.__class__.__name__
-            logger.warning(f"Reminder email send failed: {e}")
-
-    webhook_sent = False
-    webhook_error = ""
-    if channel == "webhook":
-        try:
-            import httpx
-            import json as _wjson
-            from src.integrations import load_integrations
-            # Built-in payload defaults for known presets so users don't have
-            # to configure a template just to use a standard service.
-            _PRESET_TEMPLATE_DEFAULTS = {
-                "discord_webhook": '{"embeds": [{"title": "{{title}}", "description": "{{message}}", "color": 5793266}]}',
-            }
-            intg_id = settings.get("reminder_webhook_integration_id", "").strip()
-            template = settings.get("reminder_webhook_payload_template", "").strip()
-            if not intg_id:
-                webhook_error = "No webhook integration selected"
-            else:
-                intg = next(
-                    (i for i in load_integrations()
-                     if i.get("id") == intg_id and i.get("base_url")),
-                    None,
-                )
-                if not intg:
-                    webhook_error = f"Integration {intg_id!r} not found or missing base URL"
-                else:
-                    # Fall back to a built-in default for known presets so
-                    # users don't have to configure a template for standard
-                    # services like Discord.
-                    if not template:
-                        template = _PRESET_TEMPLATE_DEFAULTS.get(intg.get("preset", ""), "")
-                    if not template:
-                        webhook_error = "No payload template configured"
-                    else:
-                        # Render template: JSON-escape the values so the result
-                        # is always valid JSON regardless of special characters.
-                        # dumps() returns `"value"` — strip outer quotes.
-                        msg = (synthesis or note_body or title or "Reminder")[:4000]
-                        _t = _wjson.dumps(title or "Reminder")[1:-1]
-                        _m = _wjson.dumps(msg)[1:-1]
-                        rendered = template.replace("{{title}}", _t).replace("{{message}}", _m)
-                        hdrs = {"Content-Type": "application/json"}
-                        api_key = intg.get("api_key", "")
-                        auth_type = (intg.get("auth_type") or "none").lower()
-                        if api_key:
-                            if auth_type == "bearer":
-                                hdrs["Authorization"] = f"Bearer {api_key}"
-                            elif auth_type == "header":
-                                hdrs[intg.get("auth_header") or "Authorization"] = api_key
-                        url = intg["base_url"].rstrip("/")
-                        # SSRF guard — matches the pattern used by webhook_routes,
-                        # CalDAV, search, and embeddings. Blocks link-local / metadata
-                        # addresses (169.254.x.x) by default; set
-                        # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true to also block
-                        # RFC-1918 ranges for locked-down deployments.
-                        import os as _os
-                        from src.url_safety import check_outbound_url as _chk
-                        _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
-                        _ok, _reason = _chk(url, block_private=_block)
-                        if not _ok:
-                            webhook_error = f"Webhook URL rejected: {_reason}"
-                        else:
-                            async with httpx.AsyncClient(timeout=10.0) as client:
-                                resp = await client.post(url, content=rendered.encode(), headers=hdrs)
-                                webhook_sent = resp.is_success
-                                if not webhook_sent:
-                                    webhook_error = f"Webhook returned HTTP {resp.status_code}"
-        except Exception as e:
-            webhook_error = str(e) or e.__class__.__name__
-            logger.warning(f"Reminder webhook send failed: {e}")
-
-    ntfy_sent = False
-    ntfy_error = ""
-    if channel == "ntfy":
-        try:
-            from src.integrations import load_integrations
-            import httpx
-            intg = next(
-                (i for i in load_integrations()
-                 if i.get("preset") == "ntfy" and i.get("enabled", True) and i.get("base_url")),
-                None,
-            )
-            if intg:
-                base = intg["base_url"].rstrip("/")
-                topic = settings.get("reminder_ntfy_topic") or "reminders"
-                ntfy_body = synthesis or note_body or title
-                hdrs = {"Title": title or "Reminder", "Priority": "high", "Tags": "bell"}
-                api_key = intg.get("api_key", "")
-                if api_key:
-                    hdrs["Authorization"] = f"Bearer {api_key}"
-                # SSRF guard — same check (and env knob) as the webhook branch
-                # above: link-local / metadata addresses are always rejected;
-                # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
-                # so a ntfy base_url can't be pointed at internal services.
-                import os as _os
-                from src.url_safety import check_outbound_url as _chk
-                _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
-                _ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
-                if not _ok:
-                    ntfy_error = f"ntfy URL rejected: {_reason}"
-                else:
-                    async with httpx.AsyncClient(timeout=10.0) as client:
-                        resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
-                        ntfy_sent = resp.is_success
-                        if not ntfy_sent:
-                            ntfy_error = f"ntfy returned HTTP {resp.status_code}"
-            else:
-                ntfy_error = "No enabled ntfy integration"
-        except Exception as e:
-            ntfy_error = str(e) or e.__class__.__name__
-            logger.warning(f"Reminder ntfy send failed: {e}")
-
-    # In-app browser notification ALWAYS fires (regardless of channel). The
-    # frontend polls `/api/tasks/notifications` and turns any entry with a
-    # `body` into a real `Notification(...)` — same surface as task-success
-    # popups. Lets the user see reminders inside the app even when the
-    # primary channel is email/ntfy and the tab is open.
-    browser_sent = False
-    local_browser_sent = (not queue_browser and channel == "browser")
-    if queue_browser and _scheduler_ref is not None:
-        try:
-            _scheduler_ref.add_notification(
-                task_name=title or "Reminder",
-                status="success",
-                task_id=f"reminder-{note_id}",
-                owner=owner or None,
-                body=(synthesis or note_body or title or "").strip()[:500] or "Reminder",
-            )
-            browser_sent = True
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: in-app notif push failed: {_e}")
-
-    # Dedupe across paths: write to the same cache file `action_ping_notes`
-    # reads, so the background scanner's REPING_MIN window suppresses a
-    # second send for the same note within 25 min. Without this, a note
-    # whose due_date fires while the user has the app open got TWO emails
-    # (frontend-fired here + background-fired by ping_notes 0–5 min later).
-    if (email_sent or ntfy_sent or webhook_sent or browser_sent or local_browser_sent) and note_id:
-        try:
-            import json as _json
-            from datetime import datetime as _dt, timezone as _tz
-            from pathlib import Path as _P
-            # Per-owner cache so the scanner's prune step on user A's run
-            # doesn't drop user B's just-fired entry (review C4).
-            _STATE = cache_path
-            if _STATE is None:
-                _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
-                _STATE = _P(DATA_DIR) / f"note_pings_{_slug}.json"
-            _STATE.parent.mkdir(parents=True, exist_ok=True)
-            try:
-                _cache = cache or (_json.loads(_STATE.read_text(encoding="utf-8")) if _STATE.exists() else {})
-            except Exception:
-                _cache = {}
-            sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "webhook" if webhook_sent else "browser"
-            _cache[cache_key or str(note_id)] = {
-                "at": _dt.now(_tz.utc).isoformat(),
-                "channel": sent_channel,
-            }
-            _STATE.write_text(_json.dumps(_cache), encoding="utf-8")
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: cache write failed: {_e}")
-
-    return {
-        "channel": channel,
-        "synthesis": synthesis,
-        "email_sent": email_sent,
-        "email_error": email_error,
-        "ntfy_sent": ntfy_sent,
-        "ntfy_error": ntfy_error,
-        "webhook_sent": webhook_sent,
-        "webhook_error": webhook_error,
-        "browser_sent": browser_sent or local_browser_sent,
-    }
-
-
-# ---------------------------------------------------------------------------
-# Router factory
-# ---------------------------------------------------------------------------
-
-def setup_note_routes(task_scheduler=None):
-    # Expose the scheduler to module-level `dispatch_reminder` so reminders
-    # can also push to the in-app notification queue (the polling system
-    # turns each entry into a real browser Notification + the existing
-    # tasks-tab badge / dot system).
-    global _scheduler_ref
-    _scheduler_ref = task_scheduler
-
-    router = APIRouter(prefix="/api/notes", tags=["notes"])
-
-    def _owner(request: Request) -> Optional[str]:
-        # require_user, not bare get_current_user: a request that reaches
-        # these owner-scoped routes with NO identity (auth-middleware
-        # regression, SSRF from a sibling service) must fail closed (401)
-        # when auth is configured — not be treated as the single-user mode
-        # and handed blanket access to every account's notes. The documented
-        # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback,
-        # unconfigured first-run) still resolve to None, the single-user
-        # path. fire_reminder below already gated this way; the CRUD routes
-        # did not.
-        return require_user(request) or None
-
-    def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
-        if user == INTERNAL_TOOL_USER:
-            return True
-        if not user:
-            # require_user() already admitted this request, which only happens
-            # for auth-disabled, loopback-bypass, or unconfigured single-user
-            # modes. There is no separate non-admin account boundary there.
-            return True
-        try:
-            from core.auth import AuthManager
-            auth_mgr = getattr(request.app.state, "auth_manager", None) or AuthManager()
-            if not getattr(auth_mgr, "is_configured", True):
-                return True
-            return bool(auth_mgr.is_admin(user))
-        except Exception:
-            return False
-
-    # --- LIST ---
-    @router.get("")
-    def list_notes(
-        request: Request,
-        archived: Optional[bool] = None,
-        label: Optional[str] = None,
-    ):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            q = db.query(Note)
-            if user is not None:
-                q = q.filter(Note.owner == user)
-            if archived is not None:
-                q = q.filter(Note.archived == archived)
-            else:
-                q = q.filter(Note.archived == False)
-            if label:
-                q = q.filter(Note.label == label)
-            # Archived view: most recently archived first. Active view: pin + manual order.
-            if archived is True:
-                notes = q.order_by(Note.updated_at.desc()).all()
-            else:
-                notes = q.order_by(Note.pinned.desc(), Note.sort_order.asc(), Note.updated_at.desc()).all()
-            return {"notes": [_note_to_dict(n) for n in notes]}
-        finally:
-            db.close()
-
-    # --- CREATE ---
-    @router.post("")
-    def create_note(request: Request, body: NoteCreate):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = Note(
-                id=str(uuid.uuid4()),
-                owner=user,
-                title=body.title,
-                content=body.content,
-                items=json.dumps(body.items) if body.items is not None else None,
-                note_type=body.note_type,
-                color=body.color,
-                label=body.label,
-                pinned=body.pinned,
-                due_date=body.due_date,
-                source=body.source,
-                session_id=body.session_id,
-                image_url=body.image_url,
-                repeat=body.repeat or "none",
-                sort_order=body.sort_order if body.sort_order is not None else 0,
-            )
-            db.add(note)
-            db.commit()
-            db.refresh(note)
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- GET ONE ---
-    @router.get("/{note_id}")
-    def get_note(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- UPDATE ---
-    @router.put("/{note_id}")
-    def update_note(request: Request, note_id: str, body: NoteUpdate):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-
-            if body.title is not None:
-                note.title = body.title
-            if body.content is not None:
-                note.content = body.content
-            if body.items is not None:
-                note.items = json.dumps(body.items)
-                flag_modified(note, "items")
-            if body.note_type is not None:
-                note.note_type = body.note_type
-            if body.color is not None:
-                note.color = body.color
-            if body.label is not None:
-                note.label = body.label
-            if body.pinned is not None:
-                note.pinned = body.pinned
-            if body.archived is not None:
-                note.archived = body.archived
-            if body.due_date is not None:
-                note.due_date = body.due_date
-            if body.image_url is not None:
-                note.image_url = body.image_url
-            if body.repeat is not None:
-                note.repeat = body.repeat
-            if body.sort_order is not None:
-                note.sort_order = body.sort_order
-            if body.agent_session_id is not None:
-                note.agent_session_id = body.agent_session_id
-
-            db.commit()
-            db.refresh(note)
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- DELETE ---
-    @router.delete("/{note_id}")
-    def delete_note(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            db.delete(note)
-            db.commit()
-            return {"ok": True}
-        finally:
-            db.close()
-
-    # --- TOGGLE PIN ---
-    @router.post("/{note_id}/pin")
-    def toggle_pin(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            note.pinned = not note.pinned
-            db.commit()
-            return {"ok": True, "pinned": note.pinned}
-        finally:
-            db.close()
-
-    # --- TOGGLE ARCHIVE ---
-    @router.post("/{note_id}/archive")
-    def toggle_archive(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            note.archived = not note.archived
-            db.commit()
-            return {"ok": True, "archived": note.archived}
-        finally:
-            db.close()
-
-    # --- TOGGLE CHECKLIST ITEM ---
-    @router.post("/{note_id}/items/{index}/toggle")
-    def toggle_item(request: Request, note_id: str, index: int):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            if not note.items:
-                raise HTTPException(400, "Note has no checklist items")
-            items = json.loads(note.items)
-            if index < 0 or index >= len(items):
-                raise HTTPException(400, f"Item index {index} out of range")
-            items[index]["done"] = not items[index].get("done", False)
-            note.items = json.dumps(items)
-            flag_modified(note, "items")
-            db.commit()
-            return {"ok": True, "items": items}
-        finally:
-            db.close()
-
-    # --- FIRE REMINDER ---
-    @router.post("/fire-reminder")
-    async def fire_reminder(request: Request):
-        """Dispatch a reminder according to user settings.
-
-        Called by the frontend when a reminder fires. Optionally generates an
-        LLM synthesis line and/or sends an email through configured SMTP.
-        Returns {synthesis, email_sent}.
-        """
-        # Gate against anonymous callers — LLM synthesis can burn tokens.
-        user = require_user(request)
-        body = await request.json()
-        note_id = str(body.get("note_id") or "").strip()
-        if not note_id:
-            raise HTTPException(400, "note_id required")
-
-        caller = _owner(request)
-        is_test = note_id.startswith("test-")
-        is_admin = _is_admin_or_single_user(request, user or caller)
-        _override: dict = {}
-        if is_test:
-            if not is_admin:
-                raise HTTPException(403, "Admin only")
-            title = (body.get("title") or "Test Reminder").strip() or "Test Reminder"
-            note_body = (body.get("body") or "").strip()
-            # Optional overrides let the admin settings test button pass the
-            # current UI values directly so it never races a pending save.
-            if body.get("channel"):
-                _override["reminder_channel"] = body["channel"]
-            if body.get("webhook_integration_id"):
-                _override["reminder_webhook_integration_id"] = body["webhook_integration_id"]
-            if body.get("webhook_payload_template"):
-                _override["reminder_webhook_payload_template"] = body["webhook_payload_template"]
-            # Mirror the in-UI AI Synthesis toggle + persona so the test
-            # actually exercises the synthesis path before/without a Save.
-            if "llm_synthesis" in body:
-                _override["reminder_llm_synthesis"] = bool(body["llm_synthesis"])
-            if "llm_persona" in body:
-                _override["reminder_llm_persona"] = str(body["llm_persona"] or "")
-        else:
-            db = SessionLocal()
-            try:
-                note = db.query(Note).filter(Note.id == note_id).first()
-                if not note:
-                    raise HTTPException(404, "Note not found")
-                if caller is not None and note.owner != caller:
-                    raise HTTPException(404, "Note not found")
-                title, note_body = _reminder_text_from_note(note)
-            finally:
-                db.close()
-
-        return await dispatch_reminder(
-            title=title, note_body=note_body, note_id=note_id,
-            owner=caller or "",
-            queue_browser=False,
-            settings_override=_override or None,
-        )
-
-    # --- REORDER NOTES ---
-    @router.post("/reorder")
-    async def reorder_notes(request: Request):
-        """Update sort_order for a list of note IDs in the order provided."""
-        user = _owner(request)
-        body = await request.json()
-        ids = body.get("ids", [])
-        if not isinstance(ids, list):
-            raise HTTPException(400, "ids must be a list")
-        # v2 review HIGH-12: drop the legacy `(owner == user) | (owner ==
-        # None)` OR which let an authenticated user silently reorder
-        # every legacy-null-owner note belonging to other accounts. In
-        # an unconfigured (single-user) auth deploy the OR is still safe
-        # because there's no second user to attack; we keep that branch
-        # explicit and gated on AuthManager.is_configured.
-        try:
-            from core.auth import AuthManager
-            _allow_null = not AuthManager().is_configured
-        except Exception:
-            _allow_null = False
-        db = SessionLocal()
-        try:
-            for i, nid in enumerate(ids):
-                q = db.query(Note).filter(Note.id == nid)
-                if user is not None:
-                    if _allow_null:
-                        q = q.filter((Note.owner == user) | (Note.owner == None))  # noqa: E711
-                    else:
-                        q = q.filter(Note.owner == user)
-                note = q.first()
-                if note:
-                    note.sort_order = i
-            db.commit()
-            return {"ok": True, "count": len(ids)}
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/personal_routes.py

@@ -1,11 +1,13 @@
 # routes/personal_routes.py
 """Routes for personal documents management."""
+import asyncio
 import os
 import logging
 import shutil
 import uuid
 from typing import Any, Dict, List, Tuple
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends
+from fastapi.concurrency import run_in_threadpool
 from src.request_models import DirectoryRequest
 from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR
 from src.rag_singleton import get_rag_manager
@@ -18,7 +20,6 @@ UPLOADS_DIR = PERSONAL_UPLOADS_DIR
 
 logger = logging.getLogger(__name__)
 
-
 def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str:
     """Return the per-owner upload directory used for direct RAG uploads."""
     owner_segment = secure_filename((owner or "local").strip())[:80] or "local"
@@ -141,6 +142,22 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     """
     router = APIRouter(prefix="/api/personal")
 
+    # Serializes directory index jobs across requests. Indexing runs in the
+    # threadpool (#5558), so concurrent requests would otherwise run in parallel
+    # and race PersonalDocsManager's unsynchronized list mutations and file
+    # writes; before the threadpool move they serialized on the blocked event
+    # loop, so one-at-a-time is behavior parity.
+    #
+    # An asyncio.Lock acquired in the async handler BEFORE offloading: a waiting
+    # request parks on the event loop instead of pinning a threadpool worker (an
+    # earlier threading.Lock taken INSIDE the worker meant queued jobs held pool
+    # tokens while blocked, starving every other run_in_threadpool caller).
+    # add/remove/reload all take this lock, so their mutations never interleave.
+    # Per-router (not module-global) so each app binds it to its own event loop.
+    # Scope is the single process: multi-worker deployments would need a shared
+    # lock (out of scope for #5558).
+    _index_job_lock = asyncio.Lock()
+
     def _rag():
         """Get the current RAG manager, retrying init if needed."""
         return get_rag_manager()
@@ -172,8 +189,12 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         return {"files": files, "directories": directories}
     
     @router.post("/reload")
-    def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
-        personal_docs_manager.refresh_index()
+    async def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
+        # refresh_index() re-extracts text across every tracked directory —
+        # blocking work. Take the shared job lock (so it cannot race an add /
+        # remove) and run it off the event loop.
+        async with _index_job_lock:
+            await run_in_threadpool(personal_docs_manager.refresh_index)
         return {"ok": True, "count": len(personal_docs_manager.index)}
     
     @router.post("/add_directory")
@@ -207,12 +228,26 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             # Use the RAGManager to index the directory
             rag = _rag()
             if rag:
-                result = rag.index_personal_documents(directory, owner=owner)
-                
+                def _index_directory():
+                    result = rag.index_personal_documents(directory, owner=owner)
+                    if result["success"]:
+                        # Also update the personal_docs_manager to track this
+                        # directory. Kept inside the offloaded call: it triggers
+                        # refresh_index(), which re-extracts text across tracked
+                        # directories.
+                        personal_docs_manager.add_directory(directory, index=False)
+                    return result
+
+                # Indexing walks, embeds, and stores the whole tree — minutes
+                # on a real directory. The handler is async, so calling it
+                # inline runs it on the event loop and every other request
+                # queues behind it until it finishes (#5558). Serialize on the
+                # async job lock BEFORE offloading so a queued request parks on
+                # the loop instead of pinning a threadpool worker.
+                async with _index_job_lock:
+                    result = await run_in_threadpool(_index_directory)
+
                 if result["success"]:
-                    # Also update the personal_docs_manager to track this directory
-                    personal_docs_manager.add_directory(directory, index=False)
-                    
                     return {
                         "success": True,
                         "message": f"Successfully indexed {result['indexed_count']} chunks from {directory}",
@@ -251,17 +286,25 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
 
             logger.info(f"Removing directory from RAG: {directory}")
 
-            # Always remove from personal_docs_manager tracking
-            if hasattr(personal_docs_manager, 'remove_directory'):
-                personal_docs_manager.remove_directory(directory)
-
-            # Remove from RAG vector store (best-effort)
             rag = _rag()
-            if rag:
-                try:
-                    rag.remove_directory(directory)
-                except Exception as e:
-                    logger.warning(f"RAG removal failed for directory {directory}: {e}")
+
+            def _remove_directory():
+                # Always remove from personal_docs_manager tracking. This
+                # mutates the same unsynchronized list/index an add job touches
+                # and re-extracts text (refresh_index), so it is blocking work.
+                if hasattr(personal_docs_manager, 'remove_directory'):
+                    personal_docs_manager.remove_directory(directory)
+                # Remove from RAG vector store (best-effort).
+                if rag:
+                    try:
+                        rag.remove_directory(directory)
+                    except Exception as e:
+                        logger.warning(f"RAG removal failed for directory {directory}: {e}")
+
+            # Same job lock as add/reload so remove cannot interleave with an
+            # in-flight add; offloaded off the event loop.
+            async with _index_job_lock:
+                await run_in_threadpool(_remove_directory)
 
             return {
                 "success": True,
@@ -289,54 +332,73 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         total_failed = 0
         uploaded_files = []
 
-        for upload in files:
-            try:
-                file_path, stored_name, safe_name = _unique_personal_upload_path(upload_dir, upload.filename)
-                content_bytes = await upload.read(PERSONAL_UPLOAD_MAX_BYTES + 1)
-                if len(content_bytes) > PERSONAL_UPLOAD_MAX_BYTES:
-                    logger.warning(f"Rejected oversized personal upload: {upload.filename!r}")
-                    total_failed += 1
-                    continue
-                with open(file_path, "wb") as f:
-                    f.write(content_bytes)
-
-                ext = os.path.splitext(safe_name)[1].lower()
-                if ext == ".pdf":
-                    from src.personal_docs import extract_pdf_text
-                    text = extract_pdf_text(file_path)
-                else:
-                    text = content_bytes.decode("utf-8", errors="replace")
-
-                if not text or not text.strip():
-                    total_failed += 1
-                    continue
-
-                # Chunk and index
-                chunks = rag._split_into_chunks(text, chunk_size=500)
-                for i, chunk in enumerate(chunks):
-                    metadata = {
-                        "source": file_path,
-                        "filename": safe_name,
-                        "stored_filename": stored_name,
-                        "directory": upload_dir,
-                        "type": ext,
-                        "chunk_id": i,
-                    }
-                    if user:
-                        metadata["owner"] = user
-                    if rag.add_document(chunk, metadata):
-                        total_indexed += 1
-                    else:
+        # Chunking, embedding and the tracking update are blocking work over the
+        # same vector/tracking state add_directory mutates (#5634). Take the
+        # shared job lock BEFORE offloading so a queued request parks on the loop
+        # instead of pinning a threadpool worker, matching add_directory.
+        # Read and process one capped payload at a time so a multi-file request
+        # cannot retain len(files) * PERSONAL_UPLOAD_MAX_BYTES in memory.
+        async with _index_job_lock:
+            for upload in files:
+                try:
+                    file_path, stored_name, safe_name = _unique_personal_upload_path(
+                        upload_dir, upload.filename
+                    )
+                    content_bytes = await upload.read(PERSONAL_UPLOAD_MAX_BYTES + 1)
+                    if len(content_bytes) > PERSONAL_UPLOAD_MAX_BYTES:
+                        logger.warning(f"Rejected oversized personal upload: {upload.filename!r}")
                         total_failed += 1
+                        continue
 
-                uploaded_files.append(safe_name)
-            except Exception as e:
-                logger.error(f"Failed to upload/index {upload.filename}: {e}")
-                total_failed += 1
+                    def _index_upload():
+                        with open(file_path, "wb") as f:
+                            f.write(content_bytes)
 
-        # Track uploads directory
-        if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
-            personal_docs_manager.add_directory(upload_dir, index=False)
+                        ext = os.path.splitext(safe_name)[1].lower()
+                        if ext == ".pdf":
+                            from src.personal_docs import extract_pdf_text
+                            text = extract_pdf_text(file_path)
+                        else:
+                            text = content_bytes.decode("utf-8", errors="replace")
+
+                        if not text or not text.strip():
+                            return 0, 1, None
+
+                        indexed = 0
+                        failed = 0
+                        chunks = rag._split_into_chunks(text, chunk_size=500)
+                        for i, chunk in enumerate(chunks):
+                            metadata = {
+                                "source": file_path,
+                                "filename": safe_name,
+                                "stored_filename": stored_name,
+                                "directory": upload_dir,
+                                "type": ext,
+                                "chunk_id": i,
+                            }
+                            if user:
+                                metadata["owner"] = user
+                            if rag.add_document(chunk, metadata):
+                                indexed += 1
+                            else:
+                                failed += 1
+                        return indexed, failed, safe_name
+
+                    indexed, failed, uploaded_name = await run_in_threadpool(_index_upload)
+                    total_indexed += indexed
+                    total_failed += failed
+                    if uploaded_name:
+                        uploaded_files.append(uploaded_name)
+                except Exception as e:
+                    logger.error(f"Failed to upload/index {upload.filename}: {e}")
+                    total_failed += 1
+
+            # Same transition, same lock: the tracking update must not land
+            # while another job is mid-write over the same state.
+            if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
+                await run_in_threadpool(
+                    personal_docs_manager.add_directory, upload_dir, index=False
+                )
 
         return {
             "success": True,
@@ -349,38 +411,47 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     async def delete_file_from_rag(filepath: str = Query(...), owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
         """Delete a specific file from RAG index and optionally from disk."""
         try:
-            # Remove chunks from RAG vector store (best-effort)
-            removed = 0
-            rag = _rag()
-            if rag:
-                try:
-                    removed = rag.delete_by_source(filepath)
-                except Exception as e:
-                    logger.warning(f"RAG removal failed for {filepath}: {e}")
+            def _delete_file():
+                # Remove chunks from RAG vector store (best-effort)
+                removed = 0
+                rag = _rag()
+                if rag:
+                    try:
+                        removed = rag.delete_by_source(filepath)
+                    except Exception as e:
+                        logger.warning(f"RAG removal failed for {filepath}: {e}")
 
-            # Delete file from disk if it's in the caller's own uploads dir.
-            # Scope to the per-owner subdir, not the shared uploads root, so one
-            # admin can't delete another user's personal files by path.
-            deleted_from_disk = False
-            try:
-                abs_target = os.path.realpath(filepath)
-                base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
-                in_uploads = (
-                    abs_target == base_abs
-                    or os.path.commonpath([abs_target, base_abs]) == base_abs
-                )
-            except ValueError:
-                # commonpath raises on mixed drives / non-comparable paths
-                in_uploads = False
-            if in_uploads and abs_target != base_abs:
+                # Delete file from disk if it's in the caller's own uploads dir.
+                # Scope to the per-owner subdir, not the shared uploads root, so one
+                # admin can't delete another user's personal files by path.
+                deleted_from_disk = False
                 try:
-                    os.remove(abs_target)
-                    deleted_from_disk = True
-                except FileNotFoundError:
-                    pass  # already gone — race with another request or cleanup
+                    abs_target = os.path.realpath(filepath)
+                    base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
+                    in_uploads = (
+                        abs_target == base_abs
+                        or os.path.commonpath([abs_target, base_abs]) == base_abs
+                    )
+                except ValueError:
+                    # commonpath raises on mixed drives / non-comparable paths
+                    in_uploads = False
+                if in_uploads and abs_target != base_abs:
+                    try:
+                        os.remove(abs_target)
+                        deleted_from_disk = True
+                    except FileNotFoundError:
+                        pass  # already gone — race with another request or cleanup
 
-            # Exclude the file from the listing (persists across restarts)
-            personal_docs_manager.exclude_file(filepath)
+                # Exclude the file from the listing (persists across restarts)
+                personal_docs_manager.exclude_file(filepath)
+                return removed, deleted_from_disk
+
+            # Vector removal, the disk unlink and the exclusion write are one
+            # transition over the same state add_directory mutates (#5634), and
+            # all three block. Take the shared job lock BEFORE offloading, as
+            # add_directory does.
+            async with _index_job_lock:
+                removed, deleted_from_disk = await run_in_threadpool(_delete_file)
 
             return {
                 "success": True,

routes/prefs_routes.py

@@ -1,12 +1,16 @@
 """User preferences API — per-user key/value store backed by a JSON file."""
 import json
-import os
 from typing import Optional
 from fastapi import APIRouter, Request
+from core.atomic_io import atomic_write_json
 from src.auth_helpers import get_current_user
 from src.constants import USER_PREFS_FILE
 
 PREFS_FILE = USER_PREFS_FILE
+_FOREGROUND_POLICY_KEYS = (
+    "foreground_fallback_enabled",
+    "foreground_model_fallbacks",
+)
 
 
 def _load():
@@ -20,26 +24,33 @@ def _load():
 
 
 def _save(prefs):
-    os.makedirs(os.path.dirname(PREFS_FILE) or ".", exist_ok=True)
-    tmp = f"{PREFS_FILE}.tmp.{os.getpid()}"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(prefs, f, indent=2)
-        f.flush()
-        os.fsync(f.fileno())
-    os.replace(tmp, PREFS_FILE)
+    atomic_write_json(PREFS_FILE, prefs, indent=2)
 
 
 def _load_for_user(user: Optional[str] = None) -> dict:
     """Load preferences for a specific user."""
     all_prefs = _load()
-    if "_users" in all_prefs:
+    users = all_prefs.get("_users")
+    if isinstance(users, dict):
         if user is None:
             # Auth disabled — return first user's prefs for backward compat
-            users = all_prefs["_users"]
-            return dict(next(iter(users.values()), {}))
-        return dict(all_prefs["_users"].get(user, {}))
-    # Legacy flat format — return as-is
-    return dict(all_prefs)
+            prefs = dict(next(iter(users.values()), {}))
+            # Foreground fallback consent is never borrowed from a named
+            # owner. Auth-disabled operation has a separate flat/root opt-in
+            # that remains inert when authentication is enabled again.
+            for key in _FOREGROUND_POLICY_KEYS:
+                prefs.pop(key, None)
+                if key in all_prefs:
+                    prefs[key] = all_prefs[key]
+            return prefs
+        prefs = users.get(user, {})
+        return dict(prefs) if isinstance(prefs, dict) else {}
+    # A legacy flat store belongs only to auth-disabled single-user mode.
+    # Copying it into the first named user's new `_users` record during an
+    # auth transition would silently transfer another user's preferences and,
+    # critically, foreground fallback consent. Named owners therefore start
+    # with an empty record and must write their own preferences explicitly.
+    return dict(all_prefs) if user is None else {}
 
 
 def _save_for_user(user: Optional[str], prefs: dict):
@@ -51,17 +62,40 @@ def _save_for_user(user: Optional[str], prefs: dict):
         # `prefs` flat would overwrite the whole `_users` map and destroy every
         # other user's preferences. Instead write back into the same (first)
         # slot _load_for_user(None) reads from, preserving the others.
-        if "_users" in all_prefs:
-            users = all_prefs["_users"]
+        users = all_prefs.get("_users")
+        if isinstance(users, dict):
             first_key = next(iter(users), None)
             if first_key is not None:
-                users[first_key] = prefs
+                existing_named = users.get(first_key)
+                existing_named = (
+                    dict(existing_named)
+                    if isinstance(existing_named, dict)
+                    else {}
+                )
+                named_foreground = {
+                    key: existing_named[key]
+                    for key in _FOREGROUND_POLICY_KEYS
+                    if key in existing_named
+                }
+                users[first_key] = {
+                    key: value
+                    for key, value in prefs.items()
+                    if key not in _FOREGROUND_POLICY_KEYS
+                }
+                users[first_key].update(named_foreground)
+                for key in _FOREGROUND_POLICY_KEYS:
+                    if key in prefs:
+                        all_prefs[key] = prefs[key]
                 _save(all_prefs)
                 return
         _save(prefs)
         return
-    if "_users" not in all_prefs:
-        all_prefs = {"_users": {}}
+    if not isinstance(all_prefs.get("_users"), dict):
+        # Preserve the flat single-user object as inert legacy data while
+        # creating the first named-owner namespace. In particular, historical
+        # fallback values must not be deleted or copied into the new owner.
+        all_prefs = dict(all_prefs)
+        all_prefs["_users"] = {}
     all_prefs["_users"][user] = prefs
     _save(all_prefs)
 

routes/research/research_routes.py

@@ -15,7 +15,7 @@ from pydantic import BaseModel, Field
 from core.middleware import INTERNAL_TOOL_USER
 from src.endpoint_resolver import resolve_endpoint
 from src.auth_helpers import _auth_disabled, get_current_user
-from core.auth import RESERVED_USERNAMES
+from src.owner_identity import REQUEST_SENTINEL_OWNERS
 from src.constants import DEEP_RESEARCH_DIR
 
 _SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
@@ -496,7 +496,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         user = require_privilege(request, "can_use_research")
         if user == INTERNAL_TOOL_USER:
             tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in RESERVED_USERNAMES:
+            if tool_owner and tool_owner not in REQUEST_SENTINEL_OWNERS:
                 auth_mgr = getattr(request.app.state, "auth_manager", None)
                 if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
                     try:

routes/search/__init__.py

@@ -0,0 +1,5 @@
+"""Search route domain package (slice 2j, #4082/#4071).
+
+Contains search_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/search_routes.py re-exports from here.
+"""

routes/search/search_routes.py

@@ -0,0 +1,111 @@
+"""Search routes — /api/search/config GET, /api/search POST."""
+
+import logging
+from typing import Dict, Any
+
+from fastapi import APIRouter, Request
+
+import time
+
+from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
+from services.search.core import _call_provider
+from services.search.providers import _get_provider_key, _get_search_instance
+
+logger = logging.getLogger(__name__)
+
+
+async def _request_values(request: Request) -> Dict[str, Any]:
+    """Accept JSON, form data, or query params for search endpoints.
+
+    The browser UI posts FormData, while the agent's generic app_api tool
+    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
+    runs, which made the model think SearXNG was broken.
+    """
+    values: Dict[str, Any] = dict(request.query_params)
+    content_type = (request.headers.get("content-type") or "").lower()
+    try:
+        if "application/json" in content_type:
+            body = await request.json()
+            if isinstance(body, dict):
+                values.update(body)
+        else:
+            form = await request.form()
+            values.update(dict(form))
+    except Exception:
+        pass
+    return values
+
+
+def setup_search_routes(config) -> APIRouter:
+    router = APIRouter(tags=["search"])
+
+    @router.get("/api/search/config")
+    async def get_search_settings() -> Dict[str, Any]:
+        return get_search_config()
+
+    @router.post("/api/search")
+    async def do_web_search(request: Request) -> Dict[str, Any]:
+        """Standalone web search — returns context string + source list.
+
+        Used by Compare mode to pre-search once and share results across panes.
+        """
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        if not query:
+            return {"context": "", "sources": [], "error": "query is required"}
+        time_filter = values.get("time_filter") or values.get("freshness")
+        if time_filter is not None:
+            time_filter = str(time_filter).strip() or None
+        try:
+            context, sources = comprehensive_web_search(
+                query, return_sources=True, time_filter=time_filter,
+            )
+            return {"context": context, "sources": sources}
+        except Exception as e:
+            logger.error(f"Standalone web search failed: {e}")
+            return {"context": "", "sources": [], "error": str(e)}
+
+    @router.get("/api/search/providers")
+    async def list_search_providers():
+        """Return available search providers with config status."""
+        providers = []
+        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
+            if pid == "disabled":
+                continue
+            available = True
+            if needs_key and not _get_provider_key(pid):
+                available = False
+            if needs_url and pid == "searxng" and not _get_search_instance():
+                available = False
+            providers.append({
+                "id": pid,
+                "label": label,
+                "available": available,
+            })
+        return providers
+
+    @router.post("/api/search/query")
+    async def search_with_provider(request: Request) -> Dict[str, Any]:
+        """Search using a specific provider. Used by compare search mode."""
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        provider = str(values.get("provider") or "").strip()
+        try:
+            count = int(values.get("count") or values.get("limit") or 10)
+        except Exception:
+            count = 10
+        if not query:
+            return {"results": [], "provider": provider, "error": "query is required"}
+        if provider not in PROVIDER_INFO or provider == "disabled":
+            return {"results": [], "provider": provider, "error": "Unknown provider"}
+        t0 = time.time()
+        try:
+            results = _call_provider(provider, query, min(count, 20))
+            elapsed = round(time.time() - t0, 2)
+            return {"results": results, "provider": provider, "time": elapsed}
+        except Exception as e:
+            elapsed = round(time.time() - t0, 2)
+            logger.error(f"Search provider {provider} failed: {e}")
+            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
+
+    return router

routes/search_routes.py

@@ -1,111 +1,13 @@
-"""Search routes — /api/search/config GET, /api/search POST."""
+"""Backward-compat shim — canonical location is routes/search/search_routes.py.
 
-import logging
-from typing import Dict, Any
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.search_routes`` and ``from routes.search_routes import X``
+keep resolving to the canonical module. Keeps existing import paths working
+after slice 2j (#4082/#4071).
+"""
 
-from fastapi import APIRouter, Request
+import sys as _sys
 
-import time
+from routes.search import search_routes as _canonical  # noqa: F401
 
-from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
-from services.search.core import _call_provider
-from services.search.providers import _get_provider_key, _get_search_instance
-
-logger = logging.getLogger(__name__)
-
-
-async def _request_values(request: Request) -> Dict[str, Any]:
-    """Accept JSON, form data, or query params for search endpoints.
-
-    The browser UI posts FormData, while the agent's generic app_api tool
-    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
-    runs, which made the model think SearXNG was broken.
-    """
-    values: Dict[str, Any] = dict(request.query_params)
-    content_type = (request.headers.get("content-type") or "").lower()
-    try:
-        if "application/json" in content_type:
-            body = await request.json()
-            if isinstance(body, dict):
-                values.update(body)
-        else:
-            form = await request.form()
-            values.update(dict(form))
-    except Exception:
-        pass
-    return values
-
-
-def setup_search_routes(config) -> APIRouter:
-    router = APIRouter(tags=["search"])
-
-    @router.get("/api/search/config")
-    async def get_search_settings() -> Dict[str, Any]:
-        return get_search_config()
-
-    @router.post("/api/search")
-    async def do_web_search(request: Request) -> Dict[str, Any]:
-        """Standalone web search — returns context string + source list.
-
-        Used by Compare mode to pre-search once and share results across panes.
-        """
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        if not query:
-            return {"context": "", "sources": [], "error": "query is required"}
-        time_filter = values.get("time_filter") or values.get("freshness")
-        if time_filter is not None:
-            time_filter = str(time_filter).strip() or None
-        try:
-            context, sources = comprehensive_web_search(
-                query, return_sources=True, time_filter=time_filter,
-            )
-            return {"context": context, "sources": sources}
-        except Exception as e:
-            logger.error(f"Standalone web search failed: {e}")
-            return {"context": "", "sources": [], "error": str(e)}
-
-    @router.get("/api/search/providers")
-    async def list_search_providers():
-        """Return available search providers with config status."""
-        providers = []
-        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
-            if pid == "disabled":
-                continue
-            available = True
-            if needs_key and not _get_provider_key(pid):
-                available = False
-            if needs_url and pid == "searxng" and not _get_search_instance():
-                available = False
-            providers.append({
-                "id": pid,
-                "label": label,
-                "available": available,
-            })
-        return providers
-
-    @router.post("/api/search/query")
-    async def search_with_provider(request: Request) -> Dict[str, Any]:
-        """Search using a specific provider. Used by compare search mode."""
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        provider = str(values.get("provider") or "").strip()
-        try:
-            count = int(values.get("count") or values.get("limit") or 10)
-        except Exception:
-            count = 10
-        if not query:
-            return {"results": [], "provider": provider, "error": "query is required"}
-        if provider not in PROVIDER_INFO or provider == "disabled":
-            return {"results": [], "provider": provider, "error": "Unknown provider"}
-        t0 = time.time()
-        try:
-            results = _call_provider(provider, query, min(count, 20))
-            elapsed = round(time.time() - t0, 2)
-            return {"results": results, "provider": provider, "time": elapsed}
-        except Exception as e:
-            elapsed = round(time.time() - t0, 2)
-            logger.error(f"Search provider {provider} failed: {e}")
-            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/session_routes.py

@@ -12,7 +12,9 @@ from core.models import ChatMessage
 from src.request_models import SessionResponse
 from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
 from src.auth_helpers import effective_user, _auth_disabled, owner_filter
+from src.session_image_cleanup import _generated_image_path_for_cleanup, session_image_refs
 from src.session_actions import is_session_recently_active
+from src.upload_handler import reserve_message_upload_references
 
 
 def _sanitize_export_filename(name: str) -> str:
@@ -203,7 +205,12 @@ def _pick_endpoint_for_sort(owner=None):
         return url, model, headers
     return None, None, None
 
-def setup_session_routes(session_manager: SessionManager, config: dict, webhook_manager=None):
+def setup_session_routes(
+    session_manager: SessionManager,
+    config: dict,
+    webhook_manager=None,
+    upload_handler=None,
+):
     """Setup session routes with the provided manager and config"""
 
     REQUEST_TIMEOUT = config.get("REQUEST_TIMEOUT", 20)
@@ -214,6 +221,7 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
     @router.get("/sessions")
     def list_sessions(request: Request):
         user = effective_user(request)
+        active_incognito_id = str(request.query_params.get("active_incognito_id") or "").strip()
         # Lazy purge: incognito sessions are ephemeral by design — wipe leftovers
         # from the DB and session_manager so they vanish on the next page refresh.
         # BUT: skip sessions that were created within the last 10 minutes.
@@ -234,6 +242,8 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
                     DbSession.created_at < _cutoff,
                 ).all()
                 for _g in _ghosts:
+                    if active_incognito_id and _g.id == active_incognito_id:
+                        continue
                     _purge_db.query(_DbMsg).filter(_DbMsg.session_id == _g.id).delete()
                     _purge_db.delete(_g)
                     if hasattr(session_manager, "delete_session"):
@@ -537,6 +547,22 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         body = await request.json()
         messages = body.get("messages", [])
         from core.models import ChatMessage
+        owner = effective_user(request)
+        try:
+            for message in messages:
+                missing_id = reserve_message_upload_references(
+                    upload_handler,
+                    owner,
+                    message.get("content"),
+                    message.get("metadata"),
+                )
+                if missing_id:
+                    raise HTTPException(
+                        409,
+                        f"Referenced upload is no longer available: {missing_id}",
+                    )
+        except (AttributeError, TypeError, ValueError) as exc:
+            raise HTTPException(400, "Invalid message attachment metadata") from exc
         for m in messages:
             sess.add_message(ChatMessage(m["role"], m["content"], metadata=m.get("metadata")))
         session_manager.save_sessions()
@@ -619,13 +645,43 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         db = SessionLocal()
         try:
             from core.database import ChatMessage as DbChatMessage
+            session_ids = [row[0] for row in db.query(DbSession.id).all()]
             count = db.query(DbSession).count()
+            image_ids: set[str] = set()
+            filenames: set[str] = set()
+            for sid in session_ids:
+                ids, names = session_image_refs(db, sid)
+                image_ids.update(ids)
+                filenames.update(names)
+            image_query = db.query(GalleryImage).filter(GalleryImage.session_id.in_(session_ids)) if session_ids else db.query(GalleryImage).filter(False)
+            if image_ids or filenames:
+                from sqlalchemy import or_
+                clauses = []
+                if session_ids:
+                    clauses.append(GalleryImage.session_id.in_(session_ids))
+                if image_ids:
+                    clauses.append(GalleryImage.id.in_(list(image_ids)))
+                if filenames:
+                    clauses.append(GalleryImage.filename.in_(list(filenames)))
+                image_query = db.query(GalleryImage).filter(or_(*clauses))
+            images = image_query.all()
+            removed_images = 0
+            for img in images:
+                img.is_active = False
+                if img.filename:
+                    path = _generated_image_path_for_cleanup(img.filename)
+                    if path and path.exists():
+                        try:
+                            path.unlink()
+                        except Exception as exc:
+                            logger.warning("Could not remove generated image %s during all-session delete: %s", img.filename, exc)
+                removed_images += 1
             db.query(DbChatMessage).delete()
             db.query(DbSession).delete()
             db.commit()
             session_manager.sessions.clear()
-            logger.info(f"Admin deleted all {count} sessions")
-            return {"status": "deleted", "count": count}
+            logger.info(f"Admin deleted all {count} sessions and {removed_images} linked images")
+            return {"status": "deleted", "count": count, "images_deleted": removed_images}
         except Exception as e:
             db.rollback()
             logger.error(f"Error deleting all sessions: {e}")
@@ -745,15 +801,6 @@ def setup_session_routes(session_manager: SessionManager, config: dict, webhook_
         finally:
             db.close()
 
-    @router.get("/history/{sid}")
-    def get_history(request: Request, sid: str):
-        _verify_session_owner(request, sid)
-        try:
-            session = session_manager.get_session(sid)
-        except KeyError:
-            raise HTTPException(404, f"Session {sid} not found")
-        return {"history": [msg.to_dict() for msg in session.history]}
-    
     @router.get("/session/{sid}/export")
     def export_session(request: Request, sid: str, fmt: str = "md", filename: str = ""):
         """Export conversation history as a downloadable file.

routes/shell_routes.py

@@ -157,6 +157,7 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
     binaries = probe.get("binaries") if isinstance(probe.get("binaries"), dict) else {}
     dists = probe.get("dists") if isinstance(probe.get("dists"), dict) else {}
     modules = probe.get("modules") if isinstance(probe.get("modules"), dict) else {}
+    files = probe.get("files") if isinstance(probe.get("files"), dict) else {}
 
     if name == "vllm":
         return bool(binaries.get("vllm"))
@@ -166,11 +167,43 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
         return bool(dists.get("sglang") or modules.get("sglang", {}).get("real_module"))
     if name == "mlx_lm":
         return bool(dists.get("mlx-lm") or modules.get("mlx_lm", {}).get("real_module"))
+    if name == "mflux":
+        return bool(
+            dists.get("mflux")
+            or modules.get("mflux", {}).get("real_module")
+            or binaries.get("mflux-generate-qwen")
+            or binaries.get("mflux-generate")
+        )
+    if name == "boogu_image_mlx":
+        return bool(
+            dists.get("boogu-image-mlx")
+            or modules.get("boogu_image_mlx", {}).get("real_module")
+        )
+    if name == "mlx_lama_swift":
+        return bool(
+            (binaries.get("odysseus-mlx-inpaint") or binaries.get("mlx-lama-serve"))
+            and (files.get("mlx.metallib") or files.get("default.metallib"))
+        )
+    if name == "mlx_ddcolor_swift":
+        return bool(
+            (binaries.get("odysseus-mlx-colorize") or binaries.get("mlx-ddcolor-serve"))
+            and (files.get("mlx.metallib") or files.get("default.metallib"))
+        )
     if name == "diffusers":
         return bool(
             (dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
             and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
         )
+    if name == "krea_diffusers":
+        return bool(
+            (dists.get("diffusers") or modules.get("diffusers", {}).get("real_module"))
+            and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
+        )
+    if name == "sam_mask":
+        return bool(
+            (dists.get("transformers") or modules.get("transformers", {}).get("real_module"))
+            and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
+        )
     if name == "hf_transfer":
         return bool(
             dists.get("hf-transfer")
@@ -183,6 +216,7 @@ def _package_status_note(name: str, probe: dict) -> str:
     binaries = probe.get("binaries") if isinstance(probe.get("binaries"), dict) else {}
     modules = probe.get("modules") if isinstance(probe.get("modules"), dict) else {}
     dists = probe.get("dists") if isinstance(probe.get("dists"), dict) else {}
+    files = probe.get("files") if isinstance(probe.get("files"), dict) else {}
     module = modules.get(name) if isinstance(modules.get(name), dict) else {}
     locations = module.get("locations") or []
     if name == "vllm":
@@ -212,10 +246,53 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}"
         return "Diffusers serving needs both diffusers and torch."
+    if name == "krea_diffusers":
+        if _package_installed_from_probe(name, probe):
+            return f"Latest Diffusers runtime: diffusers {dists.get('diffusers', 'available')} with torch {dists.get('torch', 'available')}. Use Update/Reinstall to pull latest Diffusers from Git."
+        return "Some newer image models need torch plus latest Diffusers from Git."
+    if name == "sam_mask":
+        if _package_installed_from_probe(name, probe):
+            return f"SAM object masks: transformers {dists.get('transformers', 'available')} with torch {dists.get('torch', 'available')}"
+        return "SAM click/object mask selection needs transformers and torch."
     if name == "mlx_lm":
         if _package_installed_from_probe(name, probe):
             return f"MLX LM {dists.get('mlx-lm', 'available')}"
         return "MLX serving needs mlx-lm on an Apple Silicon Mac."
+    if name == "mflux":
+        if _package_installed_from_probe(name, probe):
+            parts = []
+            if dists.get("mflux"):
+                parts.append(f"mflux {dists['mflux']}")
+            if binaries.get("mflux-generate-qwen"):
+                parts.append(f"Qwen CLI: {binaries['mflux-generate-qwen']}")
+            if binaries.get("mflux-generate"):
+                parts.append(f"Flux CLI: {binaries['mflux-generate']}")
+            return "; ".join(parts) if parts else "mflux available"
+        return "MLX image serving needs mflux on an Apple Silicon Mac."
+    if name == "boogu_image_mlx":
+        if _package_installed_from_probe(name, probe):
+            return f"Boogu MLX pipeline {dists.get('boogu-image-mlx', 'available')}"
+        return "Boogu image models need boogu-image-mlx on an Apple Silicon Mac."
+    if name == "mlx_lama_swift":
+        if _package_installed_from_probe(name, probe):
+            found = [
+                binaries.get("odysseus-mlx-inpaint"),
+                binaries.get("mlx-lama-serve"),
+            ]
+            return f"LaMa/MI-GAN Swift MLX runner: {next((p for p in found if p), 'available')}"
+        if binaries.get("odysseus-mlx-inpaint") or binaries.get("mlx-lama-serve"):
+            return "LaMa/MI-GAN Swift runner is installed, but mlx.metallib is missing next to the runner."
+        return "LaMa/MI-GAN inpainting models need an Odysseus-compatible mlx-lama-swift bridge on an Apple Silicon Mac."
+    if name == "mlx_ddcolor_swift":
+        if _package_installed_from_probe(name, probe):
+            found = [
+                binaries.get("odysseus-mlx-colorize"),
+                binaries.get("mlx-ddcolor-serve"),
+            ]
+            return f"DDColor Swift MLX runner: {next((p for p in found if p), 'available')}"
+        if binaries.get("odysseus-mlx-colorize") or binaries.get("mlx-ddcolor-serve"):
+            return "DDColor Swift runner is installed, but mlx.metallib is missing next to the runner."
+        return "DDColor colorization models need an Odysseus-compatible mlx-ddcolor-swift bridge on an Apple Silicon Mac."
     if name in dists:
         return f"{name} {dists[name]}"
     return ""
@@ -314,12 +391,22 @@ dist_names={{
     'llama_cpp':['llama-cpp-python'],
     'sglang':['sglang'],
     'mlx_lm':['mlx-lm'],
+    'mlx_vlm':['mlx-vlm'],
+    'mflux':['mflux'],
+    'boogu_image_mlx':['boogu-image-mlx'],
+    'mlx_lama_swift':[],
+    'mlx_ddcolor_swift':[],
     'diffusers':['diffusers','torch'],
+    'krea_diffusers':['diffusers','torch'],
+    'sam_mask':['transformers','torch'],
     'hf_transfer':['hf-transfer','hf_transfer'],
 }}
 bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
+    'mflux':['mflux-generate-qwen', 'mflux-generate'],
+    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
+    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
     'tmux':['tmux'],
 }}
 
@@ -372,7 +459,19 @@ def probe(n):
         mods['torch'] = mod_status('torch')
     dists = dist_status(dist_names.get(n, [n]))
     bins = {{b: shutil.which(b) for b in bin_names.get(n, [])}}
-    return {{'modules': mods, 'dists': dists, 'binaries': bins}}
+    files = {{}}
+    if n in ('mlx_lama_swift', 'mlx_ddcolor_swift'):
+        for key in ('mlx.metallib', 'default.metallib'):
+            found = None
+            for b in bins.values():
+                if not b:
+                    continue
+                p = os.path.join(os.path.dirname(b), key)
+                if os.path.exists(p):
+                    found = p
+                    break
+            files[key] = found
+    return {{'modules': mods, 'dists': dists, 'binaries': bins, 'files': files}}
 
 print(json.dumps({{n: probe(n) for n in names}}))
 """
@@ -1088,6 +1187,8 @@ def setup_shell_routes() -> APIRouter:
         ssh_port: str | None = None,
         venv: str | None = None,
         backend: str | None = None,
+        platform: str | None = None,
+        model_hint: str | None = None,
     ):
         """Check which optional packages are installed.
 
@@ -1104,6 +1205,14 @@ def setup_shell_routes() -> APIRouter:
         import site
         import sys
 
+        platform_l = (platform or "").strip().lower()
+        model_hint_l = (model_hint or "").strip().lower()
+        has_krea_model = "krea" in model_hint_l
+        has_lama_mlx_model = any(
+            key in model_hint_l
+            for key in ("lama", "mi-gan", "migan", "inpainting-mlx")
+        )
+        has_ddcolor_mlx_model = "ddcolor" in model_hint_l
         _prepend_user_install_bins_to_path()
         importlib.invalidate_caches()
         try:
@@ -1158,7 +1267,7 @@ def setup_shell_routes() -> APIRouter:
                 "name": "hf_transfer",
                 "pip": "hf_transfer",
                 "desc": "Fast model downloads from HuggingFace",
-                "category": "LLM",
+                "category": "Tools",
                 "target": "remote",
             },
             {
@@ -1210,8 +1319,52 @@ def setup_shell_routes() -> APIRouter:
             # ── Image ── editor + diffusion model serving
             {
                 "name": "diffusers",
-                "pip": "diffusers[torch]",
-                "desc": "Image generation/editing pipelines (SD, Flux) with PyTorch",
+                "pip": "diffusers[torch] torchvision accelerate scipy python-multipart",
+                "desc": "Image generation/editing pipelines with PyTorch and Diffusers",
+                "category": "Image",
+                "target": "remote",
+            },
+            {
+                "name": "krea_diffusers",
+                "pip": "git+https://github.com/huggingface/diffusers.git torchvision accelerate scipy python-multipart",
+                "desc": "Latest Diffusers from Git for newly released image pipelines",
+                "category": "Image",
+                "target": "remote",
+            },
+            {
+                "name": "mflux",
+                "pip": "mflux",
+                "desc": "MLX image generation runtime for Apple Silicon models like Qwen Image",
+                "category": "Image",
+                "target": "remote",
+            },
+            {
+                "name": "boogu_image_mlx",
+                "pip": "git+https://github.com/xocialize/boogu-image-mlx.git",
+                "desc": "MLX image generation pipeline for Boogu Image models on Apple Silicon",
+                "category": "Image",
+                "target": "remote",
+            },
+            {
+                "name": "mlx_lama_swift",
+                "pip": "",
+                "desc": "Swift MLX runtime for LaMa / MI-GAN inpainting and object removal",
+                "category": "Image",
+                "target": "remote",
+                "install_hint": "Build an Odysseus-compatible mlx-lama-swift bridge on the selected Apple Silicon Mac and put odysseus-mlx-inpaint or mlx-lama-serve on PATH. Upstream currently ships Swift libraries plus smoke executables, not a stable image-edit CLI.",
+            },
+            {
+                "name": "mlx_ddcolor_swift",
+                "pip": "",
+                "desc": "Swift MLX runtime for DDColor automatic image colorization",
+                "category": "Image",
+                "target": "remote",
+                "install_hint": "Build an Odysseus-compatible mlx-ddcolor-swift bridge on the selected Apple Silicon Mac and put odysseus-mlx-colorize or mlx-ddcolor-serve on PATH. Upstream currently ships Swift libraries plus smoke executables, not a stable colorize CLI.",
+            },
+            {
+                "name": "mlx_vlm",
+                "pip": "mlx-vlm",
+                "desc": "MLX-VLM backbone used by HiDream image models on Apple Silicon",
                 "category": "Image",
                 "target": "remote",
             },
@@ -1222,6 +1375,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Image",
                 "target": "remote",
             },
+            {
+                "name": "sam_mask",
+                "pip": "torch torchvision transformers accelerate pillow",
+                "desc": "Neutral click/box/object segmentation masks for the image editor",
+                "category": "Image",
+                "target": "local",
+            },
             {
                 "name": "rembg",
                 "pip": "rembg[gpu]",
@@ -1251,6 +1411,21 @@ def setup_shell_routes() -> APIRouter:
         for pkg in packages:
             pkg.setdefault("install_cmd", None)
             pkg.setdefault("update_cmd", None)
+        if not has_krea_model:
+            packages = [
+                p for p in packages
+                if p.get("name") not in {"krea_diffusers", "transformers"}
+            ]
+        if not has_lama_mlx_model:
+            packages = [
+                p for p in packages
+                if p.get("name") != "mlx_lama_swift"
+            ]
+        if not has_ddcolor_mlx_model:
+            packages = [
+                p for p in packages
+                if p.get("name") != "mlx_ddcolor_swift"
+            ]
         # Remote check: for remote-target packages, probe the selected server's
         # venv over SSH so a remote `pip install` actually reflects here.
         remote_status: dict = {}
@@ -1381,8 +1556,22 @@ def setup_shell_routes() -> APIRouter:
                 target_os_id = ""
             if sys.platform == "darwin":
                 target_os_id = "macos"
+        if not target_os_id and platform_l in {"darwin", "macos", "mac"}:
+            target_os_id = "macos"
 
         for pkg in packages:
+            if pkg.get("name") in {"mflux", "boogu_image_mlx", "mlx_vlm", "mlx_lama_swift", "mlx_ddcolor_swift"}:
+                is_apple_target = target_os_id == "macos" or (
+                    not host and IS_APPLE_SILICON
+                )
+                known_non_apple_target = bool(target_os_id and target_os_id != "macos") or (
+                    not host and not IS_APPLE_SILICON
+                )
+                pkg["applicable"] = is_apple_target
+                if known_non_apple_target:
+                    pkg["installed"] = None
+                    pkg["status_note"] = "Only relevant for Apple Silicon / MLX image serving."
+                    continue
             on_remote = bool(host and pkg.get("target") == "remote")
             probe = None
             if on_remote:
@@ -1588,6 +1777,10 @@ def setup_shell_routes() -> APIRouter:
             "sglang[all]",
             "diffusers",
             "diffusers[torch]",
+            "git+https://github.com/huggingface/diffusers.git",
+            "mflux",
+            "git+https://github.com/xocialize/boogu-image-mlx.git",
+            "mlx-vlm",
             "transformers",
             "TTS",
             "bark",

routes/skills_routes.py

@@ -18,6 +18,7 @@ from pydantic import BaseModel, Field
 
 from services.memory.skills import SkillsManager
 from src.auth_helpers import get_current_user
+from src.prompt_security import untrusted_context_message
 from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
@@ -107,6 +108,23 @@ def _skill_test_task(skill: dict) -> str:
     )
 
 
+def _skill_test_messages(md: str, task: str) -> list[dict]:
+    """Keep user-editable skill text out of the trusted system role."""
+    return [
+        {
+            "role": "system",
+            "content": (
+                "You are TESTING a skill. Follow the supplied reusable procedure "
+                "to complete the user's task for real, using available tools step "
+                "by step. If the skill is wrong, unclear, or references tools that "
+                "do not exist, do your best; the problems will be reviewed afterward."
+            ),
+        },
+        untrusted_context_message("skill under test", md),
+        {"role": "user", "content": task},
+    ]
+
+
 async def _eval_skill_run(skill_md: str, task: str, transcript: str,
                           url: str, model: str, headers: Optional[dict]) -> dict:
     """LLM-as-judge: grade a skill test run from its transcript. Advisory only.
@@ -411,7 +429,21 @@ async def _eval_skill_retrieval_precision(skill_md: str, others: list,
 _skill_test_jobs: dict = {}
 
 
-async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, skills_manager=None):
+async def _run_skill_test_job(
+    key,
+    name,
+    md,
+    task,
+    url,
+    model,
+    headers,
+    owner,
+    skills_manager=None,
+    *,
+    messages=None,
+    transcript=None,
+    exact_approval=None,
+):
     """Background coroutine: run the skill in an agent loop, capture a condensed
     log + transcript, then have the judge grade it. Writes into _skill_test_jobs."""
     import json as _json
@@ -421,7 +453,7 @@ async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, s
     if job is None:
         return
     log = job["log"]
-    transcript = []
+    transcript = transcript if isinstance(transcript, list) else []
     say_buf = []
 
     def _flush_say():
@@ -429,18 +461,12 @@ async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, s
             log.append({"type": "say", "text": "".join(say_buf)})
             say_buf.clear()
 
-    messages = [
-        {"role": "system", "content":
-            "You are TESTING a skill. Below is a reusable skill (a procedure). Follow it "
-            "to complete the user's task for real, using your available tools, step by "
-            "step. If the skill is wrong, unclear, or references tools that don't exist, "
-            "do your best — the problems will be reviewed afterward.\n\n=== SKILL ===\n" + md},
-        {"role": "user", "content": task},
-    ]
+    messages = list(messages) if isinstance(messages, list) else _skill_test_messages(md, task)
     try:
         async for chunk in stream_agent_loop(
             url, model, messages, headers=headers,
             temperature=0.3, max_tokens=0, max_rounds=8, owner=owner,
+            exact_approval=exact_approval,
         ):
             if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]":
                 continue
@@ -458,8 +484,25 @@ async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, s
             elif d.get("type") == "tool_output":
                 _flush_say()
                 out = str(d.get("output") or "")[:600]
-                log.append({"type": "tool_output", "output": out})
+                tool_log = {"type": "tool_output", "output": out}
+                approval = d.get("ask_user")
+                if isinstance(approval, dict):
+                    tool_log["ask_user"] = approval
+                log.append(tool_log)
                 transcript.append(f"[output] {out}\n")
+                if (
+                    isinstance(approval, dict)
+                    and approval.get("kind") == "tool_approval"
+                    and approval.get("approval_id")
+                ):
+                    # Manual skill tests have their own polling UI instead of a
+                    # chat session. Pause the run and retain only server-side
+                    # continuation state until the same owner approves/denies
+                    # this exact sealed action.
+                    job["status"] = "awaiting_approval"
+                    job["approval"] = approval
+                    job["_transcript"] = transcript
+                    return
             elif d.get("type") == "agent_step":
                 _flush_say()
                 log.append({"type": "agent_step", "round": d.get("round")})
@@ -471,6 +514,9 @@ async def _run_skill_test_job(key, name, md, task, url, model, headers, owner, s
         _flush_say()
         log.append({"type": "error", "error": str(e)})
 
+    job.pop("approval", None)
+    job.pop("_transcript", None)
+    job.pop("_run", None)
     log.append({"type": "evaluating"})
     try:
         job["verdict"] = await _eval_skill_run(md, task, "".join(transcript), url, model, headers)
@@ -694,12 +740,8 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) -
     import json as _json
     from src.agent_loop import stream_agent_loop
     transcript = []
-    messages = [
-        {"role": "system", "content":
-            "You are TESTING a skill. Follow this skill's procedure to complete the task "
-            "for real, using your tools, step by step.\n\n=== SKILL ===\n" + md},
-        {"role": "user", "content": task},
-    ]
+    approval_required = None
+    messages = _skill_test_messages(md, task)
     try:
         # max_tokens explicitly set: passing 0 lets some upstreams (Ollama,
         # OpenAI-compat) generate an empty completion, which manifested as
@@ -719,11 +761,44 @@ async def _run_skill_test_once(md: str, task: str, url, model, headers, owner) -
                 transcript.append(f"\n[tool {d.get('tool')}] {str(d.get('command') or d.get('args') or '')[:300]}\n")
             elif d.get("type") == "tool_output":
                 transcript.append(f"[output] {str(d.get('output') or '')[:600]}\n")
+                approval = d.get("ask_user")
+                if (
+                    isinstance(approval, dict)
+                    and approval.get("kind") == "tool_approval"
+                ):
+                    approval_required = approval
+                    break
             elif d.get("type") == "agent_step":
                 transcript.append(f"\n--- round {d.get('round')} ---\n")
     except Exception as e:
         transcript.append(f"\n[run error] {e}\n")
     text = "".join(transcript)
+    if approval_required is not None:
+        # Unattended audits have no authority to approve and no UI that could
+        # resume this record. Destructively deny it now instead of leaving a
+        # reusable opaque grant pending until TTL/cap eviction.
+        try:
+            from src.tool_approvals import tool_approval_store
+            tool_approval_store.consume(
+                approval_required.get("approval_id"),
+                decision="deny",
+                owner=owner,
+                session_id=None,
+            )
+        except Exception:
+            logger.debug("Could not retire unattended skill approval", exc_info=True)
+        return text, {
+            "verdict": "inconclusive",
+            "confidence": 1.0,
+            "summary": (
+                "This automated audit reached an exact action that requires "
+                "a human approval; no action was executed."
+            ),
+            "issues": [
+                "Run this skill's manual test and review the sealed action."
+            ],
+            "approval_required": True,
+        }
     verdict = await _eval_skill_run(md, task, text, url, model, headers)
     return text, verdict
 
@@ -863,6 +938,26 @@ async def _audit_one_skill(skills_manager, skill, url, model, headers,
     transcript, verdict = await _run_skill_test_once(md, task, url, model, headers, owner)
     v = verdict.get("verdict")
     log(f"{name}: verdict = {v} ({verdict.get('summary', '')[:80]})")
+    if verdict.get("approval_required"):
+        # An unattended audit is not authority for an action influenced by the
+        # skill under test. Preserve the skill's current publication/confidence
+        # state and route the exact action to the manual test UI instead of
+        # letting a safety pause demote, rewrite, or auto-publish the skill.
+        skills_manager.set_audit(
+            name,
+            "inconclusive",
+            by_teacher=False,
+            worker_model=model,
+            owner=owner,
+        )
+        status = skill.get("status") or "draft"
+        log(f"{name}: {status} unchanged — exact action needs manual approval")
+        return {
+            "skill": name,
+            "result": "approval_required",
+            "verdict": verdict,
+            "status": status,
+        }
     if v == "pass":
         # Procedure works. If the reviewer still flagged metadata (tags/category/
         # when_to_use/description), do ONE fixer pass to correct the frontmatter
@@ -1409,7 +1504,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
 
         # Prefer the configured DEFAULT (→ Utility) model — not the current chat
         # session's model. Fall back to the caller's session model only if unset.
-        url, model, headers = resolve_endpoint("default", owner=user)
+        url, model, headers = resolve_endpoint("utility", owner=user)
         if not url or not model:
             url = url or ((body.get("endpoint_url") or "").strip() or None)
             model = model or ((body.get("model") or "").strip() or None)
@@ -1431,6 +1526,19 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             logger.warning(f"Skill-test model resolve failed: {_e}")
 
         key = (user or "", name)
+        previous_job = _skill_test_jobs.get(key) or {}
+        previous_approval = previous_job.get("approval") or {}
+        if previous_approval.get("approval_id"):
+            try:
+                from src.tool_approvals import tool_approval_store
+                tool_approval_store.consume(
+                    previous_approval["approval_id"],
+                    decision="deny",
+                    owner=user,
+                    session_id=None,
+                )
+            except Exception:
+                logger.debug("Could not retire replaced skill approval", exc_info=True)
         _skill_test_jobs[key] = {
             "status": "running",
             "task": task,
@@ -1439,10 +1547,138 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             "started": _time.time(),
             "log": [{"type": "skill_test_start", "task": task, "skill": name, "model": model}],
             "verdict": None,
+            "_run": {
+                "md": md,
+                "url": url,
+                "model": model,
+                "headers": headers,
+                "owner": user,
+            },
         }
         _asyncio.create_task(_run_skill_test_job(key, name, md, task, url, model, headers, user, skills_manager))
         return {"ok": True, "status": "running", "skill": name, "model": model}
 
+    @router.post("/{skill_id}/test-approval")
+    async def approve_skill_test_action(request: Request, skill_id: str):
+        """Resume a manual skill test with one exact server-sealed action."""
+        import asyncio as _asyncio
+        from src.tool_approvals import tool_approval_store
+
+        user = _owner(request)
+        skills = skills_manager.load(owner=user)
+        match = next(
+            (s for s in skills if s.get("name") == skill_id or s.get("id") == skill_id),
+            None,
+        )
+        if not match:
+            raise HTTPException(404, "Skill not found")
+        _verify_owner(match, user)
+        name = match.get("name")
+        key = (user or "", name)
+        job = _skill_test_jobs.get(key)
+        if not job or job.get("status") != "awaiting_approval":
+            raise HTTPException(409, "This skill test is not awaiting an approval.")
+
+        body = await request.json()
+        if not isinstance(body, dict):
+            raise HTTPException(400, "Tool approval body must be a JSON object.")
+        approval_id = str(body.get("approval_id") or "")
+        decision = str(body.get("decision") or "").strip().lower()
+        expected = job.get("approval") or {}
+        if approval_id != str(expected.get("approval_id") or ""):
+            raise HTTPException(409, "This approval does not match the pending skill test action.")
+        if decision not in {"approve", "deny"}:
+            raise HTTPException(400, "Invalid tool approval decision.")
+
+        pending = tool_approval_store.peek(approval_id)
+        normalized_owner = str(user or "").strip().casefold()
+        if (
+            pending is None
+            or pending.owner != normalized_owner
+            or pending.session_id != ""
+        ):
+            raise HTTPException(409, "This tool approval is invalid or expired.")
+        exact_approval = tool_approval_store.consume(
+            approval_id,
+            decision=decision,
+            owner=user,
+            session_id=None,
+            # The button here says "Allow once" and there is no chat to carry a
+            # scope into, so the gate must re-arm behind the sealed action.
+            allow_continuation=False,
+        )
+
+        if decision == "approve" and exact_approval is None:
+            raise HTTPException(409, "This tool approval could not be consumed.")
+        job.pop("approval", None)
+        if decision == "deny":
+            job.pop("_transcript", None)
+            job.pop("_run", None)
+            job["log"].append({
+                "type": "approval_denied",
+                "text": "Exact action denied; the skill test stopped without executing it.",
+            })
+            job["verdict"] = {
+                "verdict": "inconclusive",
+                "confidence": 1.0,
+                "summary": "The test stopped because its exact action was denied.",
+                "issues": [],
+            }
+            job["status"] = "done"
+            return {"ok": True, "status": "done", "decision": "deny"}
+
+        run = job.get("_run") or {}
+        transcript = job.pop("_transcript", [])
+        # stream_agent_loop owns its per-round message list internally. Rebuild
+        # continuation context from the original untrusted skill plus the
+        # accumulated transcript so repeated approvals do not lose earlier
+        # approved results, while keeping every transcript byte tainted.
+        messages = _skill_test_messages(
+            run.get("md", ""),
+            job.get("task", ""),
+        )
+        if transcript:
+            messages.append(untrusted_context_message(
+                "skill test transcript",
+                "".join(str(item) for item in transcript),
+            ))
+        messages.extend([
+            {
+                "role": "assistant",
+                "content": str(expected.get("question") or "Allow this exact action once?"),
+            },
+            {
+                "role": "user",
+                "content": (
+                    f"Approved the exact {exact_approval.pending.tool_name} "
+                    "action shown above once."
+                ),
+            },
+        ])
+        job["status"] = "running"
+        job["log"].append({
+            "type": "approval_granted",
+            "text": (
+                f"Approved exact {exact_approval.pending.tool_name} action once; "
+                "resuming test."
+            ),
+        })
+        _asyncio.create_task(_run_skill_test_job(
+            key,
+            name,
+            run.get("md", ""),
+            job.get("task", ""),
+            run.get("url"),
+            run.get("model"),
+            run.get("headers"),
+            run.get("owner"),
+            skills_manager,
+            messages=messages,
+            transcript=transcript,
+            exact_approval=exact_approval,
+        ))
+        return {"ok": True, "status": "running", "decision": "approve"}
+
     @router.get("/{skill_id}/test-status")
     async def test_skill_status(request: Request, skill_id: str):
         """Current background-test state for a skill (status / log / verdict)."""
@@ -1459,6 +1695,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
             "model": job.get("model"),
             "log": job.get("log", []),
             "verdict": job.get("verdict"),
+            "approval": job.get("approval"),
         }
 
     @router.post("/audit-all")

routes/task/__init__.py

@@ -0,0 +1,5 @@
+"""Task route domain package (slice 2p, #4082/#4071).
+
+Contains task_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/task_routes.py re-exports from here.
+"""

routes/upload_routes.py

@@ -10,16 +10,140 @@ from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
 from typing import List, Optional
 import logging
 from core.middleware import require_admin
-from core.database import SessionLocal, GalleryImage, Session as DbSession
+from core.database import (
+    SessionLocal,
+    ChatMessage as DbChatMessage,
+    CalendarCal,
+    CalendarEvent,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    Note,
+    Session as DbSession,
+)
 from src.auth_helpers import effective_user
+from src.attachment_refs import attachment_refs_from_metadata
 from src.constants import GENERATED_IMAGES_DIR
-from src.upload_handler import count_recent_uploads
+from src.upload_handler import (
+    UploadCleanupSafetyError,
+    count_recent_uploads,
+    extract_upload_ids,
+)
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/upload", tags=["upload"])
 UPLOAD_RESPONSE_HEADERS = {"X-Content-Type-Options": "nosniff"}
 
+def _upload_ids_from_persisted_text(value: object) -> set[str]:
+    """Return canonical upload IDs embedded in persisted text.
+
+    This covers attachment reference lines/URIs and the PDF source markers
+    stored by the document editor. False positives are intentionally
+    conservative: retaining an extra upload is safer than deleting referenced
+    bytes.
+    """
+    return extract_upload_ids(value)
+
+
+def _upload_ids_from_message_metadata(raw_metadata: object) -> set[str]:
+    """Extract attachment IDs from a persisted chat metadata JSON value.
+
+    Malformed metadata raises instead of being treated as an empty reference
+    set. The admin cleanup route catches that failure and aborts cleanup.
+    """
+    if raw_metadata in (None, ""):
+        return set()
+    if isinstance(raw_metadata, str):
+        metadata = json.loads(raw_metadata)
+    else:
+        metadata = raw_metadata
+    if not isinstance(metadata, dict):
+        raise ValueError("chat message metadata must be a JSON object")
+
+    attachments = metadata.get("attachments")
+    if attachments is not None:
+        if not isinstance(attachments, list) or any(
+            not isinstance(item, dict) for item in attachments
+        ):
+            raise ValueError("chat message attachments metadata is malformed")
+
+    ids = {
+        str(ref["attachment_id"])
+        for ref in attachment_refs_from_metadata(metadata)
+        if ref.get("attachment_id")
+    }
+    # Preserve canonical IDs even in older metadata shapes not normalized by
+    # attachment_refs_from_metadata().
+    ids.update(_upload_ids_from_persisted_text(json.dumps(metadata)))
+    return ids
+
+
+def _collect_persisted_upload_references() -> tuple[set[str], set[str]]:
+    """Collect upload IDs/hashes still referenced by durable application data.
+
+    The caller must treat any exception as an incomplete scan and fail closed.
+    There is no distinct artifact table in the current schema; artifact-like
+    attachment references persisted in chat/document text are covered by the
+    canonical-ID scan.
+    """
+    referenced_ids: set[str] = set()
+    referenced_hashes: set[str] = set()
+    db = SessionLocal()
+    try:
+        for content, raw_metadata in db.query(
+            DbChatMessage.content,
+            DbChatMessage.meta_data,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+            referenced_ids.update(_upload_ids_from_message_metadata(raw_metadata))
+
+        for (content,) in db.query(Document.current_content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for (content,) in db.query(DocumentVersion.content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for filename, file_hash in db.query(
+            GalleryImage.filename,
+            GalleryImage.file_hash,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(filename))
+            if file_hash:
+                referenced_hashes.add(str(file_hash))
+
+        for image_url, color, content, items in db.query(
+            Note.image_url,
+            Note.color,
+            Note.content,
+            Note.items,
+        ).yield_per(500):
+            for value in (image_url, color, content, items):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        for (color,) in db.query(CalendarCal.color).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(color))
+
+        for color, description, location in db.query(
+            CalendarEvent.color,
+            CalendarEvent.description,
+            CalendarEvent.location,
+        ).yield_per(500):
+            for value in (color, description, location):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        return referenced_ids, referenced_hashes
+    finally:
+        db.close()
+
+
+def _run_reference_safe_cleanup(upload_handler) -> int:
+    referenced_ids, referenced_hashes = _collect_persisted_upload_references()
+    return upload_handler.cleanup_old_uploads(
+        referenced_upload_ids=referenced_ids,
+        referenced_upload_hashes=referenced_hashes,
+    )
+
 def setup_upload_routes(upload_handler):
     """Setup upload routes with the provided handler"""
 
@@ -172,7 +296,9 @@ def setup_upload_routes(upload_handler):
                     "mime": meta["mime"],
                     "size": meta["size"],
                     "hash": meta["hash"],
+                    "checksum_sha256": meta.get("checksum_sha256") or meta["hash"],
                     "uploaded_at": meta["uploaded_at"],
+                    "created_at": meta.get("created_at") or meta["uploaded_at"],
                     "width": meta.get("width"),
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
@@ -195,7 +321,23 @@ def setup_upload_routes(upload_handler):
     async def manual_cleanup(request: Request):
         """Manually trigger cleanup of old uploads."""
         require_admin(request)
-        cleaned_count = upload_handler.cleanup_old_uploads()
+        try:
+            cleaned_count = await asyncio.to_thread(
+                _run_reference_safe_cleanup,
+                upload_handler,
+            )
+        except UploadCleanupSafetyError:
+            logger.exception("Upload cleanup aborted because index safety checks failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup aborted because upload index integrity could not be verified",
+            )
+        except Exception:
+            logger.exception("Upload cleanup skipped because reference discovery failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup skipped because persisted references could not be verified",
+            )
         return {"status": "success", "files_cleaned": cleaned_count}
 
     @router.get("/stats")

routes/vault/__init__.py

@@ -0,0 +1,5 @@
+"""Vault route domain package (slice 2k, #4082/#4071).
+
+Contains vault_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/vault_routes.py re-exports from here.
+"""

routes/vault/vault_routes.py

@@ -0,0 +1,242 @@
+"""
+vault_routes.py
+
+Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
+Stores the BW_SESSION key in data/vault.json with restrictive permissions.
+"""
+
+import json
+import logging
+import os
+import shutil
+import asyncio
+from pathlib import Path
+from datetime import datetime
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+
+from core.middleware import require_admin
+from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
+from src.constants import VAULT_FILE as _VAULT_FILE
+
+logger = logging.getLogger(__name__)
+
+VAULT_FILE = Path(_VAULT_FILE)
+
+
+def _find_bw() -> str:
+    """Locate the bw binary, checking PATH and common npm-global locations.
+
+    On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
+    which_tool via PATHEXT.
+    """
+    p = which_tool("bw")
+    if p:
+        return p
+    if IS_WINDOWS:
+        appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
+        for candidate in (
+            os.path.join(appdata, "npm", "bw.cmd"),
+            os.path.join(appdata, "npm", "bw.exe"),
+        ):
+            if os.path.isfile(candidate):
+                return candidate
+        return "bw"
+    home = os.path.expanduser("~")
+    for candidate in (
+        f"{home}/.npm-global/bin/bw",
+        f"{home}/.nvm/versions/node/*/bin/bw",
+        "/usr/local/bin/bw",
+        "/opt/homebrew/bin/bw",
+    ):
+        if "*" in candidate:
+            import glob
+            for m in glob.glob(candidate):
+                if os.path.isfile(m) and os.access(m, os.X_OK):
+                    return m
+        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+            return candidate
+    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
+
+
+def _load_config() -> dict:
+    if VAULT_FILE.exists():
+        try:
+            data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
+            return data if isinstance(data, dict) else {}
+        except Exception:
+            pass
+    return {}
+
+
+def _save_config(cfg: dict):
+    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
+    VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
+    # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
+    # is ACL-restricted already).
+    safe_chmod(str(VAULT_FILE), 0o600)
+
+
+async def _run_bw(args: list, session: str = None, input_text: str = None,
+                  bw_password: str = None) -> tuple:
+    env = {}
+    env.update(os.environ)
+    if session:
+        env["BW_SESSION"] = session
+    # Secrets must never be passed as argv — process arguments are world-readable
+    # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
+    # support for bw commands that need it; unlock/login callers should prefer
+    # stdin so the master password is not left in the child environment either.
+    if bw_password is not None:
+        env["BW_PASSWORD"] = bw_password
+    bw_path = _find_bw()
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bw_path, *args,
+            stdin=asyncio.subprocess.PIPE if input_text else None,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except FileNotFoundError:
+        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
+    except Exception as e:
+        return "", f"Failed to launch bw: {e}", 1
+    try:
+        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
+    except Exception as e:
+        return "", f"bw subprocess error: {e}", 1
+    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
+
+
+class VaultConfig(BaseModel):
+    server_url: str = ""
+    email: str = ""
+
+
+class VaultUnlockRequest(BaseModel):
+    master_password: str
+
+
+class VaultLoginRequest(BaseModel):
+    email: str
+    master_password: str
+
+
+def setup_vault_routes():
+    router = APIRouter(prefix="/api/vault", tags=["vault"])
+
+    @router.get("/config")
+    async def get_config(request: Request):
+        """Return vault config (no sensitive fields)."""
+        require_admin(request)
+        cfg = _load_config()
+        return {
+            "server_url": cfg.get("server_url", ""),
+            "email": cfg.get("email", ""),
+            "unlocked": bool(cfg.get("session")),
+            "unlocked_at": cfg.get("unlocked_at", ""),
+            "bw_installed": await _check_bw_installed(),
+        }
+
+    @router.post("/config")
+    async def save_config(req: VaultConfig, request: Request):
+        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg["server_url"] = req.server_url.strip().rstrip("/")
+        cfg["email"] = req.email.strip()
+
+        if cfg["server_url"]:
+            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
+            if rc != 0:
+                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
+
+        _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/login")
+    async def login(req: VaultLoginRequest, request: Request):
+        """Log in to Vaultwarden (required once per account)."""
+        require_admin(request)
+        cfg = _load_config()
+        # Update email
+        cfg["email"] = req.email
+        _save_config(cfg)
+
+        stdout, stderr, rc = await _run_bw(
+            ["login", req.email, "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            # Already logged in is OK
+            if "already logged in" in stderr.lower():
+                return {"ok": True, "already": True}
+            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
+        # bw login --raw prints session key on success (when 2FA disabled)
+        if stdout:
+            cfg["session"] = stdout
+            cfg["unlocked_at"] = datetime.utcnow().isoformat()
+            _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/unlock")
+    async def unlock(req: VaultUnlockRequest, request: Request):
+        """Unlock the vault and save the session key."""
+        require_admin(request)
+        # Pass the master password on stdin, not argv. argv is visible through
+        # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
+        # the child process environment.
+        stdout, stderr, rc = await _run_bw(
+            ["unlock", "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
+        session = stdout.strip()
+        if not session:
+            return {"ok": False, "error": "bw returned empty session"}
+        cfg = _load_config()
+        cfg["session"] = session
+        cfg["unlocked_at"] = datetime.utcnow().isoformat()
+        _save_config(cfg)
+        return {"ok": True, "message": "Vault unlocked"}
+
+    @router.post("/lock")
+    async def lock(request: Request):
+        """Lock the vault (clear session from config)."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        # Also tell bw to lock
+        await _run_bw(["lock"])
+        return {"ok": True, "message": "Vault locked"}
+
+    @router.post("/logout")
+    async def logout(request: Request):
+        """Log out of the Bitwarden CLI completely."""
+        require_admin(request)
+        await _run_bw(["logout"])
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("email", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        return {"ok": True}
+
+    return router
+
+
+async def _check_bw_installed() -> bool:
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            _find_bw(), "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.communicate()
+        return proc.returncode == 0
+    except Exception:
+        return False

routes/vault_routes.py

@@ -1,242 +1,14 @@
-"""
-vault_routes.py
+"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
 
-Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
-Stores the BW_SESSION key in data/vault.json with restrictive permissions.
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
+and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
+by test_vault_password_not_in_argv.py all operate on the *same* object.
+Keeps existing import paths working after slice 2k (#4082/#4071).
 """
 
-import json
-import logging
-import os
-import shutil
-import asyncio
-from pathlib import Path
-from datetime import datetime
-from fastapi import APIRouter, Request
-from pydantic import BaseModel
+import sys as _sys
 
-from core.middleware import require_admin
-from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
-from src.constants import VAULT_FILE as _VAULT_FILE
+from routes.vault import vault_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-VAULT_FILE = Path(_VAULT_FILE)
-
-
-def _find_bw() -> str:
-    """Locate the bw binary, checking PATH and common npm-global locations.
-
-    On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
-    which_tool via PATHEXT.
-    """
-    p = which_tool("bw")
-    if p:
-        return p
-    if IS_WINDOWS:
-        appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
-        for candidate in (
-            os.path.join(appdata, "npm", "bw.cmd"),
-            os.path.join(appdata, "npm", "bw.exe"),
-        ):
-            if os.path.isfile(candidate):
-                return candidate
-        return "bw"
-    home = os.path.expanduser("~")
-    for candidate in (
-        f"{home}/.npm-global/bin/bw",
-        f"{home}/.nvm/versions/node/*/bin/bw",
-        "/usr/local/bin/bw",
-        "/opt/homebrew/bin/bw",
-    ):
-        if "*" in candidate:
-            import glob
-            for m in glob.glob(candidate):
-                if os.path.isfile(m) and os.access(m, os.X_OK):
-                    return m
-        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
-            return candidate
-    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
-
-
-def _load_config() -> dict:
-    if VAULT_FILE.exists():
-        try:
-            data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
-            return data if isinstance(data, dict) else {}
-        except Exception:
-            pass
-    return {}
-
-
-def _save_config(cfg: dict):
-    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
-    VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
-    # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
-    # is ACL-restricted already).
-    safe_chmod(str(VAULT_FILE), 0o600)
-
-
-async def _run_bw(args: list, session: str = None, input_text: str = None,
-                  bw_password: str = None) -> tuple:
-    env = {}
-    env.update(os.environ)
-    if session:
-        env["BW_SESSION"] = session
-    # Secrets must never be passed as argv — process arguments are world-readable
-    # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
-    # support for bw commands that need it; unlock/login callers should prefer
-    # stdin so the master password is not left in the child environment either.
-    if bw_password is not None:
-        env["BW_PASSWORD"] = bw_password
-    bw_path = _find_bw()
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            bw_path, *args,
-            stdin=asyncio.subprocess.PIPE if input_text else None,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            env=env,
-        )
-    except FileNotFoundError:
-        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
-    except Exception as e:
-        return "", f"Failed to launch bw: {e}", 1
-    try:
-        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
-    except Exception as e:
-        return "", f"bw subprocess error: {e}", 1
-    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
-
-
-class VaultConfig(BaseModel):
-    server_url: str = ""
-    email: str = ""
-
-
-class VaultUnlockRequest(BaseModel):
-    master_password: str
-
-
-class VaultLoginRequest(BaseModel):
-    email: str
-    master_password: str
-
-
-def setup_vault_routes():
-    router = APIRouter(prefix="/api/vault", tags=["vault"])
-
-    @router.get("/config")
-    async def get_config(request: Request):
-        """Return vault config (no sensitive fields)."""
-        require_admin(request)
-        cfg = _load_config()
-        return {
-            "server_url": cfg.get("server_url", ""),
-            "email": cfg.get("email", ""),
-            "unlocked": bool(cfg.get("session")),
-            "unlocked_at": cfg.get("unlocked_at", ""),
-            "bw_installed": await _check_bw_installed(),
-        }
-
-    @router.post("/config")
-    async def save_config(req: VaultConfig, request: Request):
-        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg["server_url"] = req.server_url.strip().rstrip("/")
-        cfg["email"] = req.email.strip()
-
-        if cfg["server_url"]:
-            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
-            if rc != 0:
-                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
-
-        _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/login")
-    async def login(req: VaultLoginRequest, request: Request):
-        """Log in to Vaultwarden (required once per account)."""
-        require_admin(request)
-        cfg = _load_config()
-        # Update email
-        cfg["email"] = req.email
-        _save_config(cfg)
-
-        stdout, stderr, rc = await _run_bw(
-            ["login", req.email, "--raw"],
-            input_text=req.master_password + "\n",
-        )
-        if rc != 0:
-            # Already logged in is OK
-            if "already logged in" in stderr.lower():
-                return {"ok": True, "already": True}
-            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
-        # bw login --raw prints session key on success (when 2FA disabled)
-        if stdout:
-            cfg["session"] = stdout
-            cfg["unlocked_at"] = datetime.utcnow().isoformat()
-            _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/unlock")
-    async def unlock(req: VaultUnlockRequest, request: Request):
-        """Unlock the vault and save the session key."""
-        require_admin(request)
-        # Pass the master password on stdin, not argv. argv is visible through
-        # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
-        # the child process environment.
-        stdout, stderr, rc = await _run_bw(
-            ["unlock", "--raw"],
-            input_text=req.master_password + "\n",
-        )
-        if rc != 0:
-            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
-        session = stdout.strip()
-        if not session:
-            return {"ok": False, "error": "bw returned empty session"}
-        cfg = _load_config()
-        cfg["session"] = session
-        cfg["unlocked_at"] = datetime.utcnow().isoformat()
-        _save_config(cfg)
-        return {"ok": True, "message": "Vault unlocked"}
-
-    @router.post("/lock")
-    async def lock(request: Request):
-        """Lock the vault (clear session from config)."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        # Also tell bw to lock
-        await _run_bw(["lock"])
-        return {"ok": True, "message": "Vault locked"}
-
-    @router.post("/logout")
-    async def logout(request: Request):
-        """Log out of the Bitwarden CLI completely."""
-        require_admin(request)
-        await _run_bw(["logout"])
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("email", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        return {"ok": True}
-
-    return router
-
-
-async def _check_bw_installed() -> bool:
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            _find_bw(), "--version",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        await proc.communicate()
-        return proc.returncode == 0
-    except Exception:
-        return False
+_sys.modules[__name__] = _canonical

routes/webhook/__init__.py

@@ -0,0 +1,5 @@
+"""Webhook route domain package (slice 2l, #4082/#4071).
+
+Contains webhook_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/webhook_routes.py re-exports from here.
+"""

routes/webhook/webhook_routes.py

@@ -0,0 +1,395 @@
+"""Webhook, API Token, and sync chat routes."""
+
+import uuid
+import logging
+from typing import Optional
+
+import httpx
+from fastapi import APIRouter, HTTPException, Request, Form
+from pydantic import BaseModel, Field
+
+from core.database import SessionLocal, Webhook, ModelEndpoint
+from src.auth_helpers import owner_filter
+from src.url_security import validate_public_http_url
+from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api", tags=["webhooks"])
+
+# Input limits
+MAX_NAME_LEN = 100
+MAX_URL_LEN = 2048
+MAX_SECRET_LEN = 256
+MAX_MESSAGE_LEN = 32_000
+
+
+from core.middleware import require_admin as _require_admin
+
+
+def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
+    """First enabled ModelEndpoint visible to token_owner — their own rows plus
+    legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
+    let a chat-scoped token fall back onto another user's private endpoint and
+    silently spend that owner's API key/quota. Prefer owner rows before shared
+    rows. Fails closed to null-owner rows only when token_owner is absent.
+    Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
+    """
+    query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if token_owner:
+        query = owner_filter(query, ModelEndpoint, token_owner)
+        return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
+    return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first()  # noqa: E711
+
+
+def _caller_owns_session(sess_owner, caller) -> bool:
+    """Strict session-ownership gate for the token-authenticated sync-chat
+    endpoint (`POST /api/v1/chat`).
+
+    Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
+    gates in notes/calendar/gallery: a caller may resume a session ONLY when
+    its owner matches them exactly. A null/empty session owner (legacy or
+    migrated rows) is deliberately NOT resumable by an arbitrary token — the
+    old ``sess_owner and sess_owner != caller`` form skipped the check whenever
+    ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
+    device) could resume such a session, inject a message, and read back its
+    history and reuse the owner's endpoint credentials. Fail closed: an
+    unresolvable caller also returns False.
+    """
+    if not caller:
+        return False
+    return sess_owner == caller
+
+
+def setup_webhook_routes(
+    webhook_manager: WebhookManager,
+    auth_manager,
+    session_manager=None,
+    api_key_manager=None,
+) -> APIRouter:
+
+    @router.get("/webhooks")
+    def list_webhooks(request: Request):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            hooks = db.query(Webhook).all()
+            return [
+                {
+                    "id": w.id,
+                    "name": w.name,
+                    "url": w.url,
+                    "has_secret": bool(w.secret),
+                    "events": w.events.split(",") if w.events else [],
+                    "is_active": w.is_active,
+                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
+                    "last_status_code": w.last_status_code,
+                    "last_error": w.last_error,
+                    "created_at": w.created_at.isoformat() if w.created_at else None,
+                }
+                for w in hooks
+            ]
+        finally:
+            db.close()
+
+    @router.post("/webhooks")
+    def create_webhook(
+        request: Request,
+        name: str = Form(""),
+        url: str = Form(""),
+        secret: str = Form(""),
+        events: str = Form(""),
+    ):
+        _require_admin(request)
+        name = name.strip()[:MAX_NAME_LEN]
+        if not name:
+            raise HTTPException(400, "Webhook name is required")
+        try:
+            url = validate_webhook_url(url)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+        try:
+            events = validate_events(events)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+
+        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
+        # Encrypt the secret at rest using the same Fernet key as API keys
+        encrypted_secret = None
+        if secret_val and api_key_manager:
+            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
+        elif secret_val:
+            encrypted_secret = secret_val  # Fallback if no encryption available
+
+        webhook_id = str(uuid.uuid4())[:8]
+        db = SessionLocal()
+        try:
+            db.add(Webhook(
+                id=webhook_id,
+                name=name,
+                url=url,
+                secret=encrypted_secret,
+                events=events,
+                is_active=True,
+            ))
+            db.commit()
+        finally:
+            db.close()
+
+        return {"id": webhook_id, "name": name}
+
+    @router.post("/webhooks/{webhook_id}/test")
+    async def test_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            url, secret = wh.url, wh.secret
+        finally:
+            db.close()
+
+        await webhook_manager.deliver_test(webhook_id, url, secret)
+        return {"status": "sent"}
+
+    @router.patch("/webhooks/{webhook_id}")
+    def toggle_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            wh.is_active = not wh.is_active
+            db.commit()
+            return {"id": webhook_id, "is_active": wh.is_active}
+        finally:
+            db.close()
+
+    @router.delete("/webhooks/{webhook_id}")
+    def delete_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
+            db.commit()
+            if not deleted:
+                raise HTTPException(404, "Webhook not found")
+        finally:
+            db.close()
+        return {"status": "deleted"}
+
+    # ================================================================
+    # Sync Chat Endpoint (for n8n / Make / Activepieces)
+    # ================================================================
+
+    # Known provider base URLs — auto-resolved from api_key prefix or model name
+    KNOWN_PROVIDERS = {
+        "deepseek": "https://api.deepseek.com/v1",
+        "openai": "https://api.openai.com/v1",
+        "mistral": "https://api.mistral.ai/v1",
+        "groq": "https://api.groq.com/openai/v1",
+        "together": "https://api.together.xyz/v1",
+        "openrouter": "https://openrouter.ai/api/v1",
+        "ollama": "https://ollama.com/api",
+        "opencode-zen": "https://opencode.ai/zen/v1",
+        "opencode-go": "https://opencode.ai/zen/go/v1",
+        "fireworks": "https://api.fireworks.ai/inference/v1",
+        "venice": "https://api.venice.ai/api/v1",
+        "kimi-code": "https://api.kimi.com/coding/v1",
+        "kimicode": "https://api.kimi.com/coding/v1",
+    }
+
+    # Model prefix → provider mapping for auto-detection
+    MODEL_PROVIDER_MAP = {
+        "deepseek": "deepseek",
+        "gpt-": "openai",
+        "o1": "openai",
+        "o3": "openai",
+        "o4": "openai",
+        "mistral": "mistral",
+        "llama": "groq",
+        "mixtral": "groq",
+        "kimi-for-coding": "kimi-code",
+        "kimi": "kimi-code",
+    }
+
+    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
+        """Try to auto-resolve a base URL from provider name or model prefix."""
+        if provider and provider.lower() in KNOWN_PROVIDERS:
+            return KNOWN_PROVIDERS[provider.lower()]
+        if model:
+            model_lower = model.lower()
+            for prefix, prov in MODEL_PROVIDER_MAP.items():
+                if model_lower.startswith(prefix):
+                    return KNOWN_PROVIDERS[prov]
+        return None
+
+    class SyncChatRequest(BaseModel):
+        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
+        model: Optional[str] = Field(None, max_length=200)
+        session: Optional[str] = Field(None, max_length=100)
+        api_key: Optional[str] = Field(None, max_length=256)
+        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
+        provider: Optional[str] = Field(None, max_length=50)
+
+    @router.post("/v1/chat")
+    async def sync_chat(request: Request, body: SyncChatRequest):
+        if not getattr(request.state, "api_token", False):
+            raise HTTPException(403, "This endpoint requires an API token")
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        if "chat" not in scopes:
+            raise HTTPException(403, "API token is not scoped for chat")
+        token_owner = getattr(request.state, "api_token_owner", None)
+
+        from core.models import ChatMessage
+        from src.llm_core import llm_call_async
+        from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
+
+        message = body.message.strip()
+        if not message:
+            raise HTTPException(400, "Message is required")
+
+        session_id = body.session
+        sess = None
+
+        # --- Case 1: Resume an existing session ---
+        if session_id and session_manager:
+            try:
+                sess = session_manager.get_session(session_id)
+            except (KeyError, Exception):
+                raise HTTPException(404, "Session not found")
+            # SECURITY: verify the API-token's user owns this session — without
+            # this any token holder could resume any user's chat by passing its
+            # ID. The token's user is on request.state.user (set by API-token
+            # middleware); fall back to require_user if not present.
+            try:
+                from src.auth_helpers import get_current_user as _gcu
+                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
+            except Exception:
+                _tok_user = None
+            # Strict ownership (see _caller_owns_session): fail closed so a
+            # null-owner / cross-owner session can't be resumed by an arbitrary
+            # chat-scoped token.
+            _sess_owner = getattr(sess, "owner", None)
+            if not _caller_owns_session(_sess_owner, _tok_user):
+                raise HTTPException(404, "Session not found")
+
+        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
+        if not sess and body.api_key:
+            api_key = body.api_key.strip()
+            model = body.model or "deepseek-chat"
+
+            # Validate only token-supplied direct base_url; auto-resolved known-provider
+            # URLs are not subject to extra local/LAN blocking beyond existing provider logic.
+            direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
+            if direct_base_url:
+                try:
+                    base_url = validate_public_http_url(direct_base_url)
+                except ValueError as e:
+                    detail = str(e).replace("URL", "base_url", 1)
+                    raise HTTPException(400, detail)
+            else:
+                base_url = _resolve_base_url(model, body.provider)
+            if not base_url:
+                raise HTTPException(400,
+                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
+                    "or provider ('deepseek', 'openai', 'groq', etc.)")
+            base_url = normalize_base(base_url)
+            endpoint_url = build_chat_url(base_url)
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            sess.headers = build_headers(api_key, base_url)
+            session_manager.save_sessions()
+            session_id = sid
+
+        # --- Case 3: Fall back to first configured ModelEndpoint ---
+        if not sess:
+            db = SessionLocal()
+            try:
+                ep = _select_api_chat_fallback_endpoint(db, token_owner)
+            finally:
+                db.close()
+
+            if not ep:
+                raise HTTPException(400,
+                    "No session, api_key, or configured endpoints. "
+                    "Pass api_key + model, or configure an endpoint in Admin.")
+
+            base_url = normalize_base(ep.base_url)
+            endpoint_url = build_chat_url(base_url)
+            model = body.model or "auto"
+            api_key = ep.api_key
+            if getattr(ep, "provider_auth_id", None):
+                try:
+                    from src.endpoint_resolver import resolve_endpoint_runtime
+                    base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
+                    endpoint_url = build_chat_url(base_url)
+                except Exception:
+                    raise HTTPException(500, "Could not resolve endpoint credentials")
+
+            if model == "auto":
+                try:
+                    async with httpx.AsyncClient(timeout=5) as client:
+                        models_url = build_models_url(base_url)
+                        hdrs = build_headers(api_key, base_url)
+                        if models_url:
+                            resp = await client.get(models_url, headers=hdrs)
+                            resp.raise_for_status()
+                            data = resp.json()
+                            items = data if isinstance(data, list) else (data.get("data") or [])
+                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
+                            if not ids and isinstance(data, dict):
+                                ids = [
+                                    m.get("name") or m.get("model")
+                                    for m in (data.get("models") or [])
+                                    if m.get("name") or m.get("model")
+                                ]
+                        else:
+                            import json as _json
+                            ids = _json.loads(ep.cached_models or "[]")
+                        model = ids[0] if ids else "auto"
+                except Exception:
+                    raise HTTPException(500, "Could not discover models from endpoint")
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            if api_key:
+                sess.headers = build_headers(api_key, base_url)
+                session_manager.save_sessions()
+            session_id = sid
+
+        # --- Send message and get response ---
+        sess.add_message(ChatMessage("user", message))
+
+        messages = [{"role": m.role, "content": m.content} for m in sess.history]
+
+        reply = await llm_call_async(
+            sess.endpoint_url, sess.model, messages,
+            headers=sess.headers, timeout=120,
+        )
+        sess.add_message(ChatMessage("assistant", reply))
+        session_manager.save_sessions()
+
+        webhook_manager.fire_and_forget("chat.completed", {
+            "session_id": session_id, "model": sess.model,
+            "user_message": message[:2000], "response": reply[:2000],
+        })
+
+        return {"response": reply, "session_id": session_id, "model": sess.model}
+
+    return router

routes/webhook_routes.py

@@ -1,395 +1,16 @@
-"""Webhook, API Token, and sync chat routes."""
+"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
 
-import uuid
-import logging
-from typing import Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
+``importlib.import_module("routes.webhook_routes")``, and the
+``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
+...)`` pattern used by test_null_owner_gates.py all operate on the *same*
+object. Keeps existing import paths working after slice 2l (#4082/#4071).
+Source-introspection tests read the canonical file by path.
+"""
 
-import httpx
-from fastapi import APIRouter, HTTPException, Request, Form
-from pydantic import BaseModel, Field
+import sys as _sys
 
-from core.database import SessionLocal, Webhook, ModelEndpoint
-from src.auth_helpers import owner_filter
-from src.url_security import validate_public_http_url
-from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
+from routes.webhook import webhook_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/api", tags=["webhooks"])
-
-# Input limits
-MAX_NAME_LEN = 100
-MAX_URL_LEN = 2048
-MAX_SECRET_LEN = 256
-MAX_MESSAGE_LEN = 32_000
-
-
-from core.middleware import require_admin as _require_admin
-
-
-def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
-    """First enabled ModelEndpoint visible to token_owner — their own rows plus
-    legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
-    let a chat-scoped token fall back onto another user's private endpoint and
-    silently spend that owner's API key/quota. Prefer owner rows before shared
-    rows. Fails closed to null-owner rows only when token_owner is absent.
-    Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
-    """
-    query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
-    if token_owner:
-        query = owner_filter(query, ModelEndpoint, token_owner)
-        return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
-    return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first()  # noqa: E711
-
-
-def _caller_owns_session(sess_owner, caller) -> bool:
-    """Strict session-ownership gate for the token-authenticated sync-chat
-    endpoint (`POST /api/v1/chat`).
-
-    Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
-    gates in notes/calendar/gallery: a caller may resume a session ONLY when
-    its owner matches them exactly. A null/empty session owner (legacy or
-    migrated rows) is deliberately NOT resumable by an arbitrary token — the
-    old ``sess_owner and sess_owner != caller`` form skipped the check whenever
-    ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
-    device) could resume such a session, inject a message, and read back its
-    history and reuse the owner's endpoint credentials. Fail closed: an
-    unresolvable caller also returns False.
-    """
-    if not caller:
-        return False
-    return sess_owner == caller
-
-
-def setup_webhook_routes(
-    webhook_manager: WebhookManager,
-    auth_manager,
-    session_manager=None,
-    api_key_manager=None,
-) -> APIRouter:
-
-    @router.get("/webhooks")
-    def list_webhooks(request: Request):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            hooks = db.query(Webhook).all()
-            return [
-                {
-                    "id": w.id,
-                    "name": w.name,
-                    "url": w.url,
-                    "has_secret": bool(w.secret),
-                    "events": w.events.split(",") if w.events else [],
-                    "is_active": w.is_active,
-                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
-                    "last_status_code": w.last_status_code,
-                    "last_error": w.last_error,
-                    "created_at": w.created_at.isoformat() if w.created_at else None,
-                }
-                for w in hooks
-            ]
-        finally:
-            db.close()
-
-    @router.post("/webhooks")
-    def create_webhook(
-        request: Request,
-        name: str = Form(""),
-        url: str = Form(""),
-        secret: str = Form(""),
-        events: str = Form(""),
-    ):
-        _require_admin(request)
-        name = name.strip()[:MAX_NAME_LEN]
-        if not name:
-            raise HTTPException(400, "Webhook name is required")
-        try:
-            url = validate_webhook_url(url)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-        try:
-            events = validate_events(events)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-
-        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
-        # Encrypt the secret at rest using the same Fernet key as API keys
-        encrypted_secret = None
-        if secret_val and api_key_manager:
-            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
-        elif secret_val:
-            encrypted_secret = secret_val  # Fallback if no encryption available
-
-        webhook_id = str(uuid.uuid4())[:8]
-        db = SessionLocal()
-        try:
-            db.add(Webhook(
-                id=webhook_id,
-                name=name,
-                url=url,
-                secret=encrypted_secret,
-                events=events,
-                is_active=True,
-            ))
-            db.commit()
-        finally:
-            db.close()
-
-        return {"id": webhook_id, "name": name}
-
-    @router.post("/webhooks/{webhook_id}/test")
-    async def test_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            url, secret = wh.url, wh.secret
-        finally:
-            db.close()
-
-        await webhook_manager.deliver_test(webhook_id, url, secret)
-        return {"status": "sent"}
-
-    @router.patch("/webhooks/{webhook_id}")
-    def toggle_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            wh.is_active = not wh.is_active
-            db.commit()
-            return {"id": webhook_id, "is_active": wh.is_active}
-        finally:
-            db.close()
-
-    @router.delete("/webhooks/{webhook_id}")
-    def delete_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
-            db.commit()
-            if not deleted:
-                raise HTTPException(404, "Webhook not found")
-        finally:
-            db.close()
-        return {"status": "deleted"}
-
-    # ================================================================
-    # Sync Chat Endpoint (for n8n / Make / Activepieces)
-    # ================================================================
-
-    # Known provider base URLs — auto-resolved from api_key prefix or model name
-    KNOWN_PROVIDERS = {
-        "deepseek": "https://api.deepseek.com/v1",
-        "openai": "https://api.openai.com/v1",
-        "mistral": "https://api.mistral.ai/v1",
-        "groq": "https://api.groq.com/openai/v1",
-        "together": "https://api.together.xyz/v1",
-        "openrouter": "https://openrouter.ai/api/v1",
-        "ollama": "https://ollama.com/api",
-        "opencode-zen": "https://opencode.ai/zen/v1",
-        "opencode-go": "https://opencode.ai/zen/go/v1",
-        "fireworks": "https://api.fireworks.ai/inference/v1",
-        "venice": "https://api.venice.ai/api/v1",
-        "kimi-code": "https://api.kimi.com/coding/v1",
-        "kimicode": "https://api.kimi.com/coding/v1",
-    }
-
-    # Model prefix → provider mapping for auto-detection
-    MODEL_PROVIDER_MAP = {
-        "deepseek": "deepseek",
-        "gpt-": "openai",
-        "o1": "openai",
-        "o3": "openai",
-        "o4": "openai",
-        "mistral": "mistral",
-        "llama": "groq",
-        "mixtral": "groq",
-        "kimi-for-coding": "kimi-code",
-        "kimi": "kimi-code",
-    }
-
-    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
-        """Try to auto-resolve a base URL from provider name or model prefix."""
-        if provider and provider.lower() in KNOWN_PROVIDERS:
-            return KNOWN_PROVIDERS[provider.lower()]
-        if model:
-            model_lower = model.lower()
-            for prefix, prov in MODEL_PROVIDER_MAP.items():
-                if model_lower.startswith(prefix):
-                    return KNOWN_PROVIDERS[prov]
-        return None
-
-    class SyncChatRequest(BaseModel):
-        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
-        model: Optional[str] = Field(None, max_length=200)
-        session: Optional[str] = Field(None, max_length=100)
-        api_key: Optional[str] = Field(None, max_length=256)
-        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
-        provider: Optional[str] = Field(None, max_length=50)
-
-    @router.post("/v1/chat")
-    async def sync_chat(request: Request, body: SyncChatRequest):
-        if not getattr(request.state, "api_token", False):
-            raise HTTPException(403, "This endpoint requires an API token")
-        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
-        if "chat" not in scopes:
-            raise HTTPException(403, "API token is not scoped for chat")
-        token_owner = getattr(request.state, "api_token_owner", None)
-
-        from core.models import ChatMessage
-        from src.llm_core import llm_call_async
-        from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
-
-        message = body.message.strip()
-        if not message:
-            raise HTTPException(400, "Message is required")
-
-        session_id = body.session
-        sess = None
-
-        # --- Case 1: Resume an existing session ---
-        if session_id and session_manager:
-            try:
-                sess = session_manager.get_session(session_id)
-            except (KeyError, Exception):
-                raise HTTPException(404, "Session not found")
-            # SECURITY: verify the API-token's user owns this session — without
-            # this any token holder could resume any user's chat by passing its
-            # ID. The token's user is on request.state.user (set by API-token
-            # middleware); fall back to require_user if not present.
-            try:
-                from src.auth_helpers import get_current_user as _gcu
-                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
-            except Exception:
-                _tok_user = None
-            # Strict ownership (see _caller_owns_session): fail closed so a
-            # null-owner / cross-owner session can't be resumed by an arbitrary
-            # chat-scoped token.
-            _sess_owner = getattr(sess, "owner", None)
-            if not _caller_owns_session(_sess_owner, _tok_user):
-                raise HTTPException(404, "Session not found")
-
-        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
-        if not sess and body.api_key:
-            api_key = body.api_key.strip()
-            model = body.model or "deepseek-chat"
-
-            # Validate only token-supplied direct base_url; auto-resolved known-provider
-            # URLs are not subject to extra local/LAN blocking beyond existing provider logic.
-            direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
-            if direct_base_url:
-                try:
-                    base_url = validate_public_http_url(direct_base_url)
-                except ValueError as e:
-                    detail = str(e).replace("URL", "base_url", 1)
-                    raise HTTPException(400, detail)
-            else:
-                base_url = _resolve_base_url(model, body.provider)
-            if not base_url:
-                raise HTTPException(400,
-                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
-                    "or provider ('deepseek', 'openai', 'groq', etc.)")
-            base_url = normalize_base(base_url)
-            endpoint_url = build_chat_url(base_url)
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            sess.headers = build_headers(api_key, base_url)
-            session_manager.save_sessions()
-            session_id = sid
-
-        # --- Case 3: Fall back to first configured ModelEndpoint ---
-        if not sess:
-            db = SessionLocal()
-            try:
-                ep = _select_api_chat_fallback_endpoint(db, token_owner)
-            finally:
-                db.close()
-
-            if not ep:
-                raise HTTPException(400,
-                    "No session, api_key, or configured endpoints. "
-                    "Pass api_key + model, or configure an endpoint in Admin.")
-
-            base_url = normalize_base(ep.base_url)
-            endpoint_url = build_chat_url(base_url)
-            model = body.model or "auto"
-            api_key = ep.api_key
-            if getattr(ep, "provider_auth_id", None):
-                try:
-                    from src.endpoint_resolver import resolve_endpoint_runtime
-                    base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
-                    endpoint_url = build_chat_url(base_url)
-                except Exception:
-                    raise HTTPException(500, "Could not resolve endpoint credentials")
-
-            if model == "auto":
-                try:
-                    async with httpx.AsyncClient(timeout=5) as client:
-                        models_url = build_models_url(base_url)
-                        hdrs = build_headers(api_key, base_url)
-                        if models_url:
-                            resp = await client.get(models_url, headers=hdrs)
-                            resp.raise_for_status()
-                            data = resp.json()
-                            items = data if isinstance(data, list) else (data.get("data") or [])
-                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
-                            if not ids and isinstance(data, dict):
-                                ids = [
-                                    m.get("name") or m.get("model")
-                                    for m in (data.get("models") or [])
-                                    if m.get("name") or m.get("model")
-                                ]
-                        else:
-                            import json as _json
-                            ids = _json.loads(ep.cached_models or "[]")
-                        model = ids[0] if ids else "auto"
-                except Exception:
-                    raise HTTPException(500, "Could not discover models from endpoint")
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            if api_key:
-                sess.headers = build_headers(api_key, base_url)
-                session_manager.save_sessions()
-            session_id = sid
-
-        # --- Send message and get response ---
-        sess.add_message(ChatMessage("user", message))
-
-        messages = [{"role": m.role, "content": m.content} for m in sess.history]
-
-        reply = await llm_call_async(
-            sess.endpoint_url, sess.model, messages,
-            headers=sess.headers, timeout=120,
-        )
-        sess.add_message(ChatMessage("assistant", reply))
-        session_manager.save_sessions()
-
-        webhook_manager.fire_and_forget("chat.completed", {
-            "session_id": session_id, "model": sess.model,
-            "user_message": message[:2000], "response": reply[:2000],
-        })
-
-        return {"response": reply, "session_id": session_id, "model": sess.model}
-
-    return router
+_sys.modules[__name__] = _canonical