Squash Odysseus development history

routes/_validators.py

@@ -0,0 +1,31 @@
+import re
+
+from fastapi import HTTPException
+
+
+_REMOTE_HOST_RE = re.compile(
+    r"^(?:[A-Za-z0-9][A-Za-z0-9._-]*@)?[A-Za-z0-9][A-Za-z0-9._-]*$"
+)
+_SSH_PORT_RE = re.compile(r"^\d{1,5}$")
+
+
+def validate_remote_host(v: str | None) -> str | None:
+    if v is None or v == "":
+        return None
+    if not _REMOTE_HOST_RE.match(v):
+        raise HTTPException(
+            400,
+            "Invalid remote_host — must be host or user@host, no SSH option syntax",
+        )
+    return v
+
+
+def validate_ssh_port(v: str | None) -> str | None:
+    if v is None or v == "":
+        return None
+    if not _SSH_PORT_RE.fullmatch(str(v)):
+        raise HTTPException(400, "Invalid ssh_port")
+    port = int(v)
+    if port < 1 or port > 65535:
+        raise HTTPException(400, "Invalid ssh_port")
+    return str(port)

routes/admin_wipe/__init__.py

@@ -0,0 +1,5 @@
+"""Admin wipe route domain package (slice 2h, #4082/#4071).
+
+Contains admin_wipe_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/admin_wipe_routes.py re-exports from here.
+"""

routes/admin_wipe/admin_wipe_routes.py

@@ -0,0 +1,176 @@
+"""Admin Danger Zone — per-category wipes.
+
+Each endpoint is admin-only and truncates exactly one domain so the
+user can selectively reset memory / skills / notes / etc. without
+nuking everything. The catch-all `chats` endpoint mirrors the
+existing /api/sessions/all so the Danger Zone speaks one URL pattern.
+
+URL shape: DELETE /api/admin/wipe/{kind}
+Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar.
+"""
+
+import json
+import logging
+import os
+import shutil
+from fastapi import APIRouter, HTTPException, Request
+
+from core.middleware import require_admin
+from core.database import (
+    SessionLocal,
+    Session as DbSession,
+    ChatMessage as DbChatMessage,
+    Memory,
+    Note,
+    ScheduledTask,
+    TaskRun,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    GalleryAlbum,
+    CalendarEvent,
+    CalendarCal,
+)
+from src.constants import DATA_DIR, SKILLS_DIR, SKILLS_FILE, GALLERY_DIR, GALLERY_UPLOADS_DIR
+
+logger = logging.getLogger(__name__)
+
+
+def _wipe_memory_files():
+    """Blank memory.json + drop the per-owner tidy-state sidecar so the
+    next audit doesn't try to diff against gone memories."""
+    for name in ("memory.json", "memory_tidy_state.json"):
+        p = os.path.join(DATA_DIR, name)
+        if not os.path.exists(p):
+            continue
+        try:
+            if name == "memory.json":
+                with open(p, "w", encoding="utf-8") as f:
+                    json.dump([], f)
+            else:
+                os.remove(p)
+        except OSError as e:
+            logger.warning(f"Could not reset {name}: {e}")
+
+
+def _rmtree_quiet(path: str):
+    """rmtree that doesn't crash if the path doesn't exist."""
+    if os.path.isdir(path):
+        try:
+            shutil.rmtree(path)
+        except OSError as e:
+            logger.warning(f"Could not remove {path}: {e}")
+
+
+def setup_admin_wipe_routes(session_manager):
+    """The session_manager is passed in so we can also clear its
+    in-memory cache when wiping chats — without it the DB is empty
+    but the next /api/sessions returns stale entries."""
+    router = APIRouter(prefix="/api/admin")
+
+    @router.delete("/wipe/{kind}")
+    def wipe(kind: str, request: Request):
+        require_admin(request)
+        kind = (kind or "").strip().lower()
+
+        db = SessionLocal()
+        try:
+            if kind == "chats":
+                count = db.query(DbSession).count()
+                db.query(DbChatMessage).delete()
+                db.query(DbSession).delete()
+                db.commit()
+                try:
+                    session_manager.sessions.clear()
+                except Exception:
+                    pass
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "memory":
+                count = db.query(Memory).count()
+                db.query(Memory).delete()
+                db.commit()
+                _wipe_memory_files()
+                # Drop the vector store too so semantic search doesn't
+                # return ghosts. Lazy import — chromadb may not be
+                # initialised in every deployment.
+                try:
+                    from src.memory_vector import get_memory_vector_store
+                    mv = get_memory_vector_store()
+                    if mv and hasattr(mv, "clear"):
+                        mv.clear()
+                except Exception as e:
+                    logger.info(f"Memory vector clear skipped: {e}")
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "skills":
+                # Skills live as SKILL.md files under data/skills/. Drop
+                # the entire directory; the SkillsManager re-creates the
+                # tree on next write.
+                skills_dir = SKILLS_DIR
+                count = 0
+                if os.path.isdir(skills_dir):
+                    # Count SKILL.md files for the response — quick walk.
+                    for _, _, files in os.walk(skills_dir):
+                        count += sum(1 for f in files if f == "SKILL.md")
+                    _rmtree_quiet(skills_dir)
+                # Legacy fallback file
+                legacy = SKILLS_FILE
+                if os.path.exists(legacy):
+                    try:
+                        os.remove(legacy)
+                    except OSError:
+                        pass
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "notes":
+                count = db.query(Note).count()
+                db.query(Note).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "tasks":
+                # TaskRun rows reference tasks via FK — clear them first.
+                db.query(TaskRun).delete()
+                count = db.query(ScheduledTask).count()
+                db.query(ScheduledTask).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "documents":
+                # DocumentVersion FKs Document — clear children first.
+                db.query(DocumentVersion).delete()
+                count = db.query(Document).count()
+                db.query(Document).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "gallery":
+                count = db.query(GalleryImage).count() + db.query(GalleryAlbum).count()
+                db.query(GalleryImage).delete()
+                db.query(GalleryAlbum).delete()
+                db.commit()
+                # Also drop the upload dir so disk doesn't keep orphans.
+                _rmtree_quiet(GALLERY_DIR)
+                _rmtree_quiet(GALLERY_UPLOADS_DIR)
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            if kind == "calendar":
+                # Events FK calendars — clear children first, then both.
+                db.query(CalendarEvent).delete()
+                count = db.query(CalendarCal).count()
+                db.query(CalendarCal).delete()
+                db.commit()
+                return {"status": "deleted", "kind": kind, "count": count}
+
+            raise HTTPException(400, f"Unknown wipe kind: {kind!r}")
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.exception(f"Wipe {kind} failed")
+            raise HTTPException(500, f"Wipe {kind} failed: {e}")
+        finally:
+            db.close()
+
+    return router

routes/admin_wipe_routes.py

@@ -1,174 +1,17 @@
-"""Admin Danger Zone — per-category wipes.
+"""Backward-compat shim — canonical location is routes/admin_wipe/admin_wipe_routes.py.
 
-Each endpoint is admin-only and truncates exactly one domain so the
-user can selectively reset memory / skills / notes / etc. without
-nuking everything. The catch-all `chats` endpoint mirrors the
-existing /api/sessions/all so the Danger Zone speaks one URL pattern.
-
-URL shape: DELETE /api/admin/wipe/{kind}
-Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar.
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.admin_wipe_routes``, ``from routes.admin_wipe_routes
+import X``, ``importlib.import_module("routes.admin_wipe_routes")``, and the
+``import ... as admin_wipe_routes`` + ``monkeypatch.setattr(admin_wipe_routes,
+"SessionLocal", ...)`` / ``"require_admin"`` pattern used by
+test_admin_wipe_gallery.py all operate on the *same* object the application
+actually uses. Keeps existing import paths working after slice 2h
+(#4082/#4071).
 """
 
-import json
-import logging
-import os
-import shutil
-from fastapi import APIRouter, HTTPException, Request
+import sys as _sys
 
-from core.middleware import require_admin
-from core.database import (
-    SessionLocal,
-    Session as DbSession,
-    ChatMessage as DbChatMessage,
-    Memory,
-    Note,
-    ScheduledTask,
-    TaskRun,
-    Document,
-    DocumentVersion,
-    GalleryImage,
-    CalendarEvent,
-    CalendarCal,
-)
-from src.constants import DATA_DIR
+from routes.admin_wipe import admin_wipe_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-def _wipe_memory_files():
-    """Blank memory.json + drop the per-owner tidy-state sidecar so the
-    next audit doesn't try to diff against gone memories."""
-    for name in ("memory.json", "memory_tidy_state.json"):
-        p = os.path.join(DATA_DIR, name)
-        if not os.path.exists(p):
-            continue
-        try:
-            if name == "memory.json":
-                with open(p, "w") as f:
-                    json.dump([], f)
-            else:
-                os.remove(p)
-        except OSError as e:
-            logger.warning(f"Could not reset {name}: {e}")
-
-
-def _rmtree_quiet(path: str):
-    """rmtree that doesn't crash if the path doesn't exist."""
-    if os.path.isdir(path):
-        try:
-            shutil.rmtree(path)
-        except OSError as e:
-            logger.warning(f"Could not remove {path}: {e}")
-
-
-def setup_admin_wipe_routes(session_manager):
-    """The session_manager is passed in so we can also clear its
-    in-memory cache when wiping chats — without it the DB is empty
-    but the next /api/sessions returns stale entries."""
-    router = APIRouter(prefix="/api/admin")
-
-    @router.delete("/wipe/{kind}")
-    def wipe(kind: str, request: Request):
-        require_admin(request)
-        kind = (kind or "").strip().lower()
-
-        db = SessionLocal()
-        try:
-            if kind == "chats":
-                count = db.query(DbSession).count()
-                db.query(DbChatMessage).delete()
-                db.query(DbSession).delete()
-                db.commit()
-                try:
-                    session_manager.sessions.clear()
-                except Exception:
-                    pass
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "memory":
-                count = db.query(Memory).count()
-                db.query(Memory).delete()
-                db.commit()
-                _wipe_memory_files()
-                # Drop the vector store too so semantic search doesn't
-                # return ghosts. Lazy import — chromadb may not be
-                # initialised in every deployment.
-                try:
-                    from src.memory_vector import get_memory_vector_store
-                    mv = get_memory_vector_store()
-                    if mv and hasattr(mv, "clear"):
-                        mv.clear()
-                except Exception as e:
-                    logger.info(f"Memory vector clear skipped: {e}")
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "skills":
-                # Skills live as SKILL.md files under data/skills/. Drop
-                # the entire directory; the SkillsManager re-creates the
-                # tree on next write.
-                skills_dir = os.path.join(DATA_DIR, "skills")
-                count = 0
-                if os.path.isdir(skills_dir):
-                    # Count SKILL.md files for the response — quick walk.
-                    for _, _, files in os.walk(skills_dir):
-                        count += sum(1 for f in files if f == "SKILL.md")
-                    _rmtree_quiet(skills_dir)
-                # Legacy fallback file
-                legacy = os.path.join(DATA_DIR, "skills.json")
-                if os.path.exists(legacy):
-                    try:
-                        os.remove(legacy)
-                    except OSError:
-                        pass
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "notes":
-                count = db.query(Note).count()
-                db.query(Note).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "tasks":
-                # TaskRun rows reference tasks via FK — clear them first.
-                db.query(TaskRun).delete()
-                count = db.query(ScheduledTask).count()
-                db.query(ScheduledTask).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "documents":
-                # DocumentVersion FKs Document — clear children first.
-                db.query(DocumentVersion).delete()
-                count = db.query(Document).count()
-                db.query(Document).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "gallery":
-                count = db.query(GalleryImage).count()
-                db.query(GalleryImage).delete()
-                db.commit()
-                # Also drop the upload dir so disk doesn't keep orphans.
-                _rmtree_quiet(os.path.join(DATA_DIR, "gallery"))
-                _rmtree_quiet(os.path.join(DATA_DIR, "gallery_uploads"))
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            if kind == "calendar":
-                # Events FK calendars — clear children first, then both.
-                db.query(CalendarEvent).delete()
-                count = db.query(CalendarCal).count()
-                db.query(CalendarCal).delete()
-                db.commit()
-                return {"status": "deleted", "kind": kind, "count": count}
-
-            raise HTTPException(400, f"Unknown wipe kind: {kind!r}")
-        except HTTPException:
-            raise
-        except Exception as e:
-            db.rollback()
-            logger.exception(f"Wipe {kind} failed")
-            raise HTTPException(500, f"Wipe {kind} failed: {e}")
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/api_token_routes.py

@@ -12,6 +12,65 @@ from src.auth_helpers import get_current_user
 
 MAX_NAME_LEN = 100
 DEFAULT_SCOPES = "chat"
+ALLOWED_SCOPES = {
+    "chat",
+    "todos:read",
+    "todos:write",
+    "documents:read",
+    "documents:write",
+    "email:read",
+    "email:draft",
+    "email:send",
+    "calendar:read",
+    "calendar:write",
+    "memory:read",
+    "memory:write",
+    "cookbook:read",
+    "cookbook:launch",
+}
+TOKEN_PROFILES = {
+    "chat": ["chat"],
+    "codex_todos": ["todos:read", "todos:write"],
+    "codex_documents": ["documents:read", "documents:write"],
+    "codex_email_drafts": ["email:read", "email:draft", "documents:read", "documents:write"],
+}
+
+
+def _normalize_scopes(scopes: str | list[str] | None = None, profile: str | None = None) -> list[str]:
+    profile = profile if isinstance(profile, str) else None
+    profile_key = (profile or "").strip()
+    if profile_key:
+        if profile_key not in TOKEN_PROFILES:
+            raise HTTPException(400, "Unknown token profile")
+        requested = list(TOKEN_PROFILES[profile_key])
+    elif isinstance(scopes, list):
+        requested = [str(s).strip() for s in scopes if str(s).strip()]
+    elif isinstance(scopes, str) and scopes:
+        requested = [s.strip() for s in scopes.replace(" ", ",").split(",") if s.strip()]
+    else:
+        requested = [DEFAULT_SCOPES]
+
+    normalized = []
+    for scope in requested:
+        if scope not in ALLOWED_SCOPES:
+            raise HTTPException(400, f"Unknown token scope: {scope}")
+        if scope not in normalized:
+            normalized.append(scope)
+
+    def ensure_before(write_scope: str, read_scope: str):
+        if write_scope not in normalized or read_scope in normalized:
+            return
+        idx = normalized.index(write_scope)
+        normalized.insert(idx, read_scope)
+
+    ensure_before("todos:write", "todos:read")
+    ensure_before("documents:write", "documents:read")
+    ensure_before("calendar:write", "calendar:read")
+    ensure_before("memory:write", "memory:read")
+    ensure_before("email:draft", "email:read")
+    ensure_before("cookbook:launch", "cookbook:read")
+
+    return normalized or [DEFAULT_SCOPES]
 
 
 def setup_api_token_routes() -> APIRouter:
@@ -45,13 +104,28 @@ def setup_api_token_routes() -> APIRouter:
         except Exception:
             pass
 
+    @router.get("/tokens/profiles")
+    def token_profiles(request: Request):
+        require_admin(request)
+        return {
+            "profiles": TOKEN_PROFILES,
+            "allowed_scopes": sorted(ALLOWED_SCOPES),
+        }
+
     @router.post("/tokens")
-    def create_token(request: Request, name: str = Form("")):
+    def create_token(
+        request: Request,
+        name: str = Form(""),
+        scopes: str = Form(None),
+        profile: str = Form(None),
+    ):
         require_admin(request)
         name = name.strip()[:MAX_NAME_LEN]
         if not name:
             raise HTTPException(400, "Token name is required")
         owner = get_current_user(request)
+        scope_list = _normalize_scopes(scopes, profile)
+        scopes_value = ",".join(scope_list)
 
         raw_token = "ody_" + secrets.token_urlsafe(32)
         token_hash = bcrypt.hashpw(raw_token.encode(), bcrypt.gensalt()).decode()
@@ -64,7 +138,7 @@ def setup_api_token_routes() -> APIRouter:
                 name=name,
                 token_hash=token_hash,
                 token_prefix=raw_token[:8],
-                scopes=DEFAULT_SCOPES,
+                scopes=scopes_value,
                 is_active=True,
             ))
         _invalidate_cache(request)
@@ -75,16 +149,60 @@ def setup_api_token_routes() -> APIRouter:
             "owner": owner,
             "token": raw_token,
             "token_prefix": raw_token[:8],
-            "scopes": DEFAULT_SCOPES.split(","),
+            "scopes": scope_list,
         }
 
+    @router.patch("/tokens/{token_id}")
+    async def update_token(request: Request, token_id: str):
+        require_admin(request)
+        current_user = get_current_user(request)
+        try:
+            payload = await request.json()
+        except Exception:
+            payload = {}
+        if not isinstance(payload, dict):
+            payload = {}
+        with get_db_session() as db:
+            token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
+            if not token:
+                raise HTTPException(404, "Token not found")
+            if current_user and token.owner != current_user:
+                raise HTTPException(403, "Not your token")
+            if isinstance(payload.get("name"), str) and payload["name"].strip():
+                token.name = payload["name"].strip()[:MAX_NAME_LEN]
+            # Only touch scopes when the caller actually sent them. A partial
+            # update such as a rename ({"name": ...} with no "scopes" key) must
+            # not silently reset the token to the default scope — that dropped
+            # every previously granted scope.
+            if "scopes" in payload:
+                token.scopes = ",".join(_normalize_scopes(payload.get("scopes")))
+            db.add(token)
+            current_scopes = [
+                s.strip()
+                for s in (getattr(token, "scopes", "") or DEFAULT_SCOPES).split(",")
+                if s.strip()
+            ]
+            response = {
+                "id": token_id,
+                "name": getattr(token, "name", ""),
+                "owner": getattr(token, "owner", None),
+                "token_prefix": getattr(token, "token_prefix", ""),
+                "scopes": current_scopes,
+            }
+        _invalidate_cache(request)
+        return response
+
     @router.delete("/tokens/{token_id}")
     def delete_token(request: Request, token_id: str):
         require_admin(request)
+        current_user = get_current_user(request)
         with get_db_session() as db:
-            deleted = db.query(ApiToken).filter(ApiToken.id == token_id).delete()
-            if not deleted:
+            token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
+            if not token:
                 raise HTTPException(404, "Token not found")
+            if current_user and token.owner != current_user:
+                raise HTTPException(403, "Not your token")
+            db.delete(token)
         _invalidate_cache(request)
         return {"status": "deleted"}
 

routes/assistant_routes.py

@@ -16,6 +16,7 @@ from pydantic import BaseModel
 
 from core.database import SessionLocal, CrewMember, ScheduledTask
 from src.auth_helpers import get_current_user
+from src.owner_identity import REQUEST_SENTINEL_OWNERS
 from src.task_scheduler import compute_next_run
 
 
@@ -89,11 +90,12 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
     # check-in tasks seeded. Hitting any /assistant route under one of these
     # used to seed a full CrewMember + Morning/Midday/Evening tasks under that
     # owner, which then double-fired alongside the real user's check-ins.
-    _SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""})
+    # REQUEST_SENTINEL_OWNERS covers request-only identities; Default/Local is a
+    # reserved login name but remains a valid storage owner.
 
     async def _get_or_create(owner: str) -> CrewMember:
         """Return the per-owner assistant CrewMember, creating it on demand."""
-        if not owner or owner in _SYNTHETIC_OWNERS:
+        if not owner or owner in REQUEST_SENTINEL_OWNERS:
             raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
         db = SessionLocal()
         try:

routes/auth_routes.py

@@ -3,17 +3,27 @@
 from fastapi import APIRouter, Request, Response, HTTPException
 from pydantic import BaseModel
 from typing import Optional
+import asyncio
 import logging
 import os
 
-from core.auth import AuthManager
+import json
+import re
+from pathlib import Path
+
+from core.atomic_io import atomic_write_json, atomic_write_text
+from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult, TOKEN_TTL
+from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR
 from src.rate_limiter import RateLimiter
+from src.settings_scrub import scrub_settings
 from src.settings import (
     load_settings as _load_settings,
     save_settings as _save_settings,
     load_features as _load_features,
     save_features as _save_features,
     DEFAULT_SETTINGS,
+    RETIRED_SETTING_KEYS,
+    without_retired_settings,
 )
 from src.integrations import (
     load_integrations,
@@ -21,6 +31,7 @@ from src.integrations import (
     update_integration,
     delete_integration,
     get_integration,
+    mask_integration_secret,
     execute_api_call,
     INTEGRATION_PRESETS,
     migrate_from_settings,
@@ -61,9 +72,47 @@ class DeleteUserRequest(BaseModel):
     username: str
 
 
+class RenameUserRequest(BaseModel):
+    username: str
+
+
+class SetAdminRequest(BaseModel):
+    is_admin: bool
+
+
+class SetOpenRegistrationRequest(BaseModel):
+    enabled: bool
+
 SESSION_COOKIE = "odysseus_session"
 
 
+def _secure_cookie(request: Request) -> bool:
+    """Decide the ``Secure`` attribute of the session cookie.
+
+    ``SECURE_COOKIES`` stays authoritative when it holds an explicit value:
+    ``true`` always marks the cookie Secure (the documented knob for a TLS
+    proxy), ``false`` never does, which is the escape hatch for an install
+    that still answers on plain HTTP alongside HTTPS. Anything else —
+    unset, or the present-but-empty value docker-compose injects for a
+    variable the host has not defined — derives it from the request, so an
+    HTTPS login gets a Secure cookie without any configuration.
+
+    Either the connection scheme or ``X-Forwarded-Proto`` saying https is
+    enough, which is the same test ``core/middleware.py`` applies before it
+    sends HSTS. Uvicorn's proxy-headers middleware already folds that header
+    into the scheme for the proxies it trusts, so reading it here only adds
+    the case of a terminator that is not on a trusted address; the cost is
+    that a client talking to the app directly can set the header and lock
+    its own session out over plain HTTP.
+    """
+    configured = os.getenv("SECURE_COOKIES", "").strip().lower()
+    if configured in ("true", "false"):
+        return configured == "true"
+    # A chained proxy sends a list — the client-facing hop comes first.
+    forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0]
+    return request.url.scheme == "https" or forwarded_proto.strip().lower() == "https"
+
+
 def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
     router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -82,9 +131,13 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(429, "Too many requests — try again later")
         if auth_manager.is_configured:
             raise HTTPException(400, "Already configured")
-        if len(body.password) < 8:
-            raise HTTPException(400, "Password must be at least 8 characters")
-        ok = auth_manager.setup(body.username, body.password)
+        if len(body.password) < PASSWORD_MIN_LENGTH:
+            raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
+        if len(body.username.strip()) < 1:
+            raise HTTPException(400, "Username is required")
+        if body.username.lower() in RESERVED_USERNAMES:
+            raise HTTPException(403, "Username is reserved")
+        ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password)
         if not ok:
             raise HTTPException(500, "Setup failed")
         return {"ok": True, "message": "Admin account created"}
@@ -98,11 +151,13 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(400, "Run setup first")
         if not auth_manager.signup_enabled:
             raise HTTPException(403, "Registration is disabled. Ask an admin for an account.")
-        if len(body.password) < 8:
-            raise HTTPException(400, "Password must be at least 8 characters")
+        if len(body.password) < PASSWORD_MIN_LENGTH:
+            raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
         if len(body.username.strip()) < 1:
             raise HTTPException(400, "Username is required")
-        ok = auth_manager.create_user(body.username, body.password, is_admin=False)
+        if body.username.lower() in RESERVED_USERNAMES:
+            raise HTTPException(403, "Username is reserved")
+        ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False)
         if not ok:
             raise HTTPException(409, "Username already taken")
         return {"ok": True, "message": "Account created"}
@@ -113,7 +168,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(429, "Too many requests — try again later")
         # Verify password first
         username = body.username.strip().lower()
-        if not auth_manager.verify_password(username, body.password):
+        if not await asyncio.to_thread(auth_manager.verify_password, username, body.password):
             raise HTTPException(401, "Invalid credentials")
         # Check 2FA if enabled
         if auth_manager.totp_enabled(username):
@@ -122,8 +177,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                 return {"ok": False, "requires_totp": True, "username": username}
             if not auth_manager.totp_verify(username, body.totp_code):
                 raise HTTPException(401, "Invalid 2FA code")
-        # All checks passed — create session
-        token = auth_manager.create_session(username, body.password)
+        # All checks passed — create session (password already verified above)
+        token = await asyncio.to_thread(auth_manager.create_session_trusted, username)
         if not token:
             raise HTTPException(401, "Invalid credentials")
         cookie_kwargs = dict(
@@ -131,11 +186,11 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             value=token,
             httponly=True,
             samesite="lax",
-            secure=os.getenv("SECURE_COOKIES", "false").lower() == "true",
+            secure=_secure_cookie(request),
             path="/",
         )
         if body.remember:
-            cookie_kwargs["max_age"] = 60 * 60 * 24 * 7  # 7 days
+            cookie_kwargs["max_age"] = TOKEN_TTL
         response.set_cookie(**cookie_kwargs)
         return {"ok": True, "username": username}
 
@@ -164,16 +219,23 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             pass
         return result
 
+    @router.get("/policy")
+    async def auth_policy():
+        """Return public auth policy constants for the frontend."""
+        return auth_manager.policy()
+
     @router.post("/change-password")
     async def change_password(body: ChangePasswordRequest, request: Request):
         user = _get_current_user(request)
         if not user:
             raise HTTPException(401, "Not authenticated")
-        if len(body.new_password) < 8:
-            raise HTTPException(400, "Password must be at least 8 characters")
-        ok = auth_manager.change_password(user, body.current_password, body.new_password)
+        if len(body.new_password) < PASSWORD_MIN_LENGTH:
+            raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
+        current_token = request.cookies.get(SESSION_COOKIE)
+        ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password)
         if not ok:
             raise HTTPException(400, "Current password is incorrect")
+        await asyncio.to_thread(auth_manager.revoke_user_sessions, user, current_token)
         return {"ok": True}
 
     # ------------------------------------------------------------------
@@ -248,8 +310,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         user = _get_current_user(request)
         if not user or not auth_manager.is_admin(user):
             raise HTTPException(403, "Admin only")
-        if len(body.password) < 8:
-            raise HTTPException(400, "Password must be at least 8 characters")
+        if len(body.password) < PASSWORD_MIN_LENGTH:
+            raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
+        if len(body.username.strip()) < 1:
+            raise HTTPException(400, "Username is required")
+        if body.username.lower() in RESERVED_USERNAMES:
+            raise HTTPException(403, "Username is reserved")
         ok = auth_manager.create_user(body.username, body.password, body.is_admin)
         if not ok:
             raise HTTPException(409, "Username already taken")
@@ -266,23 +332,361 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(404, "User not found or is admin")
         return {"ok": True, "privileges": auth_manager.get_privileges(username)}
 
-    @router.post("/signup-toggle")
+    @router.put("/users/{username}/rename")
+    async def rename_user(username: str, body: RenameUserRequest, request: Request):
+        user = _get_current_user(request)
+        if not user or not auth_manager.is_admin(user):
+            raise HTTPException(403, "Admin only")
+        old_username = (username or "").strip().lower()
+        new_username = (body.username or "").strip().lower()
+        if not new_username:
+            raise HTTPException(400, "Username required")
+        if old_username == new_username:
+            return {"ok": True, "username": new_username, "renamed_self": old_username == user}
+        if old_username not in auth_manager.users:
+            raise HTTPException(404, "User not found")
+        if new_username in auth_manager.users:
+            raise HTTPException(409, "Username already taken")
+
+        # Gate on auth first. Every mutation below is contingent on this
+        # succeeding — doing it last meant a rejected rename (e.g. reserved
+        # username) left file-backed owner fields already rewritten with no
+        # way to roll them back.
+        ok = auth_manager.rename_user(old_username, new_username, user)
+        if not ok:
+            raise HTTPException(400, "Cannot rename user")
+
+        def _rollback_auth_rename() -> bool:
+            # On self-rename the admin session has already moved to the new
+            # username, so the rollback must authenticate as the new user.
+            rollback_user = new_username if user == old_username else user
+            try:
+                return bool(auth_manager.rename_user(new_username, old_username, rollback_user))
+            except Exception as rollback_err:
+                logger.error(
+                    "Failed to roll back auth rename %s -> %s after owner migration failure: %s",
+                    new_username, old_username, rollback_err,
+                )
+                return False
+
+        # Usernames are ownership keys for user data. Rename the common
+        # owner-scoped DB rows so the account keeps access to its sessions,
+        # docs, email accounts, tasks, etc.
+        try:
+            from sqlalchemy import func
+            from core.database import (
+                Base,
+                EmailAccount,
+                SessionLocal,
+                lock_email_account_owner_mutations,
+            )
+            db = SessionLocal()
+            try:
+                # Email-account defaults are protected by per-owner mutex rows.
+                # A rename crosses two owner partitions, so lock both in the
+                # shared helper's canonical order before inspecting either.
+                lock_email_account_owner_mutations(
+                    db, old_username, new_username
+                )
+
+                source_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == old_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                destination_default_ids = [
+                    row[0]
+                    for row in (
+                        db.query(EmailAccount.id)
+                        .filter(
+                            func.lower(EmailAccount.owner) == new_username,
+                            EmailAccount.is_default == True,  # noqa: E712
+                        )
+                        .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc())
+                        .all()
+                    )
+                ]
+                if destination_default_ids:
+                    clear_default_ids = (
+                        destination_default_ids[1:] + source_default_ids
+                    )
+                else:
+                    clear_default_ids = source_default_ids[1:]
+                if clear_default_ids:
+                    (
+                        db.query(EmailAccount)
+                        .filter(EmailAccount.id.in_(clear_default_ids))
+                        .update(
+                            {EmailAccount.is_default: False},
+                            synchronize_session=False,
+                        )
+                    )
+
+                for mapper in Base.registry.mappers:
+                    model = mapper.class_
+                    if not hasattr(model, "owner"):
+                        continue
+                    (
+                        db.query(model)
+                        .filter(func.lower(model.owner) == old_username)
+                        .update({"owner": new_username}, synchronize_session=False)
+                    )
+                db.commit()
+            except Exception:
+                db.rollback()
+                raise
+            finally:
+                db.close()
+        except Exception as e:
+            logger.error("Failed to rename owner references %s -> %s: %s", old_username, new_username, e)
+            if not _rollback_auth_rename():
+                logger.error(
+                    "Auth rename %s -> %s could not be rolled back after owner migration failure",
+                    old_username, new_username,
+                )
+            raise HTTPException(500, "Failed to rename user data")
+
+        # Per-user prefs are JSON-backed, not SQL-backed.
+        try:
+            from routes.prefs_routes import _load as _load_prefs, _save as _save_prefs
+            prefs = _load_prefs()
+            users = prefs.get("_users") if isinstance(prefs, dict) else None
+            if isinstance(users, dict):
+                prefs_key = next(
+                    (k for k in users if str(k).strip().lower() == old_username),
+                    None,
+                )
+                new_taken = any(str(k).strip().lower() == new_username for k in users)
+                if prefs_key is not None and not new_taken:
+                    users[new_username] = users.pop(prefs_key)
+                    _save_prefs(prefs)
+        except Exception as e:
+            logger.warning("Failed to rename user prefs %s -> %s: %s", old_username, new_username, e)
+
+        # In-flight deep-research tasks live in the process-local
+        # ResearchHandler registry. They are not covered by the persisted JSON
+        # migration above, but the research routes filter and cancel by this
+        # owner field while the job is running. Do this before sweeping
+        # completed JSON files so a job that finishes during the rename saves
+        # with the new owner or is caught by the disk sweep below.
+        try:
+            rh = getattr(request.app.state, "research_handler", None)
+            rename_owner = getattr(rh, "rename_owner", None)
+            if callable(rename_owner):
+                rename_owner(old_username, new_username)
+        except Exception as e:
+            logger.warning("Failed to rename active research tasks %s -> %s: %s", old_username, new_username, e)
+
+        # deep_research: each completed report is a standalone JSON file with
+        # an `owner` field. research_routes filters by d.get("owner") == user,
+        # so a stale owner makes every report invisible to the renamed user.
+        try:
+            dr_dir = Path(DEEP_RESEARCH_DIR)
+            if dr_dir.is_dir():
+                for p in dr_dir.glob("*.json"):
+                    try:
+                        d = json.loads(p.read_text(encoding="utf-8"))
+                        if str(d.get("owner", "")).strip().lower() == old_username:
+                            d["owner"] = new_username
+                            atomic_write_json(str(p), d)
+                    except Exception as err:
+                        logger.warning("Failed to update research owner in %s: %s", p.name, err)
+        except Exception as e:
+            logger.warning("Failed to rename research owner references %s -> %s: %s", old_username, new_username, e)
+
+        # memory.json: a flat JSON array where each entry carries an `owner`
+        # field. memory_manager.load(owner=user) filters on it, so stale
+        # entries disappear from the memory panel.
+        try:
+            if os.path.isfile(MEMORY_FILE):
+                with open(MEMORY_FILE, encoding="utf-8") as fh:
+                    entries = json.loads(fh.read())
+                if isinstance(entries, list):
+                    changed = False
+                    for entry in entries:
+                        if isinstance(entry, dict) and str(entry.get("owner", "")).strip().lower() == old_username:
+                            entry["owner"] = new_username
+                            changed = True
+                    if changed:
+                        atomic_write_json(MEMORY_FILE, entries)
+        except Exception as e:
+            logger.warning("Failed to rename memory.json owner references %s -> %s: %s", old_username, new_username, e)
+
+        # uploads.json: upload rows use owner metadata for access checks and
+        # owner-prefixed index keys for dedupe. Rename both so attachments keep
+        # resolving after the account username changes.
+        try:
+            upload_handler = getattr(request.app.state, "upload_handler", None)
+            rename_owner = getattr(upload_handler, "rename_owner", None)
+            if callable(rename_owner):
+                rename_owner(old_username, new_username)
+        except Exception as e:
+            logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e)
+
+        # direct personal RAG uploads live in per-owner directories and the
+        # vector metadata also carries the username used for owner-filtered
+        # search. Keep both in sync with the auth rename.
+        try:
+            from routes.personal_routes import rename_personal_upload_owner
+            personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None)
+            if personal_docs_manager is not None:
+                rag_manager = getattr(personal_docs_manager, "rag_manager", None)
+                rename_personal_upload_owner(
+                    old_username,
+                    new_username,
+                    personal_docs_manager=personal_docs_manager,
+                    rag_manager=rag_manager,
+                )
+        except Exception as e:
+            logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e)
+
+        # skills: SKILL.md frontmatter carries owner: <username>; the usage
+        # sidecar (_usage.json) keys entries as owner::skill-name. Both must
+        # be updated or the renamed user's Skills panel goes empty.
+        try:
+            skills_root = Path(SKILLS_DIR)
+            if skills_root.is_dir():
+                _owner_re = re.compile(
+                    r'(?m)^(owner:\s*)' + re.escape(old_username) + r'\s*$',
+                    re.IGNORECASE,
+                )
+                for p in skills_root.rglob("SKILL.md"):
+                    try:
+                        text = p.read_text(encoding="utf-8")
+                        new_text = _owner_re.sub(r'\g<1>' + new_username, text)
+                        if new_text != text:
+                            atomic_write_text(str(p), new_text)
+                    except Exception as err:
+                        logger.warning("Failed to update skill owner in %s: %s", p, err)
+                usage_path = skills_root / "_usage.json"
+                if usage_path.is_file():
+                    try:
+                        usage = json.loads(usage_path.read_text(encoding="utf-8"))
+                        if isinstance(usage, dict):
+                            new_usage = {}
+                            changed = False
+                            for k, v in usage.items():
+                                owner_part, sep, skill_part = k.partition("::")
+                                if sep and owner_part.lower() == old_username:
+                                    new_usage[new_username + "::" + skill_part] = v
+                                    changed = True
+                                else:
+                                    new_usage[k] = v
+                            if changed:
+                                atomic_write_json(str(usage_path), new_usage)
+                    except Exception as err:
+                        logger.warning("Failed to update skills usage keys %s -> %s: %s", old_username, new_username, err)
+        except Exception as e:
+            logger.warning("Failed to rename skills owner references %s -> %s: %s", old_username, new_username, e)
+
+        # The in-memory session cache (session_manager.sessions) stores each
+        # session's owner at load time. Without this patch the renamed user's
+        # sessions are invisible on the next /api/sessions call because
+        # get_sessions_for_user does an exact `s.owner == username` comparison
+        # against stale in-memory values.
+        sm = getattr(request.app.state, "session_manager", None)
+        if sm is not None:
+            for sess in list(getattr(sm, "sessions", {}).values()):
+                if str(getattr(sess, "owner", None) or "").strip().lower() == old_username:
+                    sess.owner = new_username
+
+        # The owner-rename loop above updated ApiToken.owner in the DB, but the
+        # bearer-token cache still maps each token to the OLD owner. Without
+        # refreshing it, the renamed user's API tokens resolve to the old (now
+        # non-existent) owner and stop reaching their data until the cache next
+        # goes dirty. Invalidate it now, like the token CRUD routes do.
+        invalidator = getattr(request.app.state, "invalidate_token_cache", None)
+        if callable(invalidator):
+            invalidator()
+        return {"ok": True, "username": new_username, "renamed_self": old_username == user}
+
+    @router.put("/users/{username}/admin")
+    async def set_user_admin(username: str, body: SetAdminRequest, request: Request):
+        """Promote/demote a user to/from admin. Admin only.
+
+        The last remaining admin can't be demoted (no lockout). Self-demotion
+        is allowed while another admin exists; the `self` flag tells the UI to
+        reload the acting user into the normal-user view.
+        """
+        user = _get_current_user(request)
+        if not user or not auth_manager.is_admin(user):
+            raise HTTPException(403, "Admin only")
+        result = auth_manager.set_admin(username, body.is_admin, user)
+        if result is SetAdminResult.USER_NOT_FOUND:
+            raise HTTPException(404, "User not found")
+        if result is SetAdminResult.NOT_AUTHORIZED:
+            raise HTTPException(403, "Admin only")
+        if result is SetAdminResult.LAST_ADMIN:
+            raise HTTPException(400, "Cannot demote the last admin")
+        target = (username or "").strip().lower()
+        return {
+            "ok": True,
+            "is_admin": body.is_admin,
+            "self": target == (user or "").strip().lower(),
+        }
+
+    @router.post("/signup-toggle", deprecated=True)
     async def toggle_signup(request: Request):
-        """Toggle open registration on/off. Admin only."""
+        """
+        Toggle open registration on/off. Admin only.
+
+        DEPRECATED: This endpoint uses toggle semantics which can lead to unsafe state changes.
+        Use PUT /open-signup instead.
+
+        This endpoint is kept for backward compatibility and may be removed in future versions.
+        """
         user = _get_current_user(request)
         if not user or not auth_manager.is_admin(user):
             raise HTTPException(403, "Admin only")
         auth_manager.signup_enabled = not auth_manager.signup_enabled
         return {"ok": True, "signup_enabled": auth_manager.signup_enabled}
 
+    @router.put("/open-signup")
+    async def set_signup_enabled(body: SetOpenRegistrationRequest, request: Request):
+        """Set open signup enabled state. Admin only."""
+        user = _get_current_user(request)
+        if not user or not auth_manager.is_admin(user):
+            raise HTTPException(403, "Admin only")
+        auth_manager.signup_enabled = body.enabled
+        return {"ok": True,"signup_enabled": auth_manager.signup_enabled}
+
     @router.delete("/users")
     async def admin_delete_user(body: DeleteUserRequest, request: Request):
         user = _get_current_user(request)
         if not user or not auth_manager.is_admin(user):
             raise HTTPException(403, "Admin only")
-        ok = auth_manager.delete_user(body.username, user)
+
+        def _invalidate_api_token_cache():
+            try:
+                invalidator = getattr(request.app.state, "invalidate_token_cache", None)
+                if invalidator:
+                    invalidator()
+            except Exception:
+                pass
+
+        try:
+            ok = auth_manager.delete_user(body.username, user)
+        except Exception:
+            # delete_user can touch ApiToken rows before a later auth-store write
+            # fails. Dirty the bearer cache anyway so a partial token purge does
+            # not leave already-cached tokens authenticating until restart.
+            _invalidate_api_token_cache()
+            raise
         if not ok:
             raise HTTPException(400, "Cannot delete user")
+        # delete_user removes the user's ApiToken rows, but the bearer-auth
+        # middleware serves from an in-memory prefix->token cache that only
+        # rebuilds when flagged dirty. Without this, a deleted user's already
+        # cached token keeps authenticating until some other token op or a
+        # restart clears the cache. Mirror what the token routes do.
+        _invalidate_api_token_cache()
         return {"ok": True}
 
     # ---- Feature visibility (admin-managed) ----
@@ -308,39 +712,16 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
 
     # ---- App settings (admin-managed) ----
 
-    _SECRET_KEY_PATTERNS = ("_api_key", "_password", "_secret", "_token", "_key")
-
-    def _is_secret_key(name: str) -> bool:
-        n = (name or "").lower()
-        if n in ("google_pse_cx",):  # public identifier, not a secret
-            return False
-        return any(n.endswith(p) or n == p.lstrip("_") for p in _SECRET_KEY_PATTERNS)
-
-    def _scrub_settings(settings: dict) -> dict:
-        """Return a copy of settings with secret-shaped values masked.
-
-        Frontend reads /settings without auth for things like keybinds + TTS
-        prefs. Secrets (search-provider keys, IMAP/SMTP passwords) must NOT
-        be exposed to non-admin callers.
-        """
-        scrubbed = {}
-        for k, v in (settings or {}).items():
-            if _is_secret_key(k) and isinstance(v, str) and v:
-                scrubbed[k] = ""  # presence preserved, value blanked
-            else:
-                scrubbed[k] = v
-        return scrubbed
-
     @router.get("/settings")
     async def get_settings(request: Request):
         """Returns app settings. Admins get the full set; non-admins get
         a scrubbed copy with secret keys blanked. The frontend uses this
         for keybinds + TTS prefs, so it stays callable without admin."""
         user = _get_current_user(request)
-        settings = _load_settings()
+        settings = without_retired_settings(_load_settings())
         if user and auth_manager.is_admin(user):
             return settings
-        return _scrub_settings(settings)
+        return scrub_settings(settings)
 
     @router.post("/settings")
     async def set_settings(request: Request):
@@ -350,11 +731,29 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(403, "Admin only")
         body = await request.json()
         current = _load_settings()
+        # Per-key validation for numeric settings: coerce to int and clamp to a
+        # sane range so a bad value can't disable the agent or let it run away.
+        _INT_RANGES = {
+            "agent_max_rounds": (1, 200),
+            "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
+            "auto_compact_threshold_percent": (50, 95),
+        }
         for key in DEFAULT_SETTINGS:
-            if key in body:
-                current[key] = body[key]
+            if key in RETIRED_SETTING_KEYS:
+                continue
+            if key not in body:
+                continue
+            val = body[key]
+            if key in _INT_RANGES:
+                lo, hi = _INT_RANGES[key]
+                try:
+                    val = int(val)
+                except (TypeError, ValueError):
+                    raise HTTPException(400, f"{key} must be an integer")
+                val = max(lo, min(val, hi))
+            current[key] = val
         _save_settings(current)
-        return current
+        return without_retired_settings(current)
 
     # ---- Integrations CRUD ----
 
@@ -369,12 +768,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(403, "Admin only")
         items = load_integrations()
         # Mask API keys for frontend display
-        safe = []
-        for item in items:
-            copy = dict(item)
-            if copy.get("api_key"):
-                copy["api_key"] = copy["api_key"][:4] + "****"
-            safe.append(copy)
+        safe = [mask_integration_secret(item) for item in items]
         return {"integrations": safe}
 
     @router.get("/integrations/presets")
@@ -390,7 +784,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
             raise HTTPException(403, "Admin only")
         body = await request.json()
         item = add_integration(body)
-        return {"ok": True, "integration": item}
+        return {"ok": True, "integration": mask_integration_secret(item)}
 
     @router.put("/integrations/{integration_id}")
     async def update_integration_route(integration_id: str, request: Request):
@@ -402,7 +796,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         item = update_integration(integration_id, body)
         if not item:
             raise HTTPException(404, "Integration not found")
-        return {"ok": True, "integration": item}
+        return {"ok": True, "integration": mask_integration_secret(item)}
 
     @router.delete("/integrations/{integration_id}")
     async def delete_integration_route(integration_id: str, request: Request):
@@ -482,7 +876,31 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
                     }
                 return {"ok": False, "message": f"ntfy returned HTTP {r.status_code} from {full_url}: {r.text[:200]}"}
             except Exception as e:
-                return {"ok": False, "message": f"ntfy publish to {full_url} failed: {e}"[:300]}
+                hint = ""
+                if parsed.hostname not in ("127.0.0.1", "localhost"):
+                    hint = " If this is Docker Compose ntfy, set NTFY_BIND to that host/Tailscale IP and NTFY_BASE_URL to the same server URL in .env, then recreate ntfy."
+                return {"ok": False, "message": f"ntfy publish to {full_url} failed: {e}.{hint}"[:500]}
+
+        if preset == "discord_webhook":
+            import httpx
+            webhook_url = (integ.get("base_url") or "").strip()
+            if not webhook_url:
+                return {"ok": False, "message": "No webhook URL set — paste the full Discord webhook URL into the Base URL field."}
+            payload = {
+                "embeds": [{
+                    "title": "Odysseus connectivity test",
+                    "description": "If you see this, your Discord Webhook integration is wired up correctly.",
+                    "color": 5793266,
+                }]
+            }
+            try:
+                async with httpx.AsyncClient(timeout=8.0) as client:
+                    r = await client.post(webhook_url, json=payload)
+                if r.is_success:
+                    return {"ok": True, "message": "Test embed sent — check your Discord channel to confirm it arrived."}
+                return {"ok": False, "message": f"Discord returned HTTP {r.status_code}: {r.text[:200]}"}
+            except Exception as e:
+                return {"ok": False, "message": f"Request failed: {e}"[:400]}
 
         # All other presets: GET against a known health endpoint.
         # Fall back to detecting from name if preset is missing.

routes/backup_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime
 
 from fastapi import APIRouter, HTTPException, Request, Response
 from core.middleware import require_admin
+from services.memory import MemoryStoreUnreadable
 from src.auth_helpers import get_current_user
 from src.settings import load_settings, save_settings, load_features, save_features
 
@@ -76,8 +77,21 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
 
         # ── Memories ──
         if "memories" in body and isinstance(body["memories"], list):
-            existing = memory_manager.load_all()
-            existing_texts = {e.get("text", "").strip().lower() for e in existing}
+            # Strict load: importing on top of an unreadable store would write
+            # only the incoming rows back and drop everything already saved.
+            try:
+                existing = memory_manager.load_all_for_update()
+            except MemoryStoreUnreadable as e:
+                logger.error("Refusing to import memories: %s", e)
+                raise HTTPException(
+                    503, "Memory store is temporarily unreadable — nothing was imported."
+                )
+            # Dedup against THIS user's own memories only. Using every tenant's
+            # rows (load_all) meant a memory whose text matched any other
+            # user's was silently skipped, so the importing user lost their own
+            # data. The full store is still saved back below.
+            existing_texts = {e.get("text", "").strip().lower()
+                              for e in existing if e.get("owner") == user}
             added = 0
             for mem in body["memories"]:
                 if not isinstance(mem, dict) or not mem.get("text"):
@@ -96,24 +110,74 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
         # ── Skills ──
         if "skills" in body and isinstance(body["skills"], list):
             existing = skills_manager.load_all()
-            existing_ids = {s.get("id") for s in existing}
-            existing_titles = {s.get("title", "").strip().lower() for s in existing}
+            # Dedup against THIS user's own skills only. Using every tenant's
+            # rows (load_all) meant a skill whose id/name/title matched any
+            # other user's was silently skipped, so the importing user lost
+            # their own data — same cross-tenant bug fixed for memories above.
+            # The full store is still saved back below.
+            own = [s for s in existing if s.get("owner") == user]
+            existing_names = {s.get("name") for s in own if s.get("name")}
+            existing_ids = {s.get("id") for s in own if s.get("id")}
+            existing_titles = {
+                (s.get("title") or s.get("description") or "").strip().lower()
+                for s in own
+            }
             added = 0
             for skill in body["skills"]:
-                if not isinstance(skill, dict) or not skill.get("title"):
+                if not isinstance(skill, dict):
                     continue
-                # Skip if same id or same title already exists
-                if skill.get("id") in existing_ids:
+                title = (
+                    skill.get("title") or skill.get("description")
+                    or skill.get("name") or ""
+                ).strip()
+                if not title:
                     continue
-                if skill["title"].strip().lower() in existing_titles:
+                sid = skill.get("id") or skill.get("name")
+                if sid and sid in existing_ids:
                     continue
-                if user and not skill.get("owner"):
-                    skill["owner"] = user
-                existing.append(skill)
-                existing_ids.add(skill.get("id"))
-                existing_titles.add(skill["title"].strip().lower())
+                nm = skill.get("name")
+                if nm and nm in existing_names:
+                    continue
+                if title.lower() in existing_titles:
+                    continue
+                owner = skill.get("owner")
+                if user and not owner:
+                    owner = user
+                # Skills live on disk as SKILL.md files; the old JSON-era
+                # skills_manager.save() no longer exists. Write each new skill
+                # via add_skill (source="user" skips auto-dedup — this is an
+                # explicit backup restore).
+                result = skills_manager.add_skill(
+                    title=title,
+                    name=skill.get("name"),
+                    description=skill.get("description"),
+                    problem=skill.get("problem", ""),
+                    solution=skill.get("solution", ""),
+                    steps=skill.get("steps"),
+                    tags=skill.get("tags"),
+                    source="user",
+                    teacher_model=skill.get("teacher_model"),
+                    confidence=skill.get("confidence", 0.8),
+                    owner=owner,
+                    category=skill.get("category", "general"),
+                    when_to_use=skill.get("when_to_use"),
+                    procedure=skill.get("procedure"),
+                    pitfalls=skill.get("pitfalls"),
+                    verification=skill.get("verification"),
+                    platforms=skill.get("platforms"),
+                    requires_toolsets=skill.get("requires_toolsets"),
+                    fallback_for_toolsets=skill.get("fallback_for_toolsets"),
+                    status=skill.get("status", "draft"),
+                    version=skill.get("version", "1.0.0"),
+                )
+                if result.get("_deduped"):
+                    continue
+                if result.get("name"):
+                    existing_names.add(result["name"])
+                if result.get("id"):
+                    existing_ids.add(result["id"])
+                existing_titles.add(title.lower())
                 added += 1
-            skills_manager.save(existing)
             imported.append(f"{added} skills")
 
         # ── Presets ──

routes/chatgpt_subscription_routes.py

@@ -0,0 +1,170 @@
+"""ChatGPT Subscription device-flow setup routes."""
+
+import json
+import logging
+import uuid
+from typing import Dict, Optional
+
+from fastapi import HTTPException, Request
+
+from core.database import ModelEndpoint, ProviderAuthSession, SessionLocal, utcnow_naive
+from routes.device_flow import (
+    DeviceFlowPoll,
+    DeviceFlowStart,
+    PendingDeviceFlowStore,
+    create_device_flow_router,
+)
+from src.auth_helpers import get_current_user
+from src import chatgpt_subscription
+
+logger = logging.getLogger(__name__)
+
+_DEVICE_FLOW_STORE = PendingDeviceFlowStore()
+
+
+def _provision_endpoint(tokens: Dict, owner: Optional[str]) -> Dict:
+    access_token = tokens.get("access_token")
+    refresh_token = tokens.get("refresh_token")
+    if not access_token or not refresh_token:
+        raise ValueError("ChatGPT token response was missing access_token or refresh_token")
+
+    base = chatgpt_subscription.DEFAULT_CHATGPT_SUBSCRIPTION_BASE_URL
+    models = chatgpt_subscription.fetch_available_models(access_token)
+    if not models:
+        raise ValueError("ChatGPT Subscription connected, but no usable Codex models were discovered for this account.")
+    db = SessionLocal()
+    try:
+        auth = (
+            db.query(ProviderAuthSession)
+            .filter(
+                ProviderAuthSession.provider == chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER,
+                ProviderAuthSession.owner == owner,
+            )
+            .first()
+        )
+        if auth is None:
+            auth = ProviderAuthSession(
+                id=str(uuid.uuid4())[:8],
+                provider=chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER,
+                owner=owner,
+                label="ChatGPT Subscription",
+                base_url=base,
+                auth_mode="chatgpt",
+            )
+            db.add(auth)
+        auth.base_url = base
+        auth.access_token = access_token
+        auth.refresh_token = refresh_token
+        auth.last_refresh = utcnow_naive()
+        auth.auth_mode = "chatgpt"
+
+        ep = (
+            db.query(ModelEndpoint)
+            .filter(
+                ModelEndpoint.base_url == base,
+                ModelEndpoint.provider_auth_id == auth.id,
+                ModelEndpoint.owner == owner,
+            )
+            .first()
+        )
+        if ep is None:
+            ep = ModelEndpoint(
+                id=str(uuid.uuid4())[:8],
+                name="ChatGPT Subscription",
+                base_url=base,
+                model_type="llm",
+                endpoint_kind="api",
+                owner=owner,
+            )
+            db.add(ep)
+        ep.name = "ChatGPT Subscription"
+        ep.base_url = base
+        ep.api_key = None
+        ep.provider_auth_id = auth.id
+        ep.is_enabled = True
+        ep.supports_tools = False
+        ep.model_type = "llm"
+        ep.endpoint_kind = "api"
+        ep.model_refresh_mode = "manual"
+        ep.cached_models = json.dumps(models)
+        db.commit()
+        result = {
+            "id": ep.id,
+            "name": ep.name,
+            "base_url": ep.base_url,
+            "models": models,
+        }
+    finally:
+        db.close()
+
+    try:
+        from routes.model_routes import _invalidate_models_cache
+
+        _invalidate_models_cache()
+    except Exception:
+        pass
+    return result
+
+
+def _start_device_flow(request: Request, _form) -> DeviceFlowStart:
+    try:
+        data = chatgpt_subscription.request_device_code()
+    except Exception as exc:
+        raise chatgpt_subscription.to_http_exception(exc)
+
+    device_auth_id = data.get("device_auth_id")
+    user_code = data.get("user_code")
+    if not device_auth_id or not user_code:
+        raise HTTPException(502, "ChatGPT did not return a complete device code")
+    verification_uri = data.get("verification_uri") or f"{chatgpt_subscription.CHATGPT_OAUTH_ISSUER}/codex/device"
+    return DeviceFlowStart(
+        pending={
+            "device_auth_id": device_auth_id,
+            "user_code": user_code,
+            "owner": get_current_user(request) or None,
+        },
+        response={
+            "user_code": user_code,
+            "verification_uri": verification_uri,
+        },
+        interval=int(data.get("interval") or 5),
+        expires_in=int(data.get("expires_in") or 900),
+    )
+
+
+def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll:
+    try:
+        data = chatgpt_subscription.poll_device_auth(pending["device_auth_id"], pending["user_code"])
+    except Exception as exc:
+        logger.debug("ChatGPT device poll failed: %s", exc)
+        return DeviceFlowPoll.pending(str(exc))
+
+    authorization_code = data.get("authorization_code")
+    code_verifier = data.get("code_verifier")
+    if authorization_code and code_verifier:
+        try:
+            tokens = chatgpt_subscription.exchange_authorization_code(authorization_code, code_verifier)
+            result = _provision_endpoint(tokens, pending["owner"])
+        except Exception as exc:
+            logger.exception("ChatGPT Subscription endpoint provisioning failed")
+            raise chatgpt_subscription.to_http_exception(exc)
+        return DeviceFlowPoll.authorized(result)
+
+    err = data.get("error") or data.get("status")
+    if err in ("authorization_pending", "pending", None):
+        return DeviceFlowPoll.pending()
+    if err == "slow_down":
+        return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None)
+    if err in ("expired_token", "access_denied", "denied"):
+        return DeviceFlowPoll.failed(err)
+    return DeviceFlowPoll.pending(err or "unknown")
+
+
+def setup_chatgpt_subscription_routes():
+    return create_device_flow_router(
+        prefix="/api/chatgpt-subscription",
+        tags=["chatgpt-subscription"],
+        store=_DEVICE_FLOW_STORE,
+        start_flow=_start_device_flow,
+        poll_flow=_poll_device_flow,
+    )

routes/cleanup/__init__.py

@@ -0,0 +1,5 @@
+"""Cleanup route domain package (slice 2g, #4082/#4071).
+
+Contains cleanup_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/cleanup_routes.py re-exports from here.
+"""

routes/cleanup/cleanup_routes.py

@@ -0,0 +1,60 @@
+# routes/cleanup_routes.py
+"""Routes for cleanup operations."""
+import logging
+from fastapi import APIRouter, HTTPException, Request
+from src.cleanup_service import get_cleanup_preview, cleanup_sessions
+from src.auth_helpers import get_current_user
+
+logger = logging.getLogger(__name__)
+
+def setup_cleanup_routes(session_manager):
+    """
+    Setup cleanup-related routes.
+
+    Args:
+        session_manager: SessionManager instance
+
+    Returns:
+        APIRouter instance with cleanup routes
+    """
+    router = APIRouter(prefix="/api/cleanup")
+
+    @router.get("/preview")
+    async def cleanup_preview(request: Request):
+        """
+        Preview what would be cleaned up without making any changes.
+
+        Returns:
+            JSON response with lists of sessions that would be archived/deleted and estimated space savings
+        """
+        user = get_current_user(request)
+        try:
+            preview = await get_cleanup_preview(owner=user)
+            return preview
+        except Exception as e:
+            logger.error(f"Cleanup preview failed: {e}")
+            raise HTTPException(500, "Cleanup preview generation failed")
+
+    @router.post("")
+    async def cleanup_endpoint(request: Request):
+        """
+        Perform cleanup operations:
+        1. Archive inactive sessions (not accessed for 7 days)
+        2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages)
+
+        Returns:
+            JSON response with counts of deleted and archived sessions, and space freed
+        """
+        user = get_current_user(request)
+        try:
+            archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user)
+            return {
+                "archived_count": archived_count,
+                "deleted_count": deleted_count,
+                "space_freed_mb": round(space_freed_mb, 2)
+            }
+        except Exception as e:
+            logger.error(f"Cleanup failed: {e}")
+            raise HTTPException(500, "Cleanup operation failed")
+
+    return router

routes/cleanup_routes.py

@@ -1,60 +1,17 @@
-# routes/cleanup_routes.py
-"""Routes for cleanup operations."""
-import logging
-from fastapi import APIRouter, HTTPException, Request
-from src.cleanup_service import get_cleanup_preview, cleanup_sessions
-from src.auth_helpers import get_current_user
+"""Backward-compat shim — canonical location is routes/cleanup/cleanup_routes.py.
 
-logger = logging.getLogger(__name__)
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.cleanup_routes``, ``from routes.cleanup_routes import X``,
+``importlib.import_module("routes.cleanup_routes")``, and the string-targeted
+``monkeypatch.setattr("routes.cleanup_routes.get_cleanup_preview", ...)`` /
+``"routes.cleanup_routes.get_current_user"`` / ``"routes.cleanup_routes.
+cleanup_sessions"`` pattern used by test_cleanup_owner_scope.py all operate
+on the *same* object the application actually uses. Keeps existing import
+paths working after slice 2g (#4082/#4071).
+"""
 
-def setup_cleanup_routes(session_manager):
-    """
-    Setup cleanup-related routes.
+import sys as _sys
 
-    Args:
-        session_manager: SessionManager instance
+from routes.cleanup import cleanup_routes as _canonical  # noqa: F401
 
-    Returns:
-        APIRouter instance with cleanup routes
-    """
-    router = APIRouter(prefix="/api/cleanup")
-
-    @router.get("/preview")
-    async def cleanup_preview(request: Request):
-        """
-        Preview what would be cleaned up without making any changes.
-
-        Returns:
-            JSON response with lists of sessions that would be archived/deleted and estimated space savings
-        """
-        user = get_current_user(request)
-        try:
-            preview = await get_cleanup_preview(owner=user)
-            return preview
-        except Exception as e:
-            logger.error(f"Cleanup preview failed: {e}")
-            raise HTTPException(500, "Cleanup preview generation failed")
-
-    @router.post("")
-    async def cleanup_endpoint(request: Request):
-        """
-        Perform cleanup operations:
-        1. Archive inactive sessions (not accessed for 7 days)
-        2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages)
-
-        Returns:
-            JSON response with counts of deleted and archived sessions, and space freed
-        """
-        user = get_current_user(request)
-        try:
-            archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user)
-            return {
-                "archived_count": archived_count,
-                "deleted_count": deleted_count,
-                "space_freed_mb": round(space_freed_mb, 2)
-            }
-        except Exception as e:
-            logger.error(f"Cleanup failed: {e}")
-            raise HTTPException(500, "Cleanup operation failed")
-
-    return router
+_sys.modules[__name__] = _canonical

routes/codex_routes.py

@@ -0,0 +1,910 @@
+"""Codex integration routes.
+
+These are small HTTP surfaces intended for the Codex plugin/MCP bridge. They
+reuse existing Odysseus helpers and enforce API-token scopes before touching
+user data.
+"""
+
+import asyncio
+import json
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request
+from fastapi.responses import StreamingResponse
+
+from core.middleware import require_admin
+from src.auth_helpers import require_authenticated_request, require_user
+from src.tool_implementations import do_manage_notes
+from src.constants import COOKBOOK_STATE_FILE
+from routes._validators import validate_remote_host, validate_ssh_port
+
+
+COOKBOOK_READ_SCOPES = {"cookbook:read", "cookbook:launch"}
+COOKBOOK_LAUNCH_SCOPES = {"cookbook:launch"}
+TODO_READ_SCOPES = {"todos:read", "todos:write"}
+TODO_WRITE_SCOPES = {"todos:write"}
+EMAIL_READ_SCOPES = {"email:read", "email:draft", "email:send"}
+EMAIL_DRAFT_SCOPES = {"email:draft", "email:send"}
+EMAIL_SEND_SCOPES = {"email:send"}
+MEMORY_READ_SCOPES = {"memory:read", "memory:write"}
+MEMORY_WRITE_SCOPES = {"memory:write"}
+CALENDAR_READ_SCOPES = {"calendar:read", "calendar:write"}
+CALENDAR_WRITE_SCOPES = {"calendar:write"}
+DOCS_READ_SCOPES = {"documents:read", "documents:write"}
+DOCS_WRITE_SCOPES = {"documents:write"}
+WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"}
+
+
+def _ssh_prefix_for_task(task: dict) -> tuple[str, str]:
+    """Resolve a cookbook task's stored SSH target into ``(host, port_flag)``.
+
+    ``host`` is ``""`` for a local task. ``remoteHost`` / ``sshPort`` come from
+    cookbook_state.json and get interpolated into an ``ssh`` command string, so
+    validate them the same way the cookbook routes do. A tampered entry with
+    shell metacharacters in ``remoteHost`` is rejected with 400 rather than
+    injected.
+    """
+    raw_host = task.get("remoteHost")
+    raw_port = task.get("sshPort")
+    host_value = str(raw_host).strip() if raw_host is not None else None
+    port_value = str(raw_port).strip() if raw_port is not None else None
+    host = validate_remote_host(host_value or None) or ""
+    ssh_port = validate_ssh_port(port_value or None) or ""
+    port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
+    return host, port_flag
+
+
+async def _as_owner(request: Request, owner: str, fn, *args, **kwargs):
+    """Run an existing route handler with request.state.current_user temporarily
+    set to ``owner`` so its internal get_current_user/require_user calls see
+    the scope-gated owner (not the "api" pseudo-user the bearer middleware sets).
+    Restores the original value when done. Works for sync and async handlers."""
+    orig = getattr(request.state, "current_user", None)
+    orig_api_token = getattr(request.state, "api_token", None)
+    request.state.current_user = owner
+    request.state.api_token = False
+    try:
+        result = fn(*args, **kwargs)
+        if asyncio.iscoroutine(result):
+            result = await result
+        return result
+    finally:
+        request.state.current_user = orig
+        if orig_api_token is None:
+            try:
+                delattr(request.state, "api_token")
+            except AttributeError:
+                pass
+        else:
+            request.state.api_token = orig_api_token
+
+
+def _scope_owner(request: Request, allowed: set[str]) -> str:
+    """Return the data owner if the caller is allowed for this Codex action."""
+    if getattr(request.state, "api_token", False):
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        if not scopes.intersection(allowed):
+            required = " or ".join(sorted(allowed))
+            raise HTTPException(403, f"API token missing required scope: {required}")
+        owner = getattr(request.state, "api_token_owner", None)
+        if not owner:
+            raise HTTPException(403, "API token has no owner")
+        return owner
+    return require_user(request)
+
+
+def _scope_owner_all(request: Request, required: set[str]) -> str:
+    """Return owner only when an API token has every required scope."""
+    if getattr(request.state, "api_token", False):
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        missing = required - scopes
+        if missing:
+            raise HTTPException(403, f"API token missing required scope: {' and '.join(sorted(missing))}")
+        owner = getattr(request.state, "api_token_owner", None)
+        if not owner:
+            raise HTTPException(403, "API token has no owner")
+        return owner
+    return require_user(request)
+
+
+def _require_cookbook_scope(request: Request, allowed: set[str]) -> str:
+    """Authorize a Codex cookbook route.
+
+    For API-token callers, enforce the given scope set.
+    For cookie-session callers, additionally require admin privileges
+    because cookbook surfaces expose host topology, task logs, tmux
+    commands, and model-serving controls.
+    """
+    owner = _scope_owner(request, allowed)
+    if not getattr(request.state, "api_token", False):
+        require_admin(request)
+    return owner
+
+
+def _find_endpoint(router: APIRouter | None, method: str, path: str):
+    if router is None:
+        return None
+    for route in getattr(router, "routes", []):
+        if getattr(route, "path", "") == path and method in getattr(route, "methods", set()):
+            return route.endpoint
+    return None
+
+
+def _clamp_pagination(offset: Any, limit: Any, *, default_limit: int = 50, max_limit: int = 50) -> tuple[int, int]:
+    try:
+        parsed_offset = int(0 if offset in (None, "") else offset)
+    except (TypeError, ValueError):
+        raise HTTPException(400, "Invalid offset")
+    try:
+        parsed_limit = int(default_limit if limit in (None, "") else limit)
+    except (TypeError, ValueError):
+        raise HTTPException(400, "Invalid limit")
+    return max(0, parsed_offset), max(1, min(parsed_limit, max_limit))
+
+
+def setup_codex_routes(
+    email_router: APIRouter | None = None,
+    memory_router: APIRouter | None = None,
+    calendar_router: APIRouter | None = None,
+    document_router: APIRouter | None = None,
+) -> APIRouter:
+    router = APIRouter(prefix="/api/codex", tags=["codex"])
+    email_list_endpoint = _find_endpoint(email_router, "GET", "/api/email/list")
+    email_read_endpoint = _find_endpoint(email_router, "GET", "/api/email/read/{uid}")
+    email_send_endpoint = _find_endpoint(email_router, "POST", "/api/email/send")
+    email_draft_endpoint = _find_endpoint(email_router, "POST", "/api/email/draft")
+    memory_list_endpoint = _find_endpoint(memory_router, "GET", "/api/memory")
+    memory_add_endpoint = _find_endpoint(memory_router, "POST", "/api/memory/add")
+    calendar_list_events = _find_endpoint(calendar_router, "GET", "/api/calendar/events")
+    calendar_create_event = _find_endpoint(calendar_router, "POST", "/api/calendar/events")
+    documents_library_endpoint = _find_endpoint(document_router, "GET", "/api/documents/library")
+    documents_get_endpoint = _find_endpoint(document_router, "GET", "/api/document/{doc_id}")
+    documents_create_endpoint = _find_endpoint(document_router, "POST", "/api/document")
+
+    @router.get("/capabilities")
+    def capabilities(request: Request):
+        token_scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        has_token = bool(getattr(request.state, "api_token", False))
+        def scoped(allowed):
+            return bool(token_scopes.intersection(allowed)) if has_token else True
+        return {
+            "integration": "codex",
+            "token_scopes": sorted(token_scopes),
+            "tools": {
+                "todos": {
+                    "read": scoped(TODO_READ_SCOPES),
+                    "write": scoped(TODO_WRITE_SCOPES),
+                    "actions": ["list", "add", "update", "delete", "toggle_item"],
+                },
+                "email": {
+                    "read": scoped(EMAIL_READ_SCOPES),
+                    "draft": scoped(EMAIL_DRAFT_SCOPES),
+                    "send": scoped(EMAIL_SEND_SCOPES),
+                    "actions": ["list", "read", "draft_document", "draft", "send"],
+                },
+                "memory": {
+                    "read": scoped(MEMORY_READ_SCOPES),
+                    "write": scoped(MEMORY_WRITE_SCOPES),
+                    "actions": ["list", "add", "delete"],
+                    "available": memory_list_endpoint is not None,
+                },
+                "calendar": {
+                    "read": scoped(CALENDAR_READ_SCOPES),
+                    "write": scoped(CALENDAR_WRITE_SCOPES),
+                    "actions": ["list_events", "create_event", "delete_event"],
+                    "available": calendar_list_events is not None,
+                },
+                "documents": {
+                    "read": scoped(DOCS_READ_SCOPES),
+                    "write": scoped(DOCS_WRITE_SCOPES),
+                    "actions": ["library", "read", "create", "delete"],
+                    "available": documents_library_endpoint is not None,
+                },
+                "cookbook": {
+                    "read": scoped(COOKBOOK_READ_SCOPES),
+                    "launch": scoped(COOKBOOK_LAUNCH_SCOPES),
+                    "actions": ["tasks", "servers", "output", "serve", "stop"],
+                },
+            },
+            "safety": {
+                "email_send_requires_confirmation": True,
+                "destructive_actions_should_confirm": True,
+            },
+        }
+
+    @router.get("/plugin.zip")
+    def plugin_zip(request: Request):
+        require_authenticated_request(request)
+        root = Path(__file__).resolve().parent.parent / "integrations" / "codex"
+        if not root.exists():
+            raise HTTPException(404, "Codex plugin bundle not found")
+        buf = BytesIO()
+        with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+            for path in sorted(root.rglob("*")):
+                if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc":
+                    continue
+                zf.write(path, Path("odysseus") / path.relative_to(root))
+        buf.seek(0)
+        headers = {"Content-Disposition": 'attachment; filename="odysseus-codex-plugin.zip"'}
+        return StreamingResponse(buf, media_type="application/zip", headers=headers)
+
+    @router.get("/todos")
+    async def list_todos(request: Request, archived: bool = False, label: str | None = None):
+        owner = _scope_owner(request, TODO_READ_SCOPES)
+        args: dict[str, Any] = {"action": "list", "archived": archived}
+        if label:
+            args["label"] = label
+        return await do_manage_notes(json.dumps(args), owner=owner)
+
+    @router.post("/todos")
+    async def manage_todos(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        action = str(body.get("action") or "add").replace("-", "_").strip().lower()
+        allowed = TODO_WRITE_SCOPES if action in WRITE_ACTIONS else TODO_READ_SCOPES
+        owner = _scope_owner(request, allowed)
+        args = dict(body)
+        args["action"] = action
+        return await do_manage_notes(json.dumps(args), owner=owner)
+
+    @router.get("/emails")
+    async def list_emails(
+        request: Request,
+        folder: str = "INBOX",
+        limit: int = 10,
+        offset: int = 0,
+        filter: str = "all",
+        from_addr: str | None = None,
+        account_id: str | None = None,
+        has_attachments: int = 0,
+    ):
+        owner = _scope_owner(request, EMAIL_READ_SCOPES)
+        if email_list_endpoint is None:
+            raise HTTPException(503, "Email integration is not available")
+        limit = max(1, min(int(limit or 10), 50))
+        offset = max(0, int(offset or 0))
+        if account_id:
+            from routes.email_helpers import _assert_owns_account
+
+            _assert_owns_account(account_id, owner)
+        return await email_list_endpoint(
+            folder=folder,
+            limit=limit,
+            offset=offset,
+            filter=filter,
+            from_addr=from_addr,
+            account_id=account_id,
+            has_attachments=has_attachments,
+            cache_bust=None,
+            owner=owner,
+        )
+
+    @router.get("/emails/{uid}")
+    async def read_email(
+        request: Request,
+        uid: str,
+        folder: str = "INBOX",
+        account_id: str | None = None,
+        mark_seen: bool = False,
+    ):
+        owner = _scope_owner(request, EMAIL_READ_SCOPES)
+        if email_read_endpoint is None:
+            raise HTTPException(503, "Email integration is not available")
+        if account_id:
+            from routes.email_helpers import _assert_owns_account
+
+            _assert_owns_account(account_id, owner)
+        return await email_read_endpoint(
+            uid=uid,
+            folder=folder,
+            account_id=account_id,
+            mark_seen=mark_seen,
+            owner=owner,
+        )
+
+    # ── Email draft + send ────────────────────────────────────────────────
+    # Both handlers in routes/email_routes.py already accept `owner=` via
+    # FastAPI Depends, so we call them directly without patching state.
+
+    def _email_draft_document_content(body: dict[str, Any]) -> str:
+        def clean(v: Any) -> str:
+            if isinstance(v, list):
+                return ", ".join(str(x).strip() for x in v if str(x).strip())
+            return str(v or "").strip()
+
+        to = clean(body.get("to"))
+        cc = clean(body.get("cc"))
+        bcc = clean(body.get("bcc"))
+        subject = clean(body.get("subject"))
+        in_reply_to = clean(body.get("in_reply_to"))
+        references = clean(body.get("references"))
+        body_text = str(body.get("body") or body.get("body_html") or "").strip()
+        lines = [
+            f"To: {to}",
+        ]
+        if cc:
+            lines.append(f"Cc: {cc}")
+        if bcc:
+            lines.append(f"Bcc: {bcc}")
+        lines.append(f"Subject: {subject}")
+        if in_reply_to:
+            lines.append(f"In-Reply-To: {in_reply_to}")
+        if references:
+            lines.append(f"References: {references}")
+        lines.extend(["---", body_text])
+        return "\n".join(lines).rstrip() + "\n"
+
+    @router.post("/emails/draft-document")
+    async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
+        docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES)
+        if docs_owner != owner:
+            raise HTTPException(403, "API token owner mismatch")
+        if documents_create_endpoint is None:
+            raise HTTPException(503, "Documents integration is not available")
+        from routes.document_routes import DocumentCreate
+
+        subject = str(body.get("subject") or "Email draft").strip() or "Email draft"
+        title = str(body.get("title") or subject).strip() or "Email draft"
+        req = DocumentCreate(
+            session_id=body.get("session_id"),
+            title=title,
+            language="email",
+            content=_email_draft_document_content(body),
+        )
+        result = await _as_owner(request, owner, documents_create_endpoint, request, req)
+        if isinstance(result, dict):
+            result = dict(result)
+            result["draft_type"] = "document"
+            result["send_required_confirmation"] = True
+        return result
+
+    @router.post("/emails/draft")
+    async def codex_email_draft(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
+        if email_draft_endpoint is None:
+            raise HTTPException(503, "Email integration is not available")
+        from routes.email_routes import SendEmailRequest
+
+        try:
+            req = SendEmailRequest(**body)
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid draft payload: {exc}")
+        return await email_draft_endpoint(req=req, owner=owner)
+
+    @router.post("/emails/send")
+    async def codex_email_send(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, EMAIL_SEND_SCOPES)
+        if email_send_endpoint is None:
+            raise HTTPException(503, "Email integration is not available")
+        from routes.email_routes import SendEmailRequest
+
+        try:
+            req = SendEmailRequest(**body)
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid send payload: {exc}")
+        return await email_send_endpoint(req=req, background_tasks=BackgroundTasks(), owner=owner)
+
+    # ── Memory ────────────────────────────────────────────────────────────
+
+    @router.get("/memory")
+    async def codex_memory_list(request: Request):
+        owner = _scope_owner(request, MEMORY_READ_SCOPES)
+        if memory_list_endpoint is None:
+            raise HTTPException(503, "Memory integration is not available")
+        return await _as_owner(request, owner, memory_list_endpoint, request)
+
+    @router.post("/memory")
+    async def codex_memory_add(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, MEMORY_WRITE_SCOPES)
+        if memory_add_endpoint is None:
+            raise HTTPException(503, "Memory integration is not available")
+        from src.request_models import MemoryAddRequest
+
+        try:
+            memory_data = MemoryAddRequest(
+                text=str(body.get("text") or "").strip(),
+                category=body.get("category", "fact"),
+                source=body.get("source", "user"),
+                session_id=body.get("session_id"),
+            )
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid memory payload: {exc}")
+        if not memory_data.text:
+            raise HTTPException(400, "Empty memory text")
+        return await _as_owner(request, owner, memory_add_endpoint, request, memory_data)
+
+    # ── Calendar ──────────────────────────────────────────────────────────
+
+    @router.get("/calendar/events")
+    async def codex_calendar_list(request: Request, start: str, end: str, calendar: str = ""):
+        owner = _scope_owner(request, CALENDAR_READ_SCOPES)
+        if calendar_list_events is None:
+            raise HTTPException(503, "Calendar integration is not available")
+        return await _as_owner(request, owner, calendar_list_events, request, start, end, calendar)
+
+    @router.post("/calendar/events")
+    async def codex_calendar_create(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, CALENDAR_WRITE_SCOPES)
+        if calendar_create_event is None:
+            raise HTTPException(503, "Calendar integration is not available")
+        from routes.calendar_routes import EventCreate
+
+        try:
+            data = EventCreate(**body)
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid event payload: {exc}")
+        return await _as_owner(request, owner, calendar_create_event, request, data)
+
+    # ── Documents ─────────────────────────────────────────────────────────
+
+    @router.get("/documents")
+    async def codex_documents_library(
+        request: Request,
+        search: str | None = None,
+        language: str | None = None,
+        sort: str = "recent",
+        offset: int = 0,
+        limit: int = 50,
+        archived: bool = False,
+    ):
+        owner = _scope_owner(request, DOCS_READ_SCOPES)
+        if documents_library_endpoint is None:
+            raise HTTPException(503, "Documents integration is not available")
+        offset, limit = _clamp_pagination(offset, limit)
+        result = await _as_owner(
+            request, owner, documents_library_endpoint,
+            request, search, language, sort, offset, limit, archived,
+        )
+        if isinstance(result, dict):
+            docs = result.get("documents")
+            total = result.get("total")
+            if isinstance(docs, list) and isinstance(total, int):
+                next_offset = offset + len(docs)
+                result["next_offset"] = next_offset if next_offset < total else None
+        return result
+
+    @router.get("/documents/{doc_id}")
+    async def codex_documents_get(request: Request, doc_id: str):
+        owner = _scope_owner(request, DOCS_READ_SCOPES)
+        if documents_get_endpoint is None:
+            raise HTTPException(503, "Documents integration is not available")
+        return await _as_owner(request, owner, documents_get_endpoint, request, doc_id)
+
+    # ── DELETE endpoints so agents can clean up after themselves ──────────
+
+    memory_delete_endpoint = _find_endpoint(memory_router, "DELETE", "/api/memory/{memory_id}")
+    calendar_delete_event = _find_endpoint(calendar_router, "DELETE", "/api/calendar/events/{uid}")
+    documents_delete_endpoint = _find_endpoint(document_router, "DELETE", "/api/document/{doc_id}")
+
+    @router.delete("/memory/{memory_id}")
+    async def codex_memory_delete(request: Request, memory_id: str):
+        owner = _scope_owner(request, MEMORY_WRITE_SCOPES)
+        if memory_delete_endpoint is None:
+            raise HTTPException(503, "Memory delete not available")
+        return await _as_owner(request, owner, memory_delete_endpoint, request, memory_id)
+
+    @router.delete("/calendar/events/{uid}")
+    async def codex_calendar_delete(request: Request, uid: str):
+        owner = _scope_owner(request, CALENDAR_WRITE_SCOPES)
+        if calendar_delete_event is None:
+            raise HTTPException(503, "Calendar delete not available")
+        return await _as_owner(request, owner, calendar_delete_event, request, uid)
+
+    @router.delete("/documents/{doc_id}")
+    async def codex_documents_delete(request: Request, doc_id: str):
+        owner = _scope_owner(request, DOCS_WRITE_SCOPES)
+        if documents_delete_endpoint is None:
+            raise HTTPException(503, "Documents delete not available")
+        return await _as_owner(request, owner, documents_delete_endpoint, request, doc_id)
+
+    @router.post("/documents")
+    async def codex_documents_create(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        owner = _scope_owner(request, DOCS_WRITE_SCOPES)
+        if documents_create_endpoint is None:
+            raise HTTPException(503, "Documents integration is not available")
+        from routes.document_routes import DocumentCreate
+
+        try:
+            req = DocumentCreate(**body)
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid document payload: {exc}")
+        return await _as_owner(request, owner, documents_create_endpoint, request, req)
+
+    # ── Cookbook surface ──
+    # Lets the agent run the same launch / monitor / kill loop the user
+    # would do by hand in the Cookbook UI: read the current task list +
+    # tmux output, launch a serve task, stop one.  Two scopes:
+    #   cookbook:read   — list tasks + tail output + list servers
+    #   cookbook:launch — also start/stop serves (host shell exec)
+    # `cookbook:launch` is genuinely powerful: /api/model/serve runs SSH'd
+    # commands on the user's hosts. The existing _validate_serve_cmd
+    # allowlist (vllm/python3/sglang/llama-server/etc., no shell metachars)
+    # keeps the agent inside the same sandbox the UI uses.
+
+    async def _run_shell(cmd: str, timeout: float = 15.0) -> dict:
+        """Run a shell command, return {exit_code, stdout, stderr}."""
+        import asyncio as _asyncio
+        try:
+            proc = await _asyncio.create_subprocess_shell(
+                cmd,
+                stdout=_asyncio.subprocess.PIPE,
+                stderr=_asyncio.subprocess.PIPE,
+            )
+            try:
+                stdout_b, stderr_b = await _asyncio.wait_for(proc.communicate(), timeout=timeout)
+            except _asyncio.TimeoutError:
+                proc.kill()
+                return {"exit_code": -1, "stdout": "", "stderr": "timed out"}
+            return {
+                "exit_code": proc.returncode,
+                "stdout": stdout_b.decode(errors="replace"),
+                "stderr": stderr_b.decode(errors="replace"),
+            }
+        except Exception as exc:
+            return {"exit_code": -1, "stdout": "", "stderr": str(exc)}
+
+    def _read_cookbook_state() -> dict:
+        from pathlib import Path as _Path
+        import json as _json
+        p = _Path(COOKBOOK_STATE_FILE)
+        if not p.exists():
+            return {}
+        try:
+            return _json.loads(p.read_text(encoding="utf-8"))
+        except Exception:
+            return {}
+
+    def _redact_task(t: dict) -> dict:
+        """Strip secrets before returning to the agent."""
+        clean = {k: v for k, v in t.items() if k not in ("hf_token", "_secrets")}
+        if isinstance(clean.get("payload"), dict):
+            pl = clean["payload"]
+            clean["payload"] = {k: v for k, v in pl.items()
+                                if k not in ("hf_token", "_secrets")}
+        return clean
+
+    @router.get("/cookbook/tasks")
+    async def codex_cookbook_tasks(request: Request):
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
+        state = _read_cookbook_state()
+        tasks = state.get("tasks") or []
+        return {"tasks": [_redact_task(t) for t in tasks]}
+
+    @router.get("/cookbook/servers")
+    async def codex_cookbook_servers(request: Request):
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
+        state = _read_cookbook_state()
+        servers = state.get("env", {}).get("servers") or []
+        # Strip ssh creds / passwords; keep only what's needed to pick a host.
+        cleaned = []
+        for s in servers:
+            cleaned.append({
+                "name": s.get("name"),
+                "host": s.get("host"),
+                "port": s.get("port"),
+                "env": s.get("env"),
+                "envPath": s.get("envPath"),
+                "platform": s.get("platform"),
+                "modelDirs": s.get("modelDirs"),
+            })
+        return {"servers": cleaned}
+
+    @router.get("/cookbook/output/{session_id}")
+    async def codex_cookbook_output(request: Request, session_id: str, tail: int = 400):
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
+        # Defensive: session_id must be the tmux-style id we issue
+        # (`serve-XXXX` / `cookbook-XXXX` / `queue-XXXX`); anything else
+        # would let the agent run arbitrary `tmux capture-pane` targets.
+        import re as _re
+        if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id):
+            raise HTTPException(400, "Invalid session id")
+        tail = max(20, min(int(tail or 400), 4000))
+        # Resolve the task's host (if any) from cookbook state so we can
+        # ssh to the right box, exactly as the UI does in _reconnectTask.
+        state = _read_cookbook_state()
+        tasks = state.get("tasks") or []
+        task = next((t for t in tasks if t.get("sessionId") == session_id), None)
+        if task is None:
+            raise HTTPException(404, "task not found")
+        host, port_flag = _ssh_prefix_for_task(task)
+        # Prefer the persisted log file over the tmux pane. The pane gets
+        # overwritten by the post-crash neofetch banner + bash prompt the
+        # moment vllm exits; the log file is the raw stdout/stderr and
+        # survives unchanged. Falls back to pane for older tasks predating
+        # the tee-to-log runner change.
+        log_path = f"/tmp/odysseus-tmux/{session_id}.log"
+        inner = (
+            f"if [ -s {log_path} ]; then tail -n {tail} {log_path}; "
+            f"else tmux capture-pane -t {session_id} -p -S -{tail}; fi"
+        )
+        if host:
+            import shlex
+            cmd = f"ssh {port_flag}{host} {shlex.quote(inner)}"
+        else:
+            cmd = inner
+        result = await _run_shell(cmd, timeout=15)
+        return {
+            "session_id": session_id,
+            "host": host or "local",
+            "exit_code": result.get("exit_code"),
+            "output": result.get("stdout", ""),
+            "task": _redact_task(task),
+        }
+
+    @router.post("/cookbook/serve")
+    async def codex_cookbook_serve(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
+        # Wraps /api/model/serve with the SAME validation the UI uses.
+        # _validate_serve_cmd (called inside model_serve) rejects shell
+        # metachars and requires the leading binary to be in the
+        # cookbook allowlist (vllm / python3 / sglang / llama-server / ...).
+        from routes.cookbook_helpers import ServeRequest
+        # Accept friendly aliases agents naturally reach for. Without these,
+        # passing `host` silently maps to nothing and the serve runs LOCAL
+        # instead of on the intended remote — exactly the bug an agent
+        # would never debug on its own.
+        norm = dict(body or {})
+        if "host" in norm and "remote_host" not in norm:
+            norm["remote_host"] = norm.pop("host")
+        if "model" in norm and "repo_id" not in norm:
+            norm["repo_id"] = norm.pop("model")
+        if "ssh_port" not in norm and "port" in norm and (str(norm.get("port") or "").isdigit() and int(norm["port"]) >= 1000):
+            # Heuristic: if `port` looks like an SSH port (≥1000) and there's
+            # no explicit ssh_port, treat it as such. UI ports (8000, 8001,
+            # 30000) belong inside the cmd string, not here.
+            pass  # leave as-is — user's `port` here is ambiguous; skip remap.
+        try:
+            req = ServeRequest(**norm)
+        except Exception as exc:
+            raise HTTPException(400, f"Invalid serve payload: {exc}")
+        serve_endpoint = _find_endpoint(None, "POST", "/api/model/serve")
+        # Fall back to importing from the cookbook router registered on app.
+        if serve_endpoint is None:
+            from fastapi import FastAPI
+            app: FastAPI = request.app
+            for route in app.routes:
+                if getattr(route, "path", None) == "/api/model/serve" and "POST" in getattr(route, "methods", set()):
+                    serve_endpoint = route.endpoint
+                    break
+        if serve_endpoint is None:
+            raise HTTPException(503, "model serve endpoint unavailable")
+        return await serve_endpoint(request, req)
+
+    @router.post("/cookbook/stop/{session_id}")
+    async def codex_cookbook_stop(request: Request, session_id: str):
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
+        import re as _re
+        if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id):
+            raise HTTPException(400, "Invalid session id")
+        state = _read_cookbook_state()
+        tasks = state.get("tasks") or []
+        task = next((t for t in tasks if t.get("sessionId") == session_id), None)
+        host, port_flag = _ssh_prefix_for_task(task or {})
+        if host:
+            cmd = f"ssh {port_flag}{host} \"tmux kill-session -t {session_id}\""
+        else:
+            cmd = f"tmux kill-session -t {session_id}"
+        result = await _run_shell(cmd, timeout=10)
+        return {"session_id": session_id, "exit_code": result.get("exit_code"), "host": host or "local"}
+
+    @router.get("/cookbook/cached")
+    async def codex_cookbook_cached(request: Request, host: str | None = None):
+        """List cached models on a configured server (or local if host is omitted).
+        Mirrors `list_cached_models` from the chat agent so external agents have
+        the same inventory view before deciding what to serve/download."""
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
+        # Hit /api/model/cached internally, with the same modelDirs the chat
+        # agent's list_cached_models would resolve from cookbook state.
+        state = _read_cookbook_state()
+        env = state.get("env") if isinstance(state, dict) else {}
+        servers = (env.get("servers") if isinstance(env, dict) else None) or []
+        HF_DEFAULTS = {"~/.cache/huggingface/hub", "~/.cache/huggingface"}
+        def _dirs_for(srv: dict) -> str:
+            mds = srv.get("modelDirs") if isinstance(srv, dict) else None
+            if isinstance(mds, list):
+                extras = [d for d in mds if isinstance(d, str) and d.strip() and d.strip() not in HF_DEFAULTS]
+                return ",".join(extras)
+            if isinstance(mds, str) and mds.strip() not in HF_DEFAULTS:
+                return mds
+            return ""
+        # Resolve friendly host name → real host (matches list_cached_models flow).
+        resolved_host = host or ""
+        srv: dict[str, Any] = {}
+        if host:
+            srv = next(
+                (s for s in servers if isinstance(s, dict)
+                 and (s.get("name") == host or s.get("host") == host)),
+                {},
+            )
+            if srv and srv.get("host"):
+                resolved_host = srv["host"]
+        else:
+            srv = next((s for s in servers if isinstance(s, dict) and not (s.get("host") or "").strip()), {})
+        params: dict[str, str] = {}
+        if resolved_host:
+            params["host"] = resolved_host
+        md = _dirs_for(srv)
+        if md:
+            params["model_dir"] = md
+        if srv.get("port"):
+            params["ssh_port"] = str(srv["port"])
+        if srv.get("platform"):
+            params["platform"] = srv["platform"]
+        cached_endpoint = _find_endpoint(None, "GET", "/api/model/cached")
+        if cached_endpoint is None:
+            from fastapi import FastAPI
+            app: FastAPI = request.app
+            for route in app.routes:
+                if getattr(route, "path", None) == "/api/model/cached" and "GET" in getattr(route, "methods", set()):
+                    cached_endpoint = route.endpoint
+                    break
+        if cached_endpoint is None:
+            raise HTTPException(503, "model cached endpoint unavailable")
+        # The endpoint reads host/model_dir/ssh_port/platform as kwargs.
+        return await cached_endpoint(
+            request,
+            host=params.get("host") or None,
+            model_dir=params.get("model_dir") or None,
+            ssh_port=params.get("ssh_port") or None,
+            platform=params.get("platform") or None,
+        )
+
+    @router.get("/cookbook/presets")
+    async def codex_cookbook_presets(request: Request):
+        """List saved serve presets (model + host + port + launch cmd).
+        Counterpart to `list_serve_presets`. Use BEFORE composing a `serve`
+        body — the user's saved preset usually has the working cmd already."""
+        _require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
+        state = _read_cookbook_state()
+        presets = state.get("presets") or []
+        out = []
+        for p in presets:
+            if not isinstance(p, dict):
+                continue
+            out.append({
+                "name": p.get("name"),
+                "model": p.get("model") or p.get("modelId"),
+                "host": p.get("host") or p.get("remoteHost"),
+                "port": p.get("port"),
+                "cmd": p.get("cmd"),
+            })
+        return {"presets": out, "default_host": (state.get("env") or {}).get("defaultServer", "")}
+
+    @router.post("/cookbook/preset/{name}")
+    async def codex_cookbook_serve_preset(request: Request, name: str):
+        """Launch a saved preset by name. Reuses the working cmd + host the
+        user already saved, avoiding the cmd-allowlist trial-and-error loop."""
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
+        import re as _re
+        if not _re.fullmatch(r"[A-Za-z0-9 _.:@\-]+", name):
+            raise HTTPException(400, "Invalid preset name")
+        state = _read_cookbook_state()
+        presets = state.get("presets") or []
+        lname = name.lower().strip()
+        chosen = next(
+            (p for p in presets if isinstance(p, dict) and (p.get("name") or "").lower() == lname),
+            None,
+        )
+        if chosen is None:
+            chosen = next(
+                (p for p in presets if isinstance(p, dict) and lname in (p.get("name") or "").lower()),
+                None,
+            )
+        if chosen is None:
+            raise HTTPException(404, f"No preset matching {name!r}")
+        repo_id = chosen.get("model") or chosen.get("modelId") or ""
+        cmd = (chosen.get("cmd") or "").strip()
+        host = chosen.get("host") or chosen.get("remoteHost") or ""
+        if not repo_id or not cmd or cmd.startswith("(adopted"):
+            raise HTTPException(400, f"Preset {chosen.get('name')!r} has no launchable cmd "
+                                     "(adopted from external launch). Use POST /cookbook/serve "
+                                     "with the actual cmd instead.")
+        # Reuse the serve handler we already validated.
+        from routes.cookbook_helpers import ServeRequest
+        body = {"repo_id": repo_id, "cmd": cmd}
+        if host:
+            body["remote_host"] = host
+        try:
+            req = ServeRequest(**body)
+        except Exception as exc:
+            raise HTTPException(400, f"Preset payload invalid: {exc}")
+        serve_endpoint = _find_endpoint(None, "POST", "/api/model/serve")
+        if serve_endpoint is None:
+            from fastapi import FastAPI
+            app: FastAPI = request.app
+            for route in app.routes:
+                if getattr(route, "path", None) == "/api/model/serve" and "POST" in getattr(route, "methods", set()):
+                    serve_endpoint = route.endpoint
+                    break
+        if serve_endpoint is None:
+            raise HTTPException(503, "model serve endpoint unavailable")
+        return await serve_endpoint(request, req)
+
+    @router.post("/cookbook/adopt")
+    async def codex_cookbook_adopt(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
+        """Adopt an existing tmux session (one started via raw ssh+tmux) into
+        cookbook tracking. Needed when serve_model rejects a cmd and the
+        agent falls back to direct ssh — without adoption the session is
+        invisible to the UI. Body: {tmux_session, model, host?, port?}."""
+        _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
+        norm = dict(body or {})
+        sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip()
+        model = (norm.get("model") or norm.get("repo_id") or "").strip()
+        host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or ""
+        port = norm.get("port") or 8000
+        import re as _re
+        if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess):
+            raise HTTPException(400, "tmux_session required, [a-zA-Z0-9_-]+ only")
+        if not model:
+            raise HTTPException(400, "model required")
+        # Verify the tmux session exists on the target host before adopting.
+        import shlex
+        if host:
+            check = f"ssh {shlex.quote(host)} 'tmux has-session -t {shlex.quote(sess)}'"
+        else:
+            check = f"tmux has-session -t {shlex.quote(sess)}"
+        chk = await _run_shell(check, timeout=8)
+        if chk.get("exit_code") not in (0, None):
+            raise HTTPException(404, f"tmux session {sess!r} not found on {host or 'local'}")
+        # Write into cookbook_state.json.
+        import time as _t, json as _json
+        from core.atomic_io import atomic_write_json
+        from pathlib import Path as _Path
+        cookbook_state_path = _Path(COOKBOOK_STATE_FILE)
+        try:
+            state = _json.loads(cookbook_state_path.read_text(encoding="utf-8"))
+        except Exception:
+            state = {}
+        tasks = state.setdefault("tasks", [])
+        if any(isinstance(t, dict) and t.get("sessionId") == sess for t in tasks):
+            return {"ok": True, "already_tracked": True, "session_id": sess}
+        tasks.append({
+            "id": sess, "sessionId": sess,
+            "name": model.split("/")[-1] if "/" in model else model,
+            "type": "serve", "status": "running",
+            "output": f"Adopted externally-launched session {sess!r} on {host or 'local'}.",
+            "ts": int(_t.time() * 1000),
+            "payload": {"repo_id": model, "remote_host": host, "_cmd": "(adopted — launched outside cookbook)", "port": int(port)},
+            "remoteHost": host, "sshPort": "", "platform": "linux",
+            "_serveReady": False, "_endpointAdded": False, "_adoptedExternally": True,
+        })
+        try:
+            atomic_write_json(cookbook_state_path, state)
+        except Exception as exc:
+            raise HTTPException(500, f"state write failed: {exc}")
+        return {"ok": True, "session_id": sess, "host": host or "local"}
+
+    return router
+
+
+def setup_claude_routes() -> APIRouter:
+    """Serve the Claude Code skill bundle.
+
+    Claude Code uses the same scope-gated `/api/codex/*` endpoints at runtime;
+    this router only exists to deliver the skill zip via `/api/claude/plugin.zip`
+    so the user-facing setup commands stay in the Claude namespace.
+    """
+    router = APIRouter(prefix="/api/claude", tags=["claude"])
+
+    @router.get("/plugin.zip")
+    def plugin_zip(request: Request):
+        require_authenticated_request(request)
+        # Only ship the skills/ subtree so extracting at ~/.claude/ doesn't dump
+        # README.md or other bundle metadata into the user's claude config dir.
+        skills_root = Path(__file__).resolve().parent.parent / "integrations" / "claude" / "skills"
+        if not skills_root.exists():
+            raise HTTPException(404, "Claude skill bundle not found")
+        bundle_root = skills_root.parent
+        buf = BytesIO()
+        with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+            for path in sorted(skills_root.rglob("*")):
+                if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc":
+                    continue
+                zf.write(path, path.relative_to(bundle_root))
+        buf.seek(0)
+        headers = {"Content-Disposition": 'attachment; filename="odysseus-claude-skill.zip"'}
+        return StreamingResponse(buf, media_type="application/zip", headers=headers)
+
+    return router

routes/compare/__init__.py

@@ -0,0 +1,5 @@
+"""Compare route domain package (slice 2i, #4082/#4071).
+
+Contains compare_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/compare_routes.py re-exports from here.
+"""

routes/compare/compare_routes.py

@@ -0,0 +1,365 @@
+# routes/compare_routes.py
+"""Model A/B comparison routes."""
+import json
+import uuid
+import random
+from datetime import datetime
+from fastapi import APIRouter, Form, HTTPException, Request
+from typing import List
+from pydantic import BaseModel
+import logging
+
+from core.database import Comparison, SessionLocal
+from core.session_manager import SessionManager
+from src.auth_helpers import get_current_user
+from routes.session_routes import _reject_raw_endpoint_url_for_non_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/compare", tags=["compare"])
+
+
+def _owned_endpoint_by_url(db, base_url, owner):
+    """ModelEndpoint whose base_url == `base_url` and is VISIBLE to `owner`
+    (their own rows + legacy null-owner "shared" rows); None otherwise.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. start_comparison copies the matched row's api_key
+    into the caller-owned [CMP] session's headers, which then drives that session's
+    /api/chat_stream calls — so an UNSCOPED base_url match would let a user mint a
+    comparison bound to ANOTHER user's private endpoint and spend that owner's
+    api_key / reach whatever base_url they configured. Mirrors
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from core.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _owned_endpoint_by_id(db, endpoint_id, owner):
+    """ModelEndpoint whose id == `endpoint_id` and is VISIBLE to `owner` (their
+    own rows + legacy null-owner "shared" rows); None otherwise.
+
+    Preferred over _owned_endpoint_by_url for credential resolution: two visible
+    endpoints can share the same base_url but hold DIFFERENT api_keys (e.g. two
+    accounts on the same provider). A base_url-only match returns whichever row
+    sorts first, so it can copy the WRONG owner-scoped key into the [CMP] session.
+    An id pins the exact registered endpoint, so /api/compare/start prefers it and
+    only falls back to URL matching for legacy / admin raw-URL callers. Owner
+    scoping is identical to _owned_endpoint_by_url (a null/empty owner is a no-op).
+    """
+    from core.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+class RecordVoteRequest(BaseModel):
+    prompt: str
+    models: List[str]
+    winner: str           # model name or "tie"
+    is_blind: bool = True
+
+
+def setup_compare_routes(session_manager: SessionManager):
+    """Setup comparison routes."""
+
+    @router.post("/start")
+    def start_comparison(
+        request: Request,
+        prompt: str = Form(...),
+        model_a: str = Form(...),
+        model_b: str = Form(...),
+        endpoint_a: str = Form(""),
+        endpoint_b: str = Form(""),
+        endpoint_a_id: str = Form(""),
+        endpoint_b_id: str = Form(""),
+        is_blind: str = Form("true"),
+    ):
+        """Create two ephemeral sessions and a comparison record.
+
+        Returns the comparison ID and the two session IDs so the client
+        can fire two independent SSE streams to /api/chat_stream.
+        """
+        user = getattr(request.state, 'current_user', None)
+        comp_id = str(uuid.uuid4())
+        sid_a = str(uuid.uuid4())
+        sid_b = str(uuid.uuid4())
+
+        # Blind mapping: randomly assign left/right
+        blind = str(is_blind).lower() == "true"
+        if blind:
+            mapping = {"left": "a", "right": "b"}
+            if random.random() > 0.5:
+                mapping = {"left": "b", "right": "a"}
+        else:
+            mapping = {"left": "a", "right": "b"}
+
+        # Map session IDs to left/right based on blind mapping
+        session_left = sid_a if mapping["left"] == "a" else sid_b
+        session_right = sid_a if mapping["right"] == "a" else sid_b
+
+        # In blind mode, name the helper sessions by their neutral slot
+        # ("Model A" / "Model B") instead of the real model. Otherwise the
+        # session name leaks the model in the sidebar and GET /api/sessions,
+        # de-anonymizing the comparison before the user votes (issue #1285).
+        slot_name = {session_left: "Model A", session_right: "Model B"}
+
+        # SECURITY: resolve and validate BOTH endpoints before creating any
+        # session. Compare copies a registered endpoint's Authorization header
+        # into the [CMP] session, so validating one endpoint while creating its
+        # session, then rejecting the other, would leave a partial compare
+        # session behind with that header attached. Doing all the owner-scope
+        # resolution + raw-URL rejection up front means a 403 on either endpoint
+        # aborts the whole request with nothing created and no header copied.
+        from src.endpoint_resolver import build_chat_url, build_headers, normalize_base
+        resolved = []
+        db = SessionLocal()
+        try:
+            for sid, model, endpoint, endpoint_id in [
+                (sid_a, model_a, endpoint_a, endpoint_a_id),
+                (sid_b, model_b, endpoint_b, endpoint_b_id),
+            ]:
+                # Prefer an explicit endpoint id: it pins the EXACT registered
+                # endpoint (and its api_key), even when two endpoints visible to
+                # the caller share a base_url with different keys — a URL-only
+                # match would copy whichever row sorts first, i.e. possibly the
+                # wrong key. Fall back to URL resolution only for legacy / admin
+                # raw-URL callers that don't send an id.
+                eid = endpoint_id.strip() if isinstance(endpoint_id, str) else ""
+                if eid:
+                    ep = _owned_endpoint_by_id(db, eid, user)
+                    if ep is None:
+                        # An id the caller can't see (wrong owner / deleted) must
+                        # NOT silently fall back to a same-URL row with a different
+                        # key — that's exactly the mix-up ids exist to prevent.
+                        raise HTTPException(404, "Model endpoint not found")
+                    # The id already resolved the endpoint; ignore any raw URL the
+                    # caller also sent and dial the stored config instead.
+                    endpoint = ep.base_url
+                elif not endpoint:
+                    raise HTTPException(
+                        422, "endpoint_a/endpoint_b or endpoint_a_id/endpoint_b_id is required"
+                    )
+                else:
+                    # Resolve the supplied URL to a ModelEndpoint the caller owns
+                    # (their own rows + legacy null-owner shared rows), scoped so a
+                    # comparison can't borrow another user's private endpoint key.
+                    base = normalize_base(endpoint)
+                    ep = _owned_endpoint_by_url(db, base, user)
+                # Reject *unregistered* raw URLs for signed-in non-admins; a
+                # matched registered endpoint supplies an id so the caller can
+                # still compare endpoints they own. Blanket-rejecting here (the
+                # earlier `endpoint_id=None` call) locked non-admins out of
+                # compare entirely, since compare resolves endpoints by URL with
+                # no endpoint_id. Mirrors the gallery inpaint/harmonize checks.
+                # Raised here (phase 1), before any session exists.
+                _reject_raw_endpoint_url_for_non_admin(
+                    request, user, str(ep.id) if ep is not None else None, endpoint
+                )
+                # Bind the [CMP] session to the RESOLVED endpoint, not the raw
+                # caller-supplied string. When the URL matches a registered
+                # endpoint visible to the caller, use that row's own normalized
+                # base URL (the same value owner scoping + endpoint validation
+                # already vetted) so the session dials exactly where the stored
+                # config points. The raw `endpoint` only survives for callers
+                # allowed to pass one — admins / single-user mode, where
+                # `_reject_raw_endpoint_url_for_non_admin` is a no-op and `ep`
+                # is None. Mirrors the registered-endpoint path in session_routes.
+                session_endpoint_url = (
+                    build_chat_url(normalize_base(ep.base_url)) if ep is not None else endpoint
+                )
+                # Headers come only from a matched endpoint's key; None when
+                # `ep` is None (raw admin URL or no match), so a comparison can
+                # never inherit another user's key/headers.
+                headers = build_headers(ep.api_key, ep.base_url) if (ep and ep.api_key) else None
+                resolved.append((sid, model, session_endpoint_url, headers))
+        finally:
+            db.close()
+
+        # Both endpoints validated — only now create the ephemeral [CMP]
+        # sessions and copy any resolved headers.
+        for sid, model, session_endpoint_url, headers in resolved:
+            name = f"[CMP] {slot_name[sid]}" if blind else f"[CMP] {model.split('/')[-1]}"
+            session_manager.create_session(
+                session_id=sid,
+                name=name,
+                endpoint_url=session_endpoint_url,
+                model=model,
+                rag=False,
+                owner=user,
+            )
+            if headers:
+                s = session_manager.sessions.get(sid)
+                if s:
+                    s.headers = headers
+
+        # Store comparison record
+        db = SessionLocal()
+        try:
+            comp = Comparison(
+                id=comp_id,
+                prompt=prompt,
+                model_a=model_a,
+                model_b=model_b,
+                # Record the URL the session actually dials. For URL callers this
+                # is their raw input; for id-only callers (empty endpoint_a/_b)
+                # fall back to the resolved endpoint URL so the column stays
+                # meaningful and non-null. resolved is in [a, b] order.
+                endpoint_a=endpoint_a or resolved[0][2],
+                endpoint_b=endpoint_b or resolved[1][2],
+                is_blind=blind,
+                blind_mapping=json.dumps(mapping),
+                owner=user,
+            )
+            db.add(comp)
+            db.commit()
+        finally:
+            db.close()
+
+        # In blind mode, withhold the model identities AND the left/right
+        # mapping from the response. The client already knows model_a/model_b
+        # (it sent them), so returning either would defeat blind mode. They are
+        # revealed by POST /api/compare/{id}/vote once the user has voted (#1285).
+        return {
+            "id": comp_id,
+            "session_left": session_left,
+            "session_right": session_right,
+            "model_left": None if blind else (model_a if mapping["left"] == "a" else model_b),
+            "model_right": None if blind else (model_a if mapping["right"] == "a" else model_b),
+            "is_blind": blind,
+            "mapping": None if blind else mapping,
+        }
+
+    @router.post("/{comp_id}/vote")
+    def vote_comparison(
+        request: Request,
+        comp_id: str,
+        winner: str = Form(...),  # "left", "right", or "tie"
+    ):
+        """Record the user's vote and reveal model names if blind."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
+            if not comp:
+                raise HTTPException(404, "Comparison not found")
+            # SECURITY: strict ownership — null-owner Comparisons were
+            # accessible to every user.
+            if user and comp.owner != user:
+                raise HTTPException(404, "Comparison not found")
+            if comp.winner:
+                raise HTTPException(400, "Already voted")
+
+            mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"}
+
+            if winner == "tie":
+                comp.winner = "tie"
+            elif winner == "left":
+                comp.winner = mapping["left"]
+            elif winner == "right":
+                comp.winner = mapping["right"]
+            else:
+                raise HTTPException(400, "winner must be 'left', 'right', or 'tie'")
+
+            comp.voted_at = datetime.utcnow()
+            db.commit()
+
+            return {
+                "winner": comp.winner,
+                "model_a": comp.model_a,
+                "model_b": comp.model_b,
+                "revealed": {
+                    "left": comp.model_a if mapping["left"] == "a" else comp.model_b,
+                    "right": comp.model_a if mapping["right"] == "a" else comp.model_b,
+                },
+            }
+        finally:
+            db.close()
+
+    @router.post("/record")
+    def record_comparison(request: Request, body: RecordVoteRequest):
+        """Lightweight endpoint to record a comparison vote from the frontend."""
+        user = get_current_user(request)
+        comp_id = str(uuid.uuid4())
+
+        model_a = body.models[0] if len(body.models) > 0 else ""
+        model_b = body.models[1] if len(body.models) > 1 else ""
+
+        # For N>2 models, store the full list as JSON in blind_mapping
+        if len(body.models) > 2:
+            blind_mapping = json.dumps({"models": body.models})
+        else:
+            blind_mapping = None
+
+        db = SessionLocal()
+        try:
+            comp = Comparison(
+                id=comp_id,
+                prompt=body.prompt[:500],
+                model_a=model_a,
+                model_b=model_b,
+                endpoint_a="",
+                endpoint_b="",
+                winner=body.winner,
+                is_blind=body.is_blind,
+                blind_mapping=blind_mapping,
+                voted_at=datetime.utcnow(),
+                owner=user,
+            )
+            db.add(comp)
+            db.commit()
+        finally:
+            db.close()
+
+        return {"status": "ok", "id": comp_id}
+
+    @router.get("/history")
+    def list_comparisons(request: Request):
+        """List past comparisons."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            q = db.query(Comparison)
+            if user:
+                q = q.filter(Comparison.owner == user)
+            comps = q.order_by(Comparison.created_at.desc()).limit(50).all()
+            return [
+                {
+                    "id": c.id,
+                    "prompt": c.prompt[:100],
+                    "model_a": c.model_a,
+                    "model_b": c.model_b,
+                    "winner": c.winner,
+                    "is_blind": c.is_blind,
+                    "voted_at": c.voted_at.isoformat() if c.voted_at else None,
+                    "created_at": c.created_at.isoformat() if c.created_at else None,
+                }
+                for c in comps
+            ]
+        finally:
+            db.close()
+
+    @router.delete("/{comp_id}")
+    def delete_comparison(request: Request, comp_id: str):
+        """Delete a comparison and its ephemeral sessions."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
+            if not comp:
+                raise HTTPException(404, "Comparison not found")
+            # SECURITY: strict ownership — null-owner Comparisons were
+            # accessible to every user.
+            if user and comp.owner != user:
+                raise HTTPException(404, "Comparison not found")
+            db.delete(comp)
+            db.commit()
+            return {"status": "deleted"}
+        finally:
+            db.close()
+
+    return router

routes/compare_routes.py

@@ -1,246 +1,18 @@
-# routes/compare_routes.py
-"""Model A/B comparison routes."""
-import json
-import uuid
-import random
-from datetime import datetime
-from fastapi import APIRouter, Form, HTTPException, Request
-from typing import List
-from pydantic import BaseModel
-import logging
+"""Backward-compat shim — canonical location is routes/compare/compare_routes.py.
 
-from core.database import Comparison, SessionLocal
-from core.session_manager import SessionManager
-from src.auth_helpers import get_current_user
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.compare_routes``, ``from routes.compare_routes import X``,
+``importlib.import_module("routes.compare_routes")``, and the
+``import ... as cr`` + ``monkeypatch.setattr(cr, "SessionLocal", ...)`` /
+``"_owned_endpoint_by_url"`` / ``"_owned_endpoint_by_id"`` pattern used by
+test_endpoint_owner_scope_followup.py all operate on the *same* object the
+application actually uses. Keeps existing import paths working after
+slice 2i (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
-logger = logging.getLogger(__name__)
+import sys as _sys
 
-router = APIRouter(prefix="/api/compare", tags=["compare"])
+from routes.compare import compare_routes as _canonical  # noqa: F401
 
-
-class RecordVoteRequest(BaseModel):
-    prompt: str
-    models: List[str]
-    winner: str           # model name or "tie"
-    is_blind: bool = True
-
-
-def setup_compare_routes(session_manager: SessionManager):
-    """Setup comparison routes."""
-
-    @router.post("/start")
-    def start_comparison(
-        request: Request,
-        prompt: str = Form(...),
-        model_a: str = Form(...),
-        model_b: str = Form(...),
-        endpoint_a: str = Form(...),
-        endpoint_b: str = Form(...),
-        is_blind: str = Form("true"),
-    ):
-        """Create two ephemeral sessions and a comparison record.
-
-        Returns the comparison ID and the two session IDs so the client
-        can fire two independent SSE streams to /api/chat_stream.
-        """
-        comp_id = str(uuid.uuid4())
-        sid_a = str(uuid.uuid4())
-        sid_b = str(uuid.uuid4())
-
-        # Create ephemeral sessions (prefixed [CMP])
-        for sid, model, endpoint in [(sid_a, model_a, endpoint_a), (sid_b, model_b, endpoint_b)]:
-            user = getattr(request.state, 'current_user', None)
-            session_manager.create_session(
-                session_id=sid,
-                name=f"[CMP] {model.split('/')[-1]}",
-                endpoint_url=endpoint,
-                model=model,
-                rag=False,
-                owner=user,
-            )
-            # Copy API key from endpoint config
-            db = SessionLocal()
-            try:
-                from core.database import ModelEndpoint
-                # Find matching endpoint by URL
-                ep = db.query(ModelEndpoint).filter(
-                    ModelEndpoint.base_url == endpoint.replace('/chat/completions', '')
-                ).first()
-                if ep and ep.api_key:
-                    s = session_manager.sessions.get(sid)
-                    if s:
-                        s.headers = {"Authorization": f"Bearer {ep.api_key}"}
-            finally:
-                db.close()
-
-        # Blind mapping: randomly assign left/right
-        blind = str(is_blind).lower() == "true"
-        if blind:
-            mapping = {"left": "a", "right": "b"}
-            if random.random() > 0.5:
-                mapping = {"left": "b", "right": "a"}
-        else:
-            mapping = {"left": "a", "right": "b"}
-
-        # Store comparison record
-        db = SessionLocal()
-        try:
-            comp = Comparison(
-                id=comp_id,
-                prompt=prompt,
-                model_a=model_a,
-                model_b=model_b,
-                endpoint_a=endpoint_a,
-                endpoint_b=endpoint_b,
-                is_blind=blind,
-                blind_mapping=json.dumps(mapping),
-                owner=user,
-            )
-            db.add(comp)
-            db.commit()
-        finally:
-            db.close()
-
-        # Map session IDs to left/right based on blind mapping
-        session_left = sid_a if mapping["left"] == "a" else sid_b
-        session_right = sid_a if mapping["right"] == "a" else sid_b
-
-        return {
-            "id": comp_id,
-            "session_left": session_left,
-            "session_right": session_right,
-            "model_left": model_a if mapping["left"] == "a" else model_b,
-            "model_right": model_a if mapping["right"] == "a" else model_b,
-            "is_blind": blind,
-            "mapping": mapping,
-        }
-
-    @router.post("/{comp_id}/vote")
-    def vote_comparison(
-        request: Request,
-        comp_id: str,
-        winner: str = Form(...),  # "left", "right", or "tie"
-    ):
-        """Record the user's vote and reveal model names if blind."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
-            if not comp:
-                raise HTTPException(404, "Comparison not found")
-            # SECURITY: strict ownership — null-owner Comparisons were
-            # accessible to every user.
-            if user and comp.owner != user:
-                raise HTTPException(404, "Comparison not found")
-            if comp.winner:
-                raise HTTPException(400, "Already voted")
-
-            mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"}
-
-            if winner == "tie":
-                comp.winner = "tie"
-            elif winner == "left":
-                comp.winner = mapping["left"]
-            elif winner == "right":
-                comp.winner = mapping["right"]
-            else:
-                raise HTTPException(400, "winner must be 'left', 'right', or 'tie'")
-
-            comp.voted_at = datetime.utcnow()
-            db.commit()
-
-            return {
-                "winner": comp.winner,
-                "model_a": comp.model_a,
-                "model_b": comp.model_b,
-                "revealed": {
-                    "left": comp.model_a if mapping["left"] == "a" else comp.model_b,
-                    "right": comp.model_a if mapping["right"] == "a" else comp.model_b,
-                },
-            }
-        finally:
-            db.close()
-
-    @router.post("/record")
-    def record_comparison(request: Request, body: RecordVoteRequest):
-        """Lightweight endpoint to record a comparison vote from the frontend."""
-        user = get_current_user(request)
-        comp_id = str(uuid.uuid4())
-
-        model_a = body.models[0] if len(body.models) > 0 else ""
-        model_b = body.models[1] if len(body.models) > 1 else ""
-
-        # For N>2 models, store the full list as JSON in blind_mapping
-        if len(body.models) > 2:
-            blind_mapping = json.dumps({"models": body.models})
-        else:
-            blind_mapping = None
-
-        db = SessionLocal()
-        try:
-            comp = Comparison(
-                id=comp_id,
-                prompt=body.prompt[:500],
-                model_a=model_a,
-                model_b=model_b,
-                endpoint_a="",
-                endpoint_b="",
-                winner=body.winner,
-                is_blind=body.is_blind,
-                blind_mapping=blind_mapping,
-                voted_at=datetime.utcnow(),
-                owner=user,
-            )
-            db.add(comp)
-            db.commit()
-        finally:
-            db.close()
-
-        return {"status": "ok", "id": comp_id}
-
-    @router.get("/history")
-    def list_comparisons(request: Request):
-        """List past comparisons."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            q = db.query(Comparison)
-            if user:
-                q = q.filter(Comparison.owner == user)
-            comps = q.order_by(Comparison.created_at.desc()).limit(50).all()
-            return [
-                {
-                    "id": c.id,
-                    "prompt": c.prompt[:100],
-                    "model_a": c.model_a,
-                    "model_b": c.model_b,
-                    "winner": c.winner,
-                    "is_blind": c.is_blind,
-                    "voted_at": c.voted_at.isoformat() if c.voted_at else None,
-                    "created_at": c.created_at.isoformat() if c.created_at else None,
-                }
-                for c in comps
-            ]
-        finally:
-            db.close()
-
-    @router.delete("/{comp_id}")
-    def delete_comparison(request: Request, comp_id: str):
-        """Delete a comparison and its ephemeral sessions."""
-        user = get_current_user(request)
-        db = SessionLocal()
-        try:
-            comp = db.query(Comparison).filter(Comparison.id == comp_id).first()
-            if not comp:
-                raise HTTPException(404, "Comparison not found")
-            # SECURITY: strict ownership — null-owner Comparisons were
-            # accessible to every user.
-            if user and comp.owner != user:
-                raise HTTPException(404, "Comparison not found")
-            db.delete(comp)
-            db.commit()
-            return {"status": "deleted"}
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/contacts/__init__.py

@@ -0,0 +1,5 @@
+"""Contacts route domain package (slice 2e, #4082/#4071).
+
+Contains contacts_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/contacts_routes.py re-exports from here.
+"""

routes/contacts_routes.py

@@ -1,783 +1,13 @@
-"""
-contacts_routes.py
+"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
 
-CardDAV contacts integration. Reads from local Radicale, supports
-search and adding new contacts.
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
+``importlib.import_module("routes.contacts_routes")``, and string-targeted
+monkeypatches all operate on the same object the application actually uses.
 """
 
-import re
-import logging
-import uuid
-import json
-import csv
-import io
-import httpx
-from pathlib import Path
-from datetime import datetime
-from fastapi import APIRouter, Query, Depends, Response
-from typing import List, Dict, Optional
+import sys as _sys
 
-from src.auth_helpers import require_user
-from core.middleware import require_admin
+from routes.contacts import contacts_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-DATA_DIR = Path(__file__).resolve().parent.parent / "data"
-SETTINGS_FILE = DATA_DIR / "settings.json"
-LOCAL_CONTACTS_FILE = DATA_DIR / "contacts.json"
-
-
-def _load_settings():
-    if SETTINGS_FILE.exists():
-        return json.loads(SETTINGS_FILE.read_text())
-    return {}
-
-
-def _save_settings(settings):
-    from core.atomic_io import atomic_write_json
-    atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
-
-
-def _get_carddav_config():
-    import os
-    settings = _load_settings()
-    return {
-        "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
-        "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
-        "password": settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")),
-    }
-
-
-def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
-    cfg = cfg or _get_carddav_config()
-    return bool((cfg.get("url") or "").strip())
-
-
-def _normalize_contact(contact: Dict) -> Dict:
-    emails = []
-    for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
-        e = str(e or "").strip()
-        if e and e not in emails:
-            emails.append(e)
-    phones = []
-    for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
-        p = str(p or "").strip()
-        if p and p not in phones:
-            phones.append(p)
-    name = str(contact.get("name") or "").strip()
-    if not name and emails:
-        name = emails[0].split("@")[0]
-    return {
-        "uid": str(contact.get("uid") or uuid.uuid4()),
-        "name": name,
-        "emails": emails,
-        "phones": phones,
-    }
-
-
-def _load_local_contacts() -> List[Dict]:
-    try:
-        if not LOCAL_CONTACTS_FILE.exists():
-            return []
-        data = json.loads(LOCAL_CONTACTS_FILE.read_text())
-        rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
-    except Exception as e:
-        logger.error(f"Failed to load local contacts: {e}")
-        return []
-
-
-def _save_local_contacts(contacts: List[Dict]) -> None:
-    from core.atomic_io import atomic_write_json
-    DATA_DIR.mkdir(parents=True, exist_ok=True)
-    atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
-    _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
-    _contact_cache["fetched_at"] = datetime.utcnow()
-
-
-# ── vCard parsing ──
-
-def _vunesc(value: str) -> str:
-    """Reverse _vesc() — turn escaped vCard text back into the raw value.
-    Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
-    if not value:
-        return value
-    out = []
-    i = 0
-    while i < len(value):
-        ch = value[i]
-        if ch == "\\" and i + 1 < len(value):
-            nxt = value[i + 1]
-            if nxt in ("n", "N"):
-                out.append("\n")
-            elif nxt in (",", ";", "\\"):
-                out.append(nxt)
-            else:
-                out.append(nxt)
-            i += 2
-        else:
-            out.append(ch)
-            i += 1
-    return "".join(out)
-
-
-def _parse_vcards(text: str) -> List[Dict]:
-    """Parse a stream of vCards into dicts with name, email, phone."""
-    contacts = []
-    for block in re.split(r"BEGIN:VCARD", text):
-        if not block.strip():
-            continue
-        contact = {"name": "", "emails": [], "phones": [], "uid": ""}
-        for line in block.split("\n"):
-            line = line.strip()
-            if line.startswith("FN:") or line.startswith("FN;"):
-                contact["name"] = _vunesc(line.split(":", 1)[1]) if ":" in line else ""
-            elif line.startswith("EMAIL"):
-                # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
-                if ":" in line:
-                    email_addr = _vunesc(line.split(":", 1)[1])
-                    if email_addr and email_addr not in contact["emails"]:
-                        contact["emails"].append(email_addr)
-            elif line.startswith("TEL"):
-                if ":" in line:
-                    phone = _vunesc(line.split(":", 1)[1])
-                    if phone and phone not in contact["phones"]:
-                        contact["phones"].append(phone)
-            elif line.startswith("UID:"):
-                contact["uid"] = _vunesc(line[4:])
-        if contact["name"] or contact["emails"]:
-            contacts.append(contact)
-    return contacts
-
-
-def _vesc(value: str) -> str:
-    """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
-    semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
-    or any value containing a newline produces a malformed vCard (broken
-    N/FN fields) or could inject arbitrary properties."""
-    return (
-        (value or "")
-        .replace("\\", "\\\\")
-        .replace("\n", "\\n")
-        .replace("\r", "")
-        .replace(",", "\\,")
-        .replace(";", "\\;")
-    )
-
-
-def _build_vcard(name: str, email: str, uid: Optional[str] = None,
-                 emails: Optional[List[str]] = None,
-                 phones: Optional[List[str]] = None) -> str:
-    """Build a vCard. Accepts either a single `email` (legacy callers) or
-    full `emails`/`phones` lists (edit path). The first email is marked
-    PREF=1. All values are RFC-6350-escaped."""
-    if not uid:
-        uid = str(uuid.uuid4())
-    # Normalize email lists — `email` arg is a convenience for single-email
-    # creation; `emails` (if given) is authoritative.
-    email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
-    phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
-    # Try to split name into first/last
-    parts = name.strip().split()
-    if len(parts) >= 2:
-        first = parts[0]
-        last = " ".join(parts[1:])
-    else:
-        first = name
-        last = ""
-    # N field is structured (5 components separated by ';') — escape each
-    # component individually so a comma in the name doesn't split it.
-    n_field = f"{_vesc(last)};{_vesc(first)};;;"
-    lines = [
-        "BEGIN:VCARD",
-        "VERSION:4.0",
-        f"UID:{_vesc(uid)}",
-        f"FN:{_vesc(name)}",
-        f"N:{n_field}",
-    ]
-    for i, em in enumerate(email_list):
-        # First email is the preferred one.
-        lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
-    for ph in phone_list:
-        lines.append(f"TEL:{_vesc(ph)}")
-    lines.append("END:VCARD")
-    return "\r\n".join(lines) + "\r\n"
-
-
-# ── In-memory cache ──
-
-_contact_cache = {"contacts": [], "fetched_at": None}
-
-
-def _abs_url(href: str) -> str:
-    """Combine a multistatus <href> (an absolute path like
-    /user/contacts/x.vcf) with the configured CardDAV server origin so we
-    get a fully-qualified URL to PUT/DELETE. If href is already absolute
-    (http...), return it as-is."""
-    from urllib.parse import urlparse, urlunparse
-    if href.startswith("http://") or href.startswith("https://"):
-        return href
-    cfg = _get_carddav_config()
-    p = urlparse(cfg["url"])
-    return urlunparse((p.scheme, p.netloc, href, "", "", ""))
-
-
-# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
-# alongside the resource href. Lets us map each contact's UID to the real
-# server resource path (which is NOT always <uid>.vcf for contacts created
-# by other clients).
-_ADDRESSBOOK_QUERY = (
-    '<?xml version="1.0" encoding="utf-8"?>'
-    '<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
-    '<D:prop><D:getetag/><C:address-data/></D:prop>'
-    '<C:filter/>'
-    '</C:addressbook-query>'
-)
-
-
-def _fetch_via_report(cfg, auth):
-    """Try a CardDAV REPORT addressbook-query — returns contacts WITH an
-    `href` field, or None if the server doesn't support it / errors."""
-    from defusedxml import ElementTree as ET
-    try:
-        r = httpx.request(
-            "REPORT", cfg["url"],
-            content=_ADDRESSBOOK_QUERY.encode("utf-8"),
-            headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
-        )
-        if r.status_code not in (207, 200):
-            return None
-        root = ET.fromstring(r.text)
-        ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
-        out = []
-        for resp in root.findall("D:response", ns):
-            href_el = resp.find("D:href", ns)
-            data_el = resp.find(".//C:address-data", ns)
-            if href_el is None or data_el is None or not (data_el.text or "").strip():
-                continue
-            parsed = _parse_vcards(data_el.text)
-            if not parsed:
-                continue
-            c = parsed[0]
-            c["href"] = href_el.text.strip()
-            out.append(c)
-        # If the REPORT parsed to ZERO contacts, don't trust it — some
-        # CardDAV servers treat an empty <filter/> as "match nothing" and
-        # return a valid-but-empty 207. Return None so the caller falls
-        # back to the plain GET (which lists everything). A genuinely empty
-        # address book just costs one extra GET that also returns nothing.
-        if not out:
-            return None
-        return out
-    except Exception as e:
-        logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
-        return None
-
-
-def _fetch_contacts(force=False):
-    """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
-        age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
-            return _contact_cache["contacts"]
-
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-
-    try:
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        # Preferred path: REPORT gives us hrefs for reliable edit/delete.
-        contacts = _fetch_via_report(cfg, auth)
-        if contacts is None:
-            # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
-            if r.status_code != 200:
-                logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
-            contacts = _parse_vcards(r.text)
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
-        return contacts
-    except Exception as e:
-        logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
-
-
-def _resolve_resource_url(uid: str) -> str:
-    """Map a contact UID to its real CardDAV resource URL. Uses the href
-    captured during fetch when available (handles contacts whose filename
-    != UID); falls back to the <uid>.vcf guess for app-created contacts or
-    when no href is known."""
-    def _lookup():
-        for c in _contact_cache.get("contacts", []):
-            if c.get("uid") == uid and c.get("href"):
-                return _abs_url(c["href"])
-        return None
-    found = _lookup()
-    if found:
-        return found
-    # Not in cache (or no href) — refresh once and retry before guessing.
-    try:
-        _fetch_contacts(force=True)
-    except Exception:
-        pass
-    return _lookup() or _vcard_url(uid)
-
-
-def _create_contact(name: str, email: str) -> bool:
-    """Add a new contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        email_l = (email or "").strip().lower()
-        for c in contacts:
-            if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
-                return True
-        contacts.append(_normalize_contact({"name": name, "emails": [email]}))
-        _save_local_contacts(contacts)
-        return True
-
-    contact_uid = str(uuid.uuid4())
-    vcard = _build_vcard(name, email, contact_uid)
-    url = cfg["url"].rstrip("/") + "/" + contact_uid + ".vcf"
-    try:
-        auth = None
-        if cfg["username"]:
-            auth = (cfg["username"], cfg["password"])
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            # Invalidate cache
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to create contact: {e}")
-        return False
-
-
-def _vcard_url(uid: str) -> str:
-    """The CardDAV resource URL for a given contact UID. The uid is URL-
-    encoded so a value containing '/', '..' or other path chars can't
-    escape the collection and target an arbitrary CardDAV resource."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    return cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf"
-
-
-def _import_vcards(text: str) -> Dict:
-    """Import a (possibly multi-card) .vcf blob. Each card is PUT to the
-    CardDAV server PRESERVING its full original content (ADR/ORG/photo/
-    etc.) — we don't rebuild it, just ensure it has VERSION + UID and
-    normalize line endings. Returns {imported, failed, total}."""
-    from urllib.parse import quote
-    cfg = _get_carddav_config()
-    if not cfg.get("url"):
-        parsed = _parse_vcards(text)
-        contacts = _load_local_contacts()
-        existing = {
-            e.lower()
-            for c in contacts
-            for e in (c.get("emails") or [])
-            if e
-        }
-        imported = 0
-        for c in parsed:
-            emails = [e for e in (c.get("emails") or []) if e]
-            if emails and any(e.lower() in existing for e in emails):
-                continue
-            contacts.append(_normalize_contact(c))
-            for e in emails:
-                existing.add(e.lower())
-            imported += 1
-        if imported:
-            _save_local_contacts(contacts)
-        return {"imported": imported, "failed": 0, "total": len(parsed)}
-    auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-    # Split into individual cards. re.split drops the BEGIN line, so we
-    # re-add it. Normalize CRLF.
-    raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
-    blocks = []
-    for chunk in raw.split("BEGIN:VCARD"):
-        chunk = chunk.strip()
-        if not chunk:
-            continue
-        # Trim anything after END:VCARD (defensive).
-        end = chunk.upper().find("END:VCARD")
-        body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
-        blocks.append("BEGIN:VCARD\n" + body)
-    imported = 0
-    failed = 0
-    for block in blocks:
-        # Extract or assign a UID.
-        m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
-        uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
-        if not m:
-            # Inject a UID right after the VERSION line (or after BEGIN).
-            if re.search(r"^VERSION:", block, re.MULTILINE):
-                block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
-            else:
-                block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
-        elif not re.search(r"^VERSION:", block, re.MULTILINE):
-            block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
-        vcard = block.replace("\n", "\r\n") + "\r\n"
-        url = cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf"
-        try:
-            r = httpx.put(
-                url, data=vcard.encode("utf-8"),
-                headers={"Content-Type": "text/vcard; charset=utf-8"},
-                auth=auth, timeout=15,
-            )
-            if r.status_code in (200, 201, 204):
-                imported += 1
-            else:
-                failed += 1
-                logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
-        except Exception as e:
-            failed += 1
-            logger.error(f"Import PUT {uid} failed: {e}")
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": len(blocks)}
-
-
-def _import_csv_contacts(text: str) -> Dict:
-    """Import contacts from CSV. Supports common headers:
-    name/full_name/display_name, email/email_address/e-mail, phone/tel.
-    Falls back to first columns as name,email,phone when no headers exist."""
-    raw = (text or "").strip()
-    if not raw:
-        return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
-
-    try:
-        sample = raw[:2048]
-        dialect = csv.Sniffer().sniff(sample)
-    except Exception:
-        dialect = csv.excel
-
-    stream = io.StringIO(raw)
-    try:
-        has_header = csv.Sniffer().has_header(raw[:2048])
-    except Exception:
-        has_header = True
-
-    rows = []
-    if has_header:
-        reader = csv.DictReader(stream, dialect=dialect)
-        for row in reader:
-            lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
-            name = (
-                lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
-                or lowered.get("display name") or lowered.get("display_name")
-                or lowered.get("fn") or ""
-            )
-            email = (
-                lowered.get("email") or lowered.get("email address")
-                or lowered.get("email_address") or lowered.get("e-mail")
-                or lowered.get("mail") or ""
-            )
-            phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
-            rows.append((name, email, phone))
-    else:
-        stream.seek(0)
-        reader = csv.reader(stream, dialect=dialect)
-        for row in reader:
-            cols = [(c or "").strip() for c in row]
-            if not any(cols):
-                continue
-            rows.append((
-                cols[0] if len(cols) > 0 else "",
-                cols[1] if len(cols) > 1 else "",
-                cols[2] if len(cols) > 2 else "",
-            ))
-
-    imported = 0
-    failed = 0
-    total = 0
-    existing_emails = {
-        e.lower()
-        for c in _fetch_contacts()
-        for e in (c.get("emails") or [])
-        if e
-    }
-    for name, email, phone in rows:
-        email = (email or "").strip()
-        name = (name or "").strip() or (email.split("@")[0] if email else "")
-        if not email:
-            continue
-        total += 1
-        if email.lower() in existing_emails:
-            continue
-        ok = _create_contact(name, email)
-        if ok:
-            imported += 1
-            existing_emails.add(email.lower())
-            # If the CSV had a phone number, rewrite the just-created row
-            # through the richer update path so phone lands in CardDAV too.
-            if phone:
-                try:
-                    contacts = _fetch_contacts(force=True)
-                    created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
-                    if created and created.get("uid"):
-                        _update_contact(created["uid"], name, [email], [phone])
-                except Exception:
-                    pass
-        else:
-            failed += 1
-
-    if imported:
-        _contact_cache["fetched_at"] = None
-    return {"imported": imported, "failed": failed, "total": total}
-
-
-def _contacts_to_vcf(contacts: List[Dict]) -> str:
-    return "".join(
-        _build_vcard(
-            c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
-            "",
-            uid=c.get("uid") or str(uuid.uuid4()),
-            emails=c.get("emails") or [],
-            phones=c.get("phones") or [],
-        )
-        for c in contacts
-    )
-
-
-def _contacts_to_csv(contacts: List[Dict]) -> str:
-    out = io.StringIO()
-    writer = csv.writer(out)
-    writer.writerow(["name", "email", "phone"])
-    for c in contacts:
-        emails = c.get("emails") or [""]
-        phones = c.get("phones") or [""]
-        max_len = max(len(emails), len(phones), 1)
-        for i in range(max_len):
-            writer.writerow([
-                c.get("name") or "",
-                emails[i] if i < len(emails) else "",
-                phones[i] if i < len(phones) else "",
-            ])
-    return out.getvalue()
-
-
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -> bool:
-    """Rewrite an existing contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        found = False
-        out = []
-        for c in contacts:
-            if c.get("uid") == uid:
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
-                found = True
-            else:
-                out.append(c)
-        if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
-        _save_local_contacts(out)
-        return True
-
-    vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones)
-    # Use the real resource href (handles externally-created contacts whose
-    # filename != UID); falls back to the <uid>.vcf guess.
-    url = _resolve_resource_url(uid)
-    try:
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.put(
-            url,
-            data=vcard.encode("utf-8"),
-            headers={"Content-Type": "text/vcard; charset=utf-8"},
-            auth=auth,
-            timeout=10,
-        )
-        if r.status_code in (200, 201, 204):
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to update contact: {e}")
-        return False
-
-
-def _delete_contact(uid: str) -> bool:
-    """Delete a contact via CardDAV or local contacts."""
-    cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
-        _save_local_contacts(remaining)
-        return True
-
-    url = _resolve_resource_url(uid)
-    try:
-        auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
-        r = httpx.delete(url, auth=auth, timeout=10)
-        if r.status_code in (200, 204):
-            _contact_cache["fetched_at"] = None
-            return True
-        if r.status_code == 404:
-            # Resource not found at the resolved URL. With href resolution
-            # this should be rare (genuinely already deleted). Invalidate
-            # the cache and report success so the UI doesn't keep a ghost.
-            logger.info(f"CardDAV DELETE 404 for {uid} — treating as already gone")
-            _contact_cache["fetched_at"] = None
-            return True
-        logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
-        return False
-    except Exception as e:
-        logger.error(f"Failed to delete contact: {e}")
-        return False
-
-
-# ── Routes ──
-
-def setup_contacts_routes():
-    router = APIRouter(prefix="/api/contacts", tags=["contacts"])
-
-    @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
-        """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
-
-    @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
-        """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
-        if not q:
-            return {"results": []}
-        q_lower = q.lower()
-        results = []
-        for c in contacts:
-            if q_lower in c["name"].lower():
-                results.append(c)
-                continue
-            for em in c["emails"]:
-                if q_lower in em.lower():
-                    results.append(c)
-                    break
-        return {"results": results[:10]}
-
-    @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
-        """Add a new contact."""
-        name = data.get("name", "").strip()
-        email = data.get("email", "").strip()
-        if not email:
-            return {"success": False, "error": "Email required"}
-        # Check if already exists
-        contacts = _fetch_contacts()
-        for c in contacts:
-            if email.lower() in [e.lower() for e in c["emails"]]:
-                return {"success": True, "message": "Already exists", "contact": c}
-        if not name:
-            name = email.split("@")[0]
-        ok = _create_contact(name, email)
-        return {"success": ok}
-
-    @router.post("/import")
-    async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
-        """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
-        text = data.get("vcf") or data.get("text") or ""
-        csv_text = data.get("csv") or ""
-        if text.strip():
-            if "BEGIN:VCARD" not in text.upper():
-                return {"success": False, "error": "No vCard data found"}
-            result = _import_vcards(text)
-        elif csv_text.strip():
-            result = _import_csv_contacts(csv_text)
-        else:
-            return {"success": False, "error": "No contact data found"}
-        result["success"] = result.get("imported", 0) > 0
-        return result
-
-    @router.get("/export")
-    async def export_contacts(
-        format: str = Query("vcf", pattern="^(vcf|csv)$"),
-        _admin: str = Depends(require_admin),
-    ):
-        """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
-        if format == "csv":
-            content = _contacts_to_csv(contacts)
-            media_type = "text/csv; charset=utf-8"
-            filename = "odysseus-contacts.csv"
-        else:
-            content = _contacts_to_vcf(contacts)
-            media_type = "text/vcard; charset=utf-8"
-            filename = "odysseus-contacts.vcf"
-        return Response(
-            content=content,
-            media_type=media_type,
-            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
-        )
-
-    @router.get("/config")
-    async def get_config(_admin: str = Depends(require_admin)):
-        cfg = _get_carddav_config()
-        # Mask password
-        if cfg["password"]:
-            cfg["password"] = "***"
-        return cfg
-
-    @router.put("/config")
-    async def update_config(data: dict, _admin: str = Depends(require_admin)):
-        settings = _load_settings()
-        for key in ("carddav_url", "carddav_username", "carddav_password"):
-            if key in data:
-                settings[key] = data[key]
-        _save_settings(settings)
-        # Force re-fetch
-        _contact_cache["fetched_at"] = None
-        return {"success": True}
-
-    @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
-        """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
-        return {"success": True}
-
-    # NOTE: the /{uid} routes are declared LAST so the literal paths above
-    # (/list, /search, /add, /config) win — otherwise PUT /config would
-    # match PUT /{uid} with uid="config".
-    @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
-        """Edit an existing contact — name / emails / phones."""
-        name = (data.get("name") or "").strip()
-        emails = data.get("emails")
-        phones = data.get("phones")
-        if emails is None and data.get("email"):
-            emails = [data["email"]]
-        emails = [e.strip() for e in (emails or []) if e and e.strip()]
-        phones = [p.strip() for p in (phones or []) if p and p.strip()]
-        if not name and not emails:
-            return {"success": False, "error": "Name or email required"}
-        if not name and emails:
-            name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones)
-        return {"success": ok}
-
-    @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
-        """Delete a contact by UID."""
-        if not uid:
-            return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
-        return {"success": ok}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/cookbook_output.py

@@ -0,0 +1,75 @@
+"""Pure helpers for shaping cookbook task output for the status response.
+
+Kept dependency-free (no FastAPI / SQLAlchemy imports) so the behavior can be
+unit-tested without standing up the whole app.
+"""
+
+import re
+
+_FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE)
+
+# Probe scripts for the dead-session download check, run as
+# `python3 -c <PROBE> <repo_id> <cache_root>` (locally or over SSH).
+# cache_root is the task's custom download dir, '' for the default HF cache.
+# It has to be passed explicitly: the download runner exports
+# HF_HOME=<local_dir>, so that task's cache lives under <local_dir>/hub, and
+# the probe process's own environment knows nothing about it.
+HF_CACHE_COMPLETE_PROBE = (
+    "import os,sys;"
+    "repo=sys.argv[1];"
+    "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
+    "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
+    "d=os.path.join(base,'models--'+repo.replace('/','--'));"
+    "snap=os.path.join(d,'snapshots');"
+    "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));"
+    "inc=False;"
+    "blobs=os.path.join(d,'blobs');"
+    "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
+    "sys.exit(0 if ok and not inc else 1)"
+)
+
+HF_CACHE_INCOMPLETE_PROBE = (
+    "import os,sys;"
+    "repo=sys.argv[1];"
+    "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
+    "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
+    "d=os.path.join(base,'models--'+repo.replace('/','--'));"
+    "blobs=os.path.join(d,'blobs');"
+    "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
+    "sys.exit(0 if inc else 1)"
+)
+
+
+def classify_dead_download(full_snapshot: str):
+    """Resolve a dead download session's status from its runner markers.
+
+    The runner prints DOWNLOAD_OK only after exiting 0 (and DOWNLOAD_FAILED
+    otherwise), so the markers stay trustworthy after the tmux pane is gone.
+    Returns (status, zero_files), or None when the snapshot carries no marker
+    and the caller has to fall back to the cache probe. Same precedence as
+    the live-session branch: DOWNLOAD_OK wins, except a "Fetching 0 files"
+    run is an error (nothing matched the include/quant pattern).
+    """
+    if not full_snapshot:
+        return None
+    if "DOWNLOAD_OK" in full_snapshot:
+        if _FETCHING_ZERO_FILES_RE.search(full_snapshot):
+            return ("error", True)
+        return ("completed", False)
+    if "DOWNLOAD_FAILED" in full_snapshot:
+        return ("error", False)
+    return None
+
+
+def error_aware_output_tail(full_snapshot: str, status: str) -> str:
+    """Return the trailing slice of a task log for the status response.
+
+    Failed tasks return the last 50 lines so the "Copy last 50 lines" action
+    surfaces the actual error context (stack traces, build output). Running and
+    other non-error tasks keep the cheaper 12-line tail to limit the payload on
+    the 10s polling interval.
+    """
+    if not full_snapshot:
+        return ""
+    tail_lines = 50 if status == "error" else 12
+    return "\n".join(full_snapshot.splitlines()[-tail_lines:])

routes/copilot_routes.py

@@ -0,0 +1,173 @@
+# routes/copilot_routes.py
+"""GitHub Copilot device-flow login.
+
+Drives the GitHub OAuth *device flow* and, on success, creates (or refreshes)
+an owner-scoped ``ModelEndpoint`` pointing at the Copilot API with the
+device-flow access token stored as its (encrypted) ``api_key``. After that the
+endpoint behaves like any other OpenAI-compatible provider — the Copilot-
+specific request headers are injected centrally by ``build_headers`` /
+``_provider_headers`` (see :mod:`src.copilot`).
+
+Flow:
+  1. ``POST /api/copilot/device/start`` → returns a ``poll_id`` plus the
+     ``user_code`` + ``verification_uri`` to show the user. The secret
+     ``device_code`` is kept server-side, never sent to the browser.
+  2. The browser polls ``POST /api/copilot/device/poll`` with ``poll_id``.
+     While pending it returns ``{status: "pending"}``; once the user authorises
+     it provisions the endpoint and returns ``{status: "authorized", ...}``.
+
+All routes are admin-gated (endpoint/provider management is an admin action).
+"""
+
+import json
+import uuid
+import logging
+from typing import Dict, Optional
+
+import httpx
+from fastapi import HTTPException, Request
+
+from core.database import SessionLocal, ModelEndpoint
+from routes.device_flow import (
+    DeviceFlowPoll,
+    DeviceFlowStart,
+    PendingDeviceFlowStore,
+    create_device_flow_router,
+)
+from src.auth_helpers import get_current_user
+from src import copilot
+
+logger = logging.getLogger(__name__)
+
+_DEVICE_FLOW_STORE = PendingDeviceFlowStore()
+
+
+def _provision_endpoint(token: str, base: str, owner: Optional[str]) -> Dict:
+    """Create or update the owner's Copilot endpoint with a fresh token."""
+    try:
+        models = copilot.fetch_models(base, token)
+    except Exception as e:
+        logger.warning(f"Copilot model fetch failed during provisioning: {e}")
+        models = []
+    model_ids = [m["id"] for m in models]
+    # Copilot picker models support OpenAI-style tool calling; mark the endpoint
+    # tool-capable so the agent loop sends native tool schemas.
+    # Tool-capable if any picker model advertises tool_calls. When the model
+    # fetch failed (empty list) default to True, since Copilot picker models
+    # support OpenAI-style tool calling.
+    supports_tools = bool(not models or any(m.get("tool_calls") for m in models))
+
+    db = SessionLocal()
+    try:
+        ep = (
+            db.query(ModelEndpoint)
+            .filter(ModelEndpoint.base_url == base)
+            .filter((ModelEndpoint.owner.is_(None)) | (ModelEndpoint.owner == owner))
+            .order_by(ModelEndpoint.owner.desc())
+            .first()
+        )
+        if ep is None:
+            ep = ModelEndpoint(
+                id=str(uuid.uuid4())[:8],
+                name="GitHub Copilot",
+                base_url=base,
+                model_type="llm",
+                owner=owner,
+            )
+            db.add(ep)
+        ep.api_key = token
+        ep.is_enabled = True
+        ep.supports_tools = supports_tools
+        if model_ids:
+            ep.cached_models = json.dumps(model_ids)
+        db.commit()
+        result = {
+            "id": ep.id,
+            "name": ep.name,
+            "base_url": ep.base_url,
+            "models": model_ids,
+        }
+    finally:
+        db.close()
+
+    # Best-effort: refresh the model cache so the new endpoint shows up.
+    try:
+        from routes.model_routes import _invalidate_models_cache
+        _invalidate_models_cache()
+    except Exception:
+        pass
+    return result
+
+
+def _start_device_flow(request: Request, form) -> DeviceFlowStart:
+    host = copilot.GITHUB_HOST
+    ent = str(form.get("enterprise_url") or "").strip()
+    if ent:
+        host = copilot.normalize_domain(ent)
+    try:
+        data = copilot.request_device_code(host)
+    except httpx.HTTPStatusError as e:
+        status = e.response.status_code if e.response is not None else "unknown"
+        raise HTTPException(502, f"GitHub device-code request failed (HTTP {status})")
+    except Exception as e:
+        raise HTTPException(502, f"GitHub device-code request failed: {e}")
+
+    device_code = data.get("device_code")
+    if not device_code:
+        raise HTTPException(502, "GitHub did not return a device code")
+
+    # verification_uri_complete embeds the user code, so the browser tab we
+    # open lands the user straight on GitHub's "Authorize" screen with the
+    # code pre-filled — one click, no manual code entry.
+    return DeviceFlowStart(
+        pending={
+            "device_code": device_code,
+            "host": host,
+            "enterprise_url": ent,
+            "owner": get_current_user(request) or None,
+        },
+        response={
+            "user_code": data.get("user_code"),
+            "verification_uri": data.get("verification_uri"),
+            "verification_uri_complete": data.get("verification_uri_complete"),
+        },
+        interval=int(data.get("interval") or 5),
+        expires_in=int(data.get("expires_in") or 900),
+    )
+
+
+def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll:
+    try:
+        data = copilot.poll_access_token(pending["host"], pending["device_code"])
+    except Exception as e:
+        return DeviceFlowPoll.pending(f"poll error: {e}")
+
+    token = data.get("access_token")
+    if token:
+        base = copilot.enterprise_base(pending["enterprise_url"]) if pending["enterprise_url"] else copilot.COPILOT_BASE
+        try:
+            result = _provision_endpoint(token, base, pending["owner"])
+        except Exception as e:
+            logger.exception("Copilot endpoint provisioning failed")
+            raise HTTPException(500, f"Login succeeded but provisioning failed: {e}")
+        return DeviceFlowPoll.authorized(result)
+
+    err = data.get("error")
+    if err == "authorization_pending":
+        return DeviceFlowPoll.pending()
+    if err == "slow_down":
+        return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None)
+    if err in ("expired_token", "access_denied"):
+        return DeviceFlowPoll.failed(err)
+    # Unknown error — surface but keep the session for another try.
+    return DeviceFlowPoll.pending(err or "unknown")
+
+
+def setup_copilot_routes():
+    return create_device_flow_router(
+        prefix="/api/copilot",
+        tags=["copilot"],
+        store=_DEVICE_FLOW_STORE,
+        start_flow=_start_device_flow,
+        poll_flow=_poll_device_flow,
+    )

routes/device_flow.py

@@ -0,0 +1,193 @@
+"""Shared OAuth/device-flow route scaffolding for provider setup."""
+
+from __future__ import annotations
+
+import inspect
+import threading
+import time
+import uuid
+from dataclasses import dataclass
+from typing import Any, Callable, Iterable, Mapping, Optional
+
+from fastapi import APIRouter, Form, HTTPException, Request
+
+from core.middleware import require_admin
+
+
+@dataclass(frozen=True)
+class DeviceFlowStart:
+    """Provider-specific start result consumed by the shared route wrapper."""
+
+    pending: Mapping[str, Any]
+    response: Mapping[str, Any]
+    interval: int = 5
+    expires_in: int = 900
+
+
+@dataclass(frozen=True)
+class DeviceFlowPoll:
+    """Normalized provider poll outcome."""
+
+    status: str
+    endpoint: Optional[Mapping[str, Any]] = None
+    error: Optional[str] = None
+    detail: Optional[str] = None
+    interval: Optional[int] = None
+
+    @classmethod
+    def pending(cls, detail: Optional[str] = None) -> "DeviceFlowPoll":
+        return cls(status="pending", detail=detail)
+
+    @classmethod
+    def slow_down(cls, interval: Optional[int] = None, detail: Optional[str] = None) -> "DeviceFlowPoll":
+        return cls(status="slow_down", interval=interval, detail=detail)
+
+    @classmethod
+    def authorized(cls, endpoint: Mapping[str, Any]) -> "DeviceFlowPoll":
+        return cls(status="authorized", endpoint=endpoint)
+
+    @classmethod
+    def failed(cls, error: str) -> "DeviceFlowPoll":
+        return cls(status="failed", error=error)
+
+
+class PendingDeviceFlowStore:
+    """Thread-safe in-memory pending device-flow store.
+
+    Device codes and provider-side secrets stay inside this process. Each entry
+    stores provider payload separately from poll metadata so provider callbacks
+    only receive the fields they created.
+    """
+
+    def __init__(self, *, time_func: Callable[[], float] = time.time):
+        self._pending: dict[str, dict[str, Any]] = {}
+        self._lock = threading.Lock()
+        self._time = time_func
+
+    def _now(self) -> float:
+        return float(self._time())
+
+    def prune_expired(self) -> None:
+        now = self._now()
+        with self._lock:
+            for key in [k for k, v in self._pending.items() if v.get("expires_at", 0) < now]:
+                self._pending.pop(key, None)
+
+    def add(self, payload: Mapping[str, Any], *, interval: int, expires_in: int) -> str:
+        self.prune_expired()
+        poll_id = uuid.uuid4().hex
+        with self._lock:
+            self._pending[poll_id] = {
+                "payload": dict(payload),
+                "interval": max(int(interval or 5), 1),
+                "expires_at": self._now() + max(int(expires_in or 900), 1),
+                "next_poll_at": 0.0,
+            }
+        return poll_id
+
+    def get_payload(self, poll_id: str) -> Optional[dict[str, Any]]:
+        self.prune_expired()
+        with self._lock:
+            entry = self._pending.get(poll_id)
+            if entry is None:
+                return None
+            return dict(entry.get("payload") or {})
+
+    def is_throttled(self, poll_id: str) -> bool:
+        with self._lock:
+            entry = self._pending.get(poll_id)
+            return bool(entry and self._now() < float(entry.get("next_poll_at") or 0))
+
+    def schedule_next(self, poll_id: str) -> None:
+        now = self._now()
+        with self._lock:
+            entry = self._pending.get(poll_id)
+            if entry is not None:
+                entry["next_poll_at"] = now + int(entry.get("interval") or 5)
+
+    def slow_down(self, poll_id: str, interval: Optional[int] = None) -> None:
+        now = self._now()
+        with self._lock:
+            entry = self._pending.get(poll_id)
+            if entry is not None:
+                new_interval = int(interval or (int(entry.get("interval") or 5) + 5))
+                entry["interval"] = max(new_interval, 1)
+                entry["next_poll_at"] = now + entry["interval"]
+
+    def pop(self, poll_id: str) -> None:
+        with self._lock:
+            self._pending.pop(poll_id, None)
+
+
+async def _maybe_await(value: Any) -> Any:
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+def _pending_response(detail: Optional[str] = None) -> dict[str, Any]:
+    response: dict[str, Any] = {"status": "pending"}
+    if detail:
+        response["detail"] = detail
+    return response
+
+
+def create_device_flow_router(
+    *,
+    prefix: str,
+    tags: Iterable[str],
+    store: PendingDeviceFlowStore,
+    start_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowStart],
+    poll_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowPoll],
+) -> APIRouter:
+    """Create standard `/device/start|poll|cancel` routes for a provider."""
+
+    router = APIRouter(prefix=prefix, tags=list(tags))
+
+    @router.post("/device/start")
+    async def device_start(request: Request):
+        require_admin(request)
+        form = await request.form()
+        start = await _maybe_await(start_flow(request, form))
+        interval = int(start.interval or 5)
+        expires_in = int(start.expires_in or 900)
+        poll_id = store.add(start.pending, interval=interval, expires_in=expires_in)
+        response = dict(start.response)
+        response.update({"poll_id": poll_id, "interval": interval, "expires_in": expires_in})
+        return response
+
+    @router.post("/device/poll")
+    async def device_poll(request: Request, poll_id: str = Form(...)):
+        require_admin(request)
+        payload = store.get_payload(poll_id)
+        if payload is None:
+            raise HTTPException(404, "Unknown or expired login session")
+        if store.is_throttled(poll_id):
+            return {"status": "pending"}
+
+        try:
+            outcome = await _maybe_await(poll_flow(request, payload))
+        except Exception:
+            store.pop(poll_id)
+            raise
+
+        if outcome.status == "authorized":
+            store.pop(poll_id)
+            return {"status": "authorized", "endpoint": dict(outcome.endpoint or {})}
+        if outcome.status == "failed":
+            store.pop(poll_id)
+            return {"status": "failed", "error": outcome.error or "denied"}
+        if outcome.status == "slow_down":
+            store.slow_down(poll_id, outcome.interval)
+            return _pending_response(outcome.detail)
+
+        store.schedule_next(poll_id)
+        return _pending_response(outcome.detail)
+
+    @router.post("/device/cancel")
+    def device_cancel(request: Request, poll_id: str = Form(...)):
+        require_admin(request)
+        store.pop(poll_id)
+        return {"status": "cancelled"}
+
+    return router

routes/diagnostics_routes.py

@@ -1,12 +1,14 @@
 """Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research."""
 
 import logging
+import os
 from typing import Dict, Any
 
-from fastapi import APIRouter, HTTPException, Form
+from fastapi import APIRouter, HTTPException, Form, Request
 
 from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async
-from core.constants import DEFAULT_HOST
+from core.constants import DEFAULT_HOST, DATA_DIR
+from core.middleware import require_admin
 
 logger = logging.getLogger(__name__)
 
@@ -15,11 +17,45 @@ def setup_diagnostics_routes(
     rag_manager,
     rag_available: bool,
     research_handler,
+    memory_vector=None,
 ) -> APIRouter:
     router = APIRouter(tags=["diagnostics"])
 
+    @router.get("/api/diagnostics/services")
+    async def get_service_health(request: Request) -> Dict[str, Any]:
+        """Consolidated degraded-state report for ChromaDB, SearXNG, email,
+        ntfy, and provider endpoints. Non-intrusive probes — safe to poll."""
+        require_admin(request)
+        from src.service_health import collect_service_health
+        return await collect_service_health(rag_manager, memory_vector)
+
+    @router.get("/api/diagnostics/logs")
+    async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]:
+        require_admin(request)
+        limit = max(1, min(limit, 1000))
+        try:
+            log_file = os.path.join(DATA_DIR, "logs", "app.log")
+            if not os.path.exists(log_file):
+                return {"status": "success", "logs": []}
+
+            # Safe tail read of the log file (max 5MB via rotation)
+            with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
+                lines = f.readlines()
+
+            tail_lines = lines[-limit:] if len(lines) > limit else lines
+            tail_lines = [line.rstrip('\r\n') for line in tail_lines]
+
+            return {
+                "status": "success",
+                "logs": tail_lines
+            }
+        except Exception as e:
+            logger.error(f"Diagnostics logs retrieval error: {e}")
+            raise HTTPException(500, f"Failed to retrieve logs: {str(e)}")
+
     @router.get("/api/db/stats")
-    async def get_database_stats() -> Dict[str, Any]:
+    async def get_database_stats(request: Request) -> Dict[str, Any]:
+        require_admin(request)
         try:
             from core.database import get_detailed_stats
             return get_detailed_stats()
@@ -28,13 +64,15 @@ def setup_diagnostics_routes(
             raise HTTPException(500, "Failed to retrieve database statistics")
 
     @router.get("/api/rag/stats")
-    async def get_rag_stats() -> Dict[str, Any]:
+    async def get_rag_stats(request: Request) -> Dict[str, Any]:
+        require_admin(request)
         if rag_available and rag_manager:
             return rag_manager.get_stats()
         return {"error": "RAG system not available"}
 
     @router.get("/api/test/youtube")
-    async def test_youtube(url: str) -> Dict[str, Any]:
+    async def test_youtube(request: Request, url: str) -> Dict[str, Any]:
+        require_admin(request)
         try:
             video_id = extract_youtube_id(url)
             if not video_id:
@@ -54,7 +92,8 @@ def setup_diagnostics_routes(
             return {"error": str(e)}
 
     @router.post("/api/test-research")
-    async def test_research(query: str = Form("What is machine learning?")) -> Dict[str, Any]:
+    async def test_research(request: Request, query: str = Form("What is machine learning?")) -> Dict[str, Any]:
+        require_admin(request)
         try:
             endpoint = f"http://{DEFAULT_HOST}:8000/v1/chat/completions"
             model = "gpt-oss-120b"

routes/document/__init__.py

@@ -0,0 +1,6 @@
+"""Document route domain package (slice 2m, #4082/#4071).
+
+Contains document_routes.py and document_helpers.py, migrated from the flat
+routes/ directory. Backward-compat shims at routes/document_routes.py and
+routes/document_helpers.py re-export from here.
+"""

routes/document/document_helpers.py

@@ -0,0 +1,243 @@
+"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+
+"""Document routes — CRUD for living documents with version history."""
+
+import logging
+import os
+import re
+from typing import Any, Dict, Optional
+
+from fastapi import HTTPException, Request
+from pydantic import BaseModel
+
+from core.database import Document, DocumentVersion
+from core.database import Session as DbSession
+from src.auth_helpers import _auth_disabled
+from src.upload_handler import UploadHandler
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class DocumentCreate(BaseModel):
+    session_id: Optional[str] = None
+    title: str = "Untitled"
+    language: Optional[str] = None
+    content: str = ""
+
+class DocumentUpdate(BaseModel):
+    content: str
+    summary: Optional[str] = None
+    force_version: bool = False
+
+class DocumentPatch(BaseModel):
+    title: Optional[str] = None
+    language: Optional[str] = None
+    session_id: Optional[str] = None  # link/unlink document to a session
+
+
+# ---- Helpers ----
+
+def _doc_to_dict(doc: Document) -> Dict[str, Any]:
+    return {
+        "id": doc.id,
+        "session_id": doc.session_id,
+        "title": doc.title,
+        "language": doc.language,
+        "current_content": doc.current_content,
+        "version_count": doc.version_count,
+        "is_active": doc.is_active,
+        "archived": bool(getattr(doc, "archived", False)),
+        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
+        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
+        # Source-email provenance (set when doc was created from an email
+        # attachment) — drives the "Send signed reply" menu item.
+        "source_email_uid":        getattr(doc, "source_email_uid", None),
+        "source_email_folder":     getattr(doc, "source_email_folder", None),
+        "source_email_account_id": getattr(doc, "source_email_account_id", None),
+        "source_email_message_id": getattr(doc, "source_email_message_id", None),
+    }
+
+def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
+    return {
+        "id": v.id,
+        "document_id": v.document_id,
+        "version_number": v.version_number,
+        "content": v.content,
+        "summary": v.summary,
+        "source": v.source,
+        "created_at": v.created_at.isoformat() if v.created_at else None,
+    }
+
+
+def _verify_doc_owner(db, doc: Document, user: str):
+    """Verify `user` owns this document. Raise 404 if not.
+
+    Documents now carry their own `owner` column, so a doc whose session
+    was deleted (session_id → NULL) can still prove ownership and stay
+    openable / cloneable. We trust that column first and only fall back to
+    the session join for any not-yet-backfilled legacy row.
+    """
+    if user is None:
+        if _auth_disabled():
+            return  # Single-user / no-auth mode: allow access
+        raise HTTPException(403, "Authentication required")
+    if doc.owner is not None:
+        if doc.owner != user:
+            raise HTTPException(404, "Document not found")
+        return
+    # Legacy fallback: derive ownership from the linked session.
+    if not doc.session_id:
+        raise HTTPException(404, "Document not found")
+    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
+    if not session or session.owner != user:
+        raise HTTPException(404, "Document not found")
+
+
+def _owner_session_filter(q, user):
+    """Restrict a documents query to those owned by `user`.
+
+    Documents now carry their own `owner` column (backfilled at boot from
+    the linked session, or assigned to the admin user for legacy/orphaned
+    docs). We filter on that directly rather than on a session join, so a
+    document whose session was deleted (session_id → NULL) still shows up
+    for its owner instead of silently vanishing from the Library + search.
+
+    The owner backfill runs in init_db before the app serves requests, so
+    by the time this filter is live there are no NULL-owner rows to leak;
+    we therefore match the owner strictly for authenticated callers."""
+    if not user:
+        if user == "" or _auth_disabled():
+            return q
+        return q.filter(False)
+    return q.filter(Document.owner == user)
+
+
+
+def _slug(name: str) -> str:
+    """Filesystem-friendly version of a document title.
+
+    Whitespace becomes underscores; other unsafe punctuation is dropped.
+    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
+    """
+    import re as _re
+    s = (name or "").strip()
+    # Drop the trailing extension if the title happens to include one
+    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
+    s = _re.sub(r'\s+', '_', s)
+    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
+    s = _re.sub(r'_+', '_', s).strip('_')
+    return s or "form"
+
+
+# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
+_PDF_RENDER_SCALE = 2.0
+
+
+def _upload_path_inside(upload_dir: str, path: str) -> bool:
+    base = os.path.realpath(upload_dir)
+    p = os.path.realpath(path)
+    try:
+        return os.path.commonpath([base, p]) == base
+    except Exception:
+        return False
+
+
+def _resolve_user_upload_path(
+    upload_handler: Any,
+    upload_id: str,
+    owner: Optional[str],
+    auth_manager=None,
+) -> Optional[str]:
+    """Resolve an upload id to a filesystem path the caller may read."""
+    if upload_handler is None:
+        return None
+    resolved = upload_handler.resolve_upload(
+        upload_id,
+        owner=owner,
+        auth_manager=auth_manager,
+    )
+    if not isinstance(resolved, dict) or not resolved:
+        return None
+    path = resolved.get("path")
+    upload_dir = getattr(upload_handler, "upload_dir", None)
+    if path and upload_dir and not _upload_path_inside(upload_dir, path):
+        logger.warning("Upload path outside upload directory: %s", path)
+        return None
+    return path
+
+
+def _locate_upload(
+    upload_dir: str,
+    file_id: str,
+    owner: Optional[str] = None,
+    auth_manager=None,
+    upload_handler: Any = None,
+):
+    """Find an upload by its filename ID via UploadHandler.resolve_upload."""
+    if upload_handler is None:
+        from src.upload_handler import UploadHandler
+
+        base_dir = os.path.dirname(os.path.abspath(upload_dir))
+        upload_handler = UploadHandler(base_dir, upload_dir)
+    return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager)
+
+
+def _assert_pdf_marker_upload_owned(
+    request: Request,
+    content: str,
+    user: Optional[str],
+    upload_handler: Any,
+) -> None:
+    """Reject document content whose pdf_source marker points at another user's upload."""
+    if upload_handler is None:
+        return
+    from src.pdf_form_doc import find_source_upload_id
+
+    upload_id = find_source_upload_id(content or "")
+    if not upload_id:
+        return
+    auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None)
+    if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager):
+        raise HTTPException(
+            400,
+            "Document PDF marker references an upload you do not own",
+        )
+
+
+def _derive_title(content: str) -> str:
+    """Derive a title from document content."""
+    import re
+    if not isinstance(content, str):
+        return "Untitled"
+    text = content.strip()
+    if not text:
+        return "Untitled"
+
+    # Markdown header
+    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
+    if md:
+        title = md.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # HTML heading
+    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
+    if html:
+        title = html.group(1).strip()
+        if len(title) > 50:
+            title = title[:48] + "…"
+        return title
+
+    # First non-empty line (if short enough)
+    for line in text.split('\n'):
+        line = line.strip()
+        if line and 2 <= len(line) <= 60:
+            title = re.sub(r'[:#*`]+$', '', line).strip()
+            if title and len(title) > 50:
+                title = title[:48] + "…"
+            return title or "Untitled"
+
+    return "Untitled"

routes/document_helpers.py

@@ -1,198 +1,14 @@
-"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py."""
+"""Backward-compat shim — canonical location is routes/document/document_helpers.py.
 
-"""Document routes — CRUD for living documents with version history."""
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.document_helpers``, ``from routes.document_helpers import
+X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import
+pattern used by test_security_regressions.py all operate on the *same* object.
+Keeps existing import paths working after slice 2m (#4082/#4071).
+"""
 
-import logging
-from typing import Dict, Any, Optional
+import sys as _sys
 
-from fastapi import HTTPException
-from pydantic import BaseModel
+from routes.document import document_helpers as _canonical  # noqa: F401
 
-from core.database import Document, DocumentVersion
-from core.database import Session as DbSession
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class DocumentCreate(BaseModel):
-    session_id: Optional[str] = None
-    title: str = "Untitled"
-    language: Optional[str] = None
-    content: str = ""
-
-class DocumentUpdate(BaseModel):
-    content: str
-    summary: Optional[str] = None
-
-class DocumentPatch(BaseModel):
-    title: Optional[str] = None
-    language: Optional[str] = None
-    session_id: Optional[str] = None  # link/unlink document to a session
-
-
-# ---- Helpers ----
-
-def _doc_to_dict(doc: Document) -> Dict[str, Any]:
-    return {
-        "id": doc.id,
-        "session_id": doc.session_id,
-        "title": doc.title,
-        "language": doc.language,
-        "current_content": doc.current_content,
-        "version_count": doc.version_count,
-        "is_active": doc.is_active,
-        "archived": bool(getattr(doc, "archived", False)),
-        "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
-        "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None,
-        # Source-email provenance (set when doc was created from an email
-        # attachment) — drives the "Send signed reply" menu item.
-        "source_email_uid":        getattr(doc, "source_email_uid", None),
-        "source_email_folder":     getattr(doc, "source_email_folder", None),
-        "source_email_account_id": getattr(doc, "source_email_account_id", None),
-        "source_email_message_id": getattr(doc, "source_email_message_id", None),
-    }
-
-def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]:
-    return {
-        "id": v.id,
-        "document_id": v.document_id,
-        "version_number": v.version_number,
-        "content": v.content,
-        "summary": v.summary,
-        "source": v.source,
-        "created_at": v.created_at.isoformat() if v.created_at else None,
-    }
-
-
-def _verify_doc_owner(db, doc: Document, user: str):
-    """Verify `user` owns this document. Raise 404 if not.
-
-    Documents now carry their own `owner` column, so a doc whose session
-    was deleted (session_id → NULL) can still prove ownership and stay
-    openable / cloneable. We trust that column first and only fall back to
-    the session join for any not-yet-backfilled legacy row.
-    """
-    if user is None:
-        raise HTTPException(403, "Authentication required")
-    if doc.owner is not None:
-        if doc.owner != user:
-            raise HTTPException(404, "Document not found")
-        return
-    # Legacy fallback: derive ownership from the linked session.
-    if not doc.session_id:
-        raise HTTPException(404, "Document not found")
-    session = db.query(DbSession).filter(DbSession.id == doc.session_id).first()
-    if not session or session.owner != user:
-        raise HTTPException(404, "Document not found")
-
-
-def _owner_session_filter(q, user):
-    """Restrict a documents query to those owned by `user`.
-
-    Documents now carry their own `owner` column (backfilled at boot from
-    the linked session, or assigned to the admin user for legacy/orphaned
-    docs). We filter on that directly rather than on a session join, so a
-    document whose session was deleted (session_id → NULL) still shows up
-    for its owner instead of silently vanishing from the Library + search.
-
-    The owner backfill runs in init_db before the app serves requests, so
-    by the time this filter is live there are no NULL-owner rows to leak;
-    we therefore match the owner strictly."""
-    if user is None:
-        return q.filter(False)
-    return q.filter(Document.owner == user)
-
-
-
-def _slug(name: str) -> str:
-    """Filesystem-friendly version of a document title.
-
-    Whitespace becomes underscores; other unsafe punctuation is dropped.
-    Preserves letters, digits, dot, hyphen, underscore. Idempotent.
-    """
-    import re as _re
-    s = (name or "").strip()
-    # Drop the trailing extension if the title happens to include one
-    s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE)
-    s = _re.sub(r'\s+', '_', s)
-    s = _re.sub(r'[^A-Za-z0-9._-]', '', s)
-    s = _re.sub(r'_+', '_', s).strip('_')
-    return s or "form"
-
-
-# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units).
-_PDF_RENDER_SCALE = 2.0
-
-
-def _locate_upload(upload_dir: str, file_id: str):
-    """Find an upload by its filename ID.
-
-    Lookup order:
-      1. Direct hit at `upload_dir/file_id` (very small deployments).
-      2. The `uploads.json` index that `UploadHandler.save_upload` maintains —
-         maps file_hash → metadata containing the full path. O(1) once loaded.
-      3. Fallback: `os.walk` the date-bucketed tree. Slow on large stores;
-         only triggers for legacy uploads recorded before the index existed.
-
-    `followlinks=False` keeps a stray symlink loop in `data/uploads/` from
-    spinning the walker into infinite recursion.
-    """
-    import os
-    import json as _json
-    direct = os.path.join(upload_dir, file_id)
-    if os.path.exists(direct):
-        return direct
-    # O(1) via uploads.json
-    try:
-        idx_path = os.path.join(upload_dir, "uploads.json")
-        if os.path.exists(idx_path):
-            with open(idx_path, "r") as f:
-                idx = _json.load(f)
-            for meta in (idx.values() if isinstance(idx, dict) else []):
-                if meta.get("id") == file_id:
-                    p = meta.get("path")
-                    if p and os.path.exists(p):
-                        return p
-    except Exception:
-        pass
-    for root, _dirs, files in os.walk(upload_dir, followlinks=False):
-        if file_id in files:
-            return os.path.join(root, file_id)
-    return None
-
-
-def _derive_title(content: str) -> str:
-    """Derive a title from document content."""
-    import re
-    text = content.strip()
-    if not text:
-        return "Untitled"
-
-    # Markdown header
-    md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
-    if md:
-        title = md.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # HTML heading
-    html = re.search(r'<h[1-3][^>]*>([^<]+)</h[1-3]>', text, re.IGNORECASE)
-    if html:
-        title = html.group(1).strip()
-        if len(title) > 50:
-            title = title[:48] + "…"
-        return title
-
-    # First non-empty line (if short enough)
-    for line in text.split('\n'):
-        line = line.strip()
-        if line and 2 <= len(line) <= 60:
-            title = re.sub(r'[:#*`]+$', '', line).strip()
-            if title and len(title) > 50:
-                title = title[:48] + "…"
-            return title or "Untitled"
-
-    return "Untitled"
+_sys.modules[__name__] = _canonical

routes/editor_draft_routes.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel
 
 from core.database import EditorDraft, SessionLocal
 from src.auth_helpers import get_current_user
+from src.upload_limits import EDITOR_DRAFT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
 
@@ -67,6 +68,24 @@ def _summary(d: EditorDraft) -> Dict[str, Any]:
     }
 
 
+def _load_payload(raw: Optional[str]) -> Dict[str, Any]:
+    try:
+        payload = json.loads(raw) if raw else {}
+    except Exception:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _dump_payload(payload: Dict[str, Any]) -> str:
+    raw = json.dumps(payload or {}, separators=(",", ":"))
+    if len(raw.encode("utf-8")) > EDITOR_DRAFT_MAX_BYTES:
+        raise HTTPException(
+            413,
+            f"Editor draft exceeds the {EDITOR_DRAFT_MAX_BYTES // (1024 * 1024)} MB safety limit",
+        )
+    return raw
+
+
 def setup_editor_draft_routes() -> APIRouter:
     router = APIRouter(tags=["editor-drafts"])
 
@@ -93,13 +112,9 @@ def setup_editor_draft_routes() -> APIRouter:
             ).first()
             if not d or not _owns(d, user):
                 raise HTTPException(404, "Draft not found")
-            try:
-                payload = json.loads(d.payload) if d.payload else {}
-            except Exception:
-                payload = {}
             return {
                 **_summary(d),
-                "payload": payload,
+                "payload": _load_payload(d.payload),
             }
         finally:
             db.close()
@@ -116,13 +131,15 @@ def setup_editor_draft_routes() -> APIRouter:
                 source_image_id=body.source_image_id,
                 width=body.width,
                 height=body.height,
-                payload=json.dumps(body.payload or {}),
+                payload=_dump_payload(body.payload),
                 thumbnail=body.thumbnail,
             )
             db.add(d)
             db.commit()
             db.refresh(d)
             return _summary(d)
+        except HTTPException:
+            raise
         except Exception as e:
             db.rollback()
             logger.warning(f"editor-draft create failed: {e}")
@@ -147,7 +164,7 @@ def setup_editor_draft_routes() -> APIRouter:
             if body.height is not None:
                 d.height = body.height
             if body.payload is not None:
-                d.payload = json.dumps(body.payload)
+                d.payload = _dump_payload(body.payload)
             if body.thumbnail is not None:
                 d.thumbnail = body.thumbnail
             db.commit()

routes/embedding_routes.py

@@ -6,12 +6,14 @@ import shutil
 import logging
 import asyncio
 from pathlib import Path
-from fastapi import APIRouter, HTTPException, Form
-from core.constants import BASE_DIR
+from fastapi import APIRouter, HTTPException, Form, Depends
+from core.constants import EMBEDDING_ENDPOINT_FILE, FASTEMBED_CACHE_DIR
+from core.middleware import require_admin
+from src.runtime_paths import get_app_root
 
 logger = logging.getLogger(__name__)
 
-_ENDPOINT_FILE = os.path.join(BASE_DIR, "data", "embedding_endpoint.json")
+_ENDPOINT_FILE = EMBEDDING_ENDPOINT_FILE
 
 # Track in-progress downloads
 _downloading: dict = {}
@@ -34,13 +36,7 @@ def _cache_dir() -> str:
     default lived in /tmp, which many systems wipe on reboot — forcing a
     full re-download of the embedding model after every restart.
     """
-    env = os.environ.get("FASTEMBED_CACHE_PATH")
-    if env:
-        return env
-    return os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        "data", "fastembed_cache",
-    )
+    return FASTEMBED_CACHE_DIR
 
 
 def _model_cache_name(hf_source: str) -> str:
@@ -48,19 +44,35 @@ def _model_cache_name(hf_source: str) -> str:
     return "models--" + hf_source.replace("/", "--")
 
 
+def _model_cache_path(hf_source: str) -> Path:
+    """Return a confined cache path for a fastembed HF source."""
+    root = Path(_cache_dir()).expanduser().resolve()
+    raw_path = root / _model_cache_name(hf_source)
+    if raw_path.is_symlink():
+        raise ValueError("Model cache path must not be a symlink")
+    path = raw_path.resolve(strict=False)
+    try:
+        path.relative_to(root)
+    except ValueError:
+        raise ValueError("Model cache path escapes cache root")
+    return path
+
+
 def _is_downloaded(hf_source: str) -> bool:
     """Check if a model is already cached."""
-    cache = _cache_dir()
-    model_dir = os.path.join(cache, _model_cache_name(hf_source))
-    if not os.path.isdir(model_dir):
+    try:
+        model_dir = _model_cache_path(hf_source)
+    except ValueError:
+        return False
+    if not model_dir.is_dir():
         return False
     # Check for actual model files (not just empty dir)
-    snapshots = os.path.join(model_dir, "snapshots")
-    if os.path.isdir(snapshots):
-        return any(os.listdir(snapshots))
+    snapshots = model_dir / "snapshots"
+    if snapshots.is_dir():
+        return any(snapshots.iterdir())
     # Also check for blobs (older cache format)
-    blobs = os.path.join(model_dir, "blobs")
-    return os.path.isdir(blobs) and any(os.listdir(blobs))
+    blobs = model_dir / "blobs"
+    return blobs.is_dir() and any(blobs.iterdir())
 
 
 def _active_model() -> str:
@@ -85,7 +97,8 @@ def _load_custom_endpoint() -> dict:
     """Load the saved custom embedding endpoint, if any."""
     try:
         if os.path.exists(_ENDPOINT_FILE):
-            return json.loads(Path(_ENDPOINT_FILE).read_text())
+            data = json.loads(Path(_ENDPOINT_FILE).read_text(encoding="utf-8"))
+            return data if isinstance(data, dict) else {}
     except Exception:
         pass
     return {}
@@ -93,11 +106,11 @@ def _load_custom_endpoint() -> dict:
 
 def _save_custom_endpoint(data: dict):
     Path(_ENDPOINT_FILE).parent.mkdir(parents=True, exist_ok=True)
-    Path(_ENDPOINT_FILE).write_text(json.dumps(data, indent=2))
+    Path(_ENDPOINT_FILE).write_text(json.dumps(data, indent=2), encoding="utf-8")
 
 
 def setup_embedding_routes():
-    router = APIRouter(prefix="/api/embeddings")
+    router = APIRouter(prefix="/api/embeddings", dependencies=[Depends(require_admin)])
 
     @router.get("/models")
     def list_models():
@@ -117,8 +130,10 @@ def setup_embedding_routes():
 
             cached_size = None
             if downloaded and hf_src:
-                model_path = os.path.join(_cache_dir(), _model_cache_name(hf_src))
-                cached_size = _dir_size_mb(model_path)
+                try:
+                    cached_size = _dir_size_mb(str(_model_cache_path(hf_src)))
+                except ValueError:
+                    cached_size = None
 
             result.append({
                 "model": m["model"],
@@ -159,7 +174,7 @@ def setup_embedding_routes():
         _downloading[model_name] = True
         try:
             # Run in thread to not block the event loop
-            loop = asyncio.get_event_loop()
+            loop = asyncio.get_running_loop()
             cache = _cache_dir()
             await loop.run_in_executor(
                 None,
@@ -215,8 +230,11 @@ def setup_embedding_routes():
         if not hf_src:
             raise HTTPException(400, "No cache source for this model")
 
-        model_path = os.path.join(_cache_dir(), _model_cache_name(hf_src))
-        if not os.path.isdir(model_path):
+        try:
+            model_path = _model_cache_path(hf_src)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+        if not model_path.is_dir():
             return {"deleted": False, "message": "Model not cached"}
 
         shutil.rmtree(model_path)
@@ -235,18 +253,31 @@ def setup_embedding_routes():
         }
 
     @router.post("/endpoint")
-    def set_endpoint(url: str = Form(...), model: str = Form("")):
+    def set_endpoint(url: str = Form(...), model: str = Form(""), api_key: str = Form("")):
         """Save a custom embedding endpoint URL."""
         url = url.strip()
         if not url:
             raise HTTPException(400, "URL is required")
 
+        # SSRF hardening: validate the user-supplied URL before any outbound
+        # request. Local-first means loopback/LAN endpoints are allowed by
+        # default; non-HTTP(S) schemes and the cloud metadata range are always
+        # rejected. Set EMBEDDING_BLOCK_PRIVATE_IPS=true for full lockdown.
+        from src.url_safety import check_outbound_url
+        ok, reason = check_outbound_url(
+            url,
+            block_private=os.getenv("EMBEDDING_BLOCK_PRIVATE_IPS", "false").lower() == "true",
+        )
+        if not ok:
+            raise HTTPException(400, f"Rejected endpoint URL: {reason}")
+
         # Quick health check
         try:
             import httpx
             resp = httpx.post(
                 url,
                 json={"input": ["test"], "model": model or "test"},
+                headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
                 timeout=10,
             )
             resp.raise_for_status()
@@ -257,10 +288,16 @@ def setup_embedding_routes():
         data = {"url": url}
         if model:
             data["model"] = model
+        if api_key:
+            from src.secret_storage import encrypt
+            data["api_key"] = encrypt(api_key)
+
         _save_custom_endpoint(data)
         os.environ["EMBEDDING_URL"] = url
         if model:
             os.environ["EMBEDDING_MODEL"] = model
+        if api_key:
+            os.environ["EMBEDDING_API_KEY"] = api_key
 
         # Reset the RAG singleton so it picks up the new endpoint
         import src.rag_singleton as _rs
@@ -274,6 +311,16 @@ def setup_embedding_routes():
             reset_http_embed_state()
         except Exception:
             pass
+        try:
+            from src.embedding_lanes import reset_embedding_lane_state
+            reset_embedding_lane_state()
+        except Exception:
+            pass
+        try:
+            from src.tool_index import reset_tool_index
+            reset_tool_index()
+        except Exception:
+            pass
 
         # Reset ChromaDB client (collections will be recreated with new embeddings)
         try:
@@ -294,6 +341,7 @@ def setup_embedding_routes():
         # Remove from environment
         os.environ.pop("EMBEDDING_URL", None)
         os.environ.pop("EMBEDDING_MODEL", None)
+        os.environ.pop("EMBEDDING_API_KEY", None)
 
         # Reset the RAG singleton so it falls back to fastembed
         import src.rag_singleton as _rs
@@ -304,6 +352,16 @@ def setup_embedding_routes():
             reset_http_embed_state()
         except Exception:
             pass
+        try:
+            from src.embedding_lanes import reset_embedding_lane_state
+            reset_embedding_lane_state()
+        except Exception:
+            pass
+        try:
+            from src.tool_index import reset_tool_index
+            reset_tool_index()
+        except Exception:
+            pass
 
         # Reset ChromaDB client
         try:

routes/emoji_routes.py

@@ -16,22 +16,54 @@ from pathlib import Path
 
 import httpx
 from fastapi import APIRouter
-from fastapi.responses import FileResponse, Response
+from fastapi.responses import Response
+
+from src.constants import EMOJI_CACHE_DIR
 
 logger = logging.getLogger(__name__)
 
-_CACHE_DIR = Path(__file__).resolve().parent.parent / "data" / "emoji_cache"
+_CACHE_DIR = Path(EMOJI_CACHE_DIR)
 # OpenMoji "black" set = monochrome line-art SVGs. Filenames are the codepoints
 # in UPPERCASE (FE0F dropped, same as we compute), '-' joined.
 _OPENMOJI_BASE = "https://cdn.jsdelivr.net/npm/openmoji@15.0.0/black/svg"
 # codepoints like "1f600" or "1f468-200d-1f469-200d-1f467" (lowercase hex, '-' joined)
 _CODE_RE = re.compile(r"^[0-9a-f]{2,6}(?:-[0-9a-f]{2,6})*$")
-_SVG_HEADERS = {"Cache-Control": "public, max-age=31536000, immutable"}
+_MAX_SVG_BYTES = 256 * 1024
+_BLOCKED_SVG_RE = re.compile(
+    br"<\s*(?:script|foreignObject|iframe|object|embed|image)\b|"
+    br"\bon[a-z0-9_-]+\s*=",
+    re.IGNORECASE,
+)
+_EXTERNAL_REF_RE = re.compile(
+    br"\b(?:href|xlink:href)\s*=\s*['\"](?:https?:|//|data:|javascript:)",
+    re.IGNORECASE,
+)
+_SVG_SECURITY_HEADERS = {
+    "X-Content-Type-Options": "nosniff",
+    "Content-Security-Policy": "sandbox",
+    "Cross-Origin-Resource-Policy": "same-origin",
+}
+_SVG_HEADERS = {
+    "Cache-Control": "public, max-age=31536000, immutable",
+    **_SVG_SECURITY_HEADERS,
+}
 # Returned when a codepoint is unknown/unreachable: an empty (transparent) SVG,
 # so the CSS mask renders nothing instead of a solid box. Not cached, so a later
 # request can still pick up the real glyph once the CDN is reachable.
 _BLANK_SVG = b'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1 1"></svg>'
-_BLANK_HEADERS = {"Cache-Control": "no-store"}
+_BLANK_HEADERS = {"Cache-Control": "no-store", **_SVG_SECURITY_HEADERS}
+
+
+def _is_safe_svg(content: bytes) -> bool:
+    if not isinstance(content, bytes) or not content:
+        return False
+    if len(content) > _MAX_SVG_BYTES:
+        return False
+    if b"<svg" not in content[:256].lower():
+        return False
+    if _BLOCKED_SVG_RE.search(content) or _EXTERNAL_REF_RE.search(content):
+        return False
+    return True
 
 
 def setup_emoji_routes() -> APIRouter:
@@ -49,14 +81,21 @@ def setup_emoji_routes() -> APIRouter:
         _CACHE_DIR.mkdir(parents=True, exist_ok=True)
         fp = _CACHE_DIR / f"{code}.svg"
         if fp.exists():
-            return FileResponse(fp, media_type="image/svg+xml", headers=_SVG_HEADERS)
+            try:
+                content = fp.read_bytes()
+                if _is_safe_svg(content):
+                    return Response(content, media_type="image/svg+xml", headers=_SVG_HEADERS)
+                fp.unlink(missing_ok=True)
+            except Exception as e:
+                logger.warning("emoji cache read %s failed: %s", code, e)
+            return _blank()
 
         # First time we've seen this emoji — fetch the OpenMoji black SVG + cache
         # it. OpenMoji filenames are the codepoints uppercased.
         try:
             async with httpx.AsyncClient(timeout=8.0) as client:
                 r = await client.get(f"{_OPENMOJI_BASE}/{code.upper()}.svg")
-            if r.status_code == 200 and b"<svg" in r.content[:256]:
+            if r.status_code == 200 and _is_safe_svg(r.content):
                 try:
                     fp.write_bytes(r.content)
                 except Exception:

routes/font_routes.py

@@ -5,6 +5,15 @@ from fastapi import APIRouter
 
 CUSTOM_FONTS_DIR = os.path.join("static", "fonts", "custom")
 FONT_EXTENSIONS = {".ttf", ".otf", ".woff", ".woff2"}
+FAMILY_SUFFIX_WORDS = ("Display", "Rounded", "Serif", "Sans", "Mono", "Code", "Text")
+
+
+def _split_family_token(token):
+    """Split common compact font-family suffixes without breaking brand names."""
+    for suffix in FAMILY_SUFFIX_WORDS:
+        if token.endswith(suffix) and len(token) > len(suffix):
+            return f"{token[:-len(suffix)]} {suffix}"
+    return re.sub(r'(?<=[a-z])(?=[A-Z])', ' ', token)
 
 
 def _derive_family(filename):
@@ -15,10 +24,9 @@ def _derive_family(filename):
         r'[-_ ]?(Thin|ExtraLight|UltraLight|Light|Regular|Medium|SemiBold|DemiBold|Bold|ExtraBold|UltraBold|Black|Heavy|Italic|Oblique|Variable|VF)$',
         '', name, flags=re.IGNORECASE
     )
-    # Insert spaces before uppercase runs: "JetBrainsMono" → "Jet Brains Mono"
-    name = re.sub(r'(?<=[a-z])(?=[A-Z])', ' ', name)
     # Replace dashes/underscores with spaces
     name = re.sub(r'[-_]+', ' ', name).strip()
+    name = " ".join(_split_family_token(part) for part in name.split())
     return name or filename
 
 

routes/gallery/__init__.py

@@ -0,0 +1,6 @@
+"""Gallery route domain package (slice 2a, #4082/#4071).
+
+Contains gallery_routes.py and gallery_helpers.py, migrated from the flat
+routes/ directory. Backward-compat shims at routes/gallery_routes.py and
+routes/gallery_helpers.py re-export from here.
+"""

routes/gallery/gallery_helpers.py

@@ -0,0 +1,145 @@
+"""gallery_helpers.py — extracted helpers, models, and small utilities.
+
+Imported by gallery_routes.py."""
+
+"""Gallery routes — browsable library for photos and AI-generated images."""
+
+import logging
+from datetime import datetime
+from typing import Dict, Any, Optional
+
+from pydantic import BaseModel
+
+from core.database import GalleryImage
+from src.auth_helpers import _auth_disabled
+
+logger = logging.getLogger(__name__)
+
+
+# ---- Request schemas ----
+
+class GalleryPatch(BaseModel):
+    tags: Optional[str] = None
+    favorite: Optional[bool] = None
+    album_id: Optional[str] = None
+
+
+# ---- EXIF extraction ----
+
+def _extract_exif(content: bytes) -> dict:
+    """Extract EXIF metadata from image bytes. Returns dict of fields."""
+    result = {"width": None, "height": None}
+    try:
+        from PIL import Image
+        from io import BytesIO
+        img = Image.open(BytesIO(content))
+        # Read the raw EXIF before any transpose: exif_transpose strips the
+        # orientation tag and with it the parsed EXIF view.
+        exif = img._getexif() if hasattr(img, '_getexif') else None
+
+        # Record DISPLAY dimensions (EXIF-rotated), matching upload_handler.
+        # A phone photo with Orientation 6/8 is stored landscape but shown
+        # portrait, so the raw width/height swap the aspect ratio.
+        try:
+            from PIL import ImageOps
+            img = ImageOps.exif_transpose(img) or img
+        except Exception:
+            pass
+        result["width"] = img.width
+        result["height"] = img.height
+
+        if not exif:
+            return result
+
+        # EXIF tag IDs
+        # 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
+        # 34853=GPSInfo
+        result["camera_make"] = str(exif.get(271, "")).strip() or None
+        result["camera_model"] = str(exif.get(272, "")).strip() or None
+
+        # Date taken
+        for tag_id in (36867, 36868, 306):  # DateTimeOriginal, DateTimeDigitized, DateTime
+            raw = exif.get(tag_id)
+            if raw:
+                try:
+                    result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
+                    break
+                except (ValueError, TypeError):
+                    pass
+
+        # GPS
+        gps_info = exif.get(34853)
+        if gps_info and isinstance(gps_info, dict):
+            try:
+                def _to_deg(vals):
+                    d, m, s = [float(v) for v in vals]
+                    return d + m / 60 + s / 3600
+                if 2 in gps_info and 4 in gps_info:
+                    lat = _to_deg(gps_info[2])
+                    lng = _to_deg(gps_info[4])
+                    if gps_info.get(1) == 'S': lat = -lat
+                    if gps_info.get(3) == 'W': lng = -lng
+                    result["gps_lat"] = f"{lat:.6f}"
+                    result["gps_lng"] = f"{lng:.6f}"
+            except Exception:
+                pass
+    except Exception as e:
+        # User-visible failure (photo loses metadata): surface at WARNING
+        # and record on the result so the upload endpoint can pass it back.
+        logger.warning(f"EXIF extraction failed: {e}")
+        result["exif_error"] = str(e)
+    return result
+
+
+# ---- Helpers ----
+
+def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
+    return {
+        "id": img.id,
+        "filename": img.filename,
+        "url": f"/api/generated-image/{img.filename}",
+        "prompt": img.prompt,
+        "caption": img.caption or "",
+        "model": img.model,
+        "size": img.size,
+        "quality": img.quality,
+        "tags": img.tags or "",
+        "ai_tags": img.ai_tags or "",
+        "user_tags": img.tags or "",
+        "session_id": img.session_id,
+        "session_name": session_name,
+        "album_id": img.album_id,
+        "is_active": img.is_active,
+        "favorite": img.favorite or False,
+        "taken_at": img.taken_at.isoformat() if img.taken_at else None,
+        "camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
+        "gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
+        "width": img.width,
+        "height": img.height,
+        "file_size": img.file_size,
+        "created_at": img.created_at.isoformat() if img.created_at else None,
+        "updated_at": img.updated_at.isoformat() if img.updated_at else None,
+    }
+
+
+def _owner_filter(q, user, model_cls=GalleryImage):
+    """Apply owner filtering to a gallery query.
+
+    ``get_current_user`` returns None both in auth-disabled single-user mode
+    and when auth is enabled but no current user was resolved. Preserve the
+    single-user behavior, but fail closed for auth-enabled null-user states.
+    """
+    if user is not None:
+        return q.filter(model_cls.owner == user)
+    if _auth_disabled():
+        return q
+    return q.filter(False)
+
+
+
+def _human_size(nbytes):
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if abs(nbytes) < 1024:
+            return f"{nbytes:.1f} {unit}"
+        nbytes /= 1024
+    return f"{nbytes:.1f} PB"

routes/gallery_helpers.py

@@ -1,125 +1,14 @@
-"""gallery_helpers.py — extracted helpers, models, and small utilities.
+"""Backward-compat shim - canonical location is routes/gallery/gallery_helpers.py.
 
-Imported by gallery_routes.py."""
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.gallery_helpers``, ``from routes.gallery_helpers import X``,
+``importlib.import_module("routes.gallery_helpers")``, and
+``monkeypatch.setattr(routes.gallery_helpers, ...)`` all operate on the same
+object. Keeps existing import paths working after slice 2a (#4082/#4071).
+"""
 
-"""Gallery routes — browsable library for photos and AI-generated images."""
+import sys as _sys
 
-import logging
-from datetime import datetime
-from typing import Dict, Any, Optional
+from routes.gallery import gallery_helpers as _canonical  # noqa: F401
 
-from pydantic import BaseModel
-
-from core.database import GalleryImage
-
-logger = logging.getLogger(__name__)
-
-
-# ---- Request schemas ----
-
-class GalleryPatch(BaseModel):
-    tags: Optional[str] = None
-    favorite: Optional[bool] = None
-    album_id: Optional[str] = None
-
-
-# ---- EXIF extraction ----
-
-def _extract_exif(content: bytes) -> dict:
-    """Extract EXIF metadata from image bytes. Returns dict of fields."""
-    result = {"width": None, "height": None}
-    try:
-        from PIL import Image
-        from io import BytesIO
-        img = Image.open(BytesIO(content))
-        result["width"] = img.width
-        result["height"] = img.height
-
-        exif = img._getexif() if hasattr(img, '_getexif') else None
-        if not exif:
-            return result
-
-        # EXIF tag IDs
-        # 271=Make, 272=Model, 306=DateTime, 36867=DateTimeOriginal
-        # 34853=GPSInfo
-        result["camera_make"] = str(exif.get(271, "")).strip() or None
-        result["camera_model"] = str(exif.get(272, "")).strip() or None
-
-        # Date taken
-        for tag_id in (36867, 36868, 306):  # DateTimeOriginal, DateTimeDigitized, DateTime
-            raw = exif.get(tag_id)
-            if raw:
-                try:
-                    result["taken_at"] = datetime.strptime(str(raw).strip(), "%Y:%m:%d %H:%M:%S")
-                    break
-                except (ValueError, TypeError):
-                    pass
-
-        # GPS
-        gps_info = exif.get(34853)
-        if gps_info and isinstance(gps_info, dict):
-            try:
-                def _to_deg(vals):
-                    d, m, s = [float(v) for v in vals]
-                    return d + m / 60 + s / 3600
-                if 2 in gps_info and 4 in gps_info:
-                    lat = _to_deg(gps_info[2])
-                    lng = _to_deg(gps_info[4])
-                    if gps_info.get(1) == 'S': lat = -lat
-                    if gps_info.get(3) == 'W': lng = -lng
-                    result["gps_lat"] = f"{lat:.6f}"
-                    result["gps_lng"] = f"{lng:.6f}"
-            except Exception:
-                pass
-    except Exception as e:
-        # User-visible failure (photo loses metadata): surface at WARNING
-        # and record on the result so the upload endpoint can pass it back.
-        logger.warning(f"EXIF extraction failed: {e}")
-        result["exif_error"] = str(e)
-    return result
-
-
-# ---- Helpers ----
-
-def _image_to_dict(img: GalleryImage, session_name: str = None) -> Dict[str, Any]:
-    return {
-        "id": img.id,
-        "filename": img.filename,
-        "url": f"/api/generated-image/{img.filename}",
-        "prompt": img.prompt,
-        "model": img.model,
-        "size": img.size,
-        "quality": img.quality,
-        "tags": img.tags or "",
-        "ai_tags": img.ai_tags or "",
-        "user_tags": img.tags or "",
-        "session_id": img.session_id,
-        "session_name": session_name,
-        "album_id": img.album_id,
-        "is_active": img.is_active,
-        "favorite": img.favorite or False,
-        "taken_at": img.taken_at.isoformat() if img.taken_at else None,
-        "camera": f"{img.camera_make or ''} {img.camera_model or ''}".strip() or None,
-        "gps": {"lat": img.gps_lat, "lng": img.gps_lng} if img.gps_lat else None,
-        "width": img.width,
-        "height": img.height,
-        "file_size": img.file_size,
-        "created_at": img.created_at.isoformat() if img.created_at else None,
-        "updated_at": img.updated_at.isoformat() if img.updated_at else None,
-    }
-
-
-def _owner_filter(q, user):
-    """Apply owner filtering to a gallery query."""
-    if user is None:
-        return q.filter(False)
-    return q.filter(GalleryImage.owner == user)
-
-
-
-def _human_size(nbytes):
-    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
-        if abs(nbytes) < 1024:
-            return f"{nbytes:.1f} {unit}"
-        nbytes /= 1024
-    return f"{nbytes:.1f} PB"
+_sys.modules[__name__] = _canonical

routes/history/__init__.py

@@ -0,0 +1,5 @@
+"""History route domain package (slice 2d, #4082/#4071).
+
+Contains history_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/history_routes.py re-exports from here.
+"""

routes/history_routes.py

@@ -1,619 +1,17 @@
-"""History routes — session history, truncation, fork, conversation topics."""
+"""Backward-compat shim — canonical location is routes/history/history_routes.py.
 
-import json
-import uuid
-import logging
-from typing import Dict, Any
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.history_routes``, ``from routes.history_routes import X``,
+``importlib.import_module("routes.history_routes")``, and the
+``import ... as history_routes`` + ``monkeypatch.setattr(history_routes, ...)``
+pattern used by test_history_compact_tool_calls.py / test_fork_session_metadata.py
+all operate on the *same* object the application actually uses. Keeps existing
+import paths working after slice 2d (#4082/#4071). Source-introspection tests
+read the canonical file by path.
+"""
 
-from fastapi import APIRouter, Request, HTTPException
+import sys as _sys
 
-from core.models import ChatMessage
-from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
-from src.topic_analyzer import analyze_topics
-from routes.session_routes import _verify_session_owner
+from routes.history import history_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-def setup_history_routes(session_manager) -> APIRouter:
-    router = APIRouter(tags=["history"])
-
-    @router.get("/api/history/{session_id}")
-    async def get_session_history(request: Request, session_id: str) -> Dict[str, Any]:
-        _verify_session_owner(request, session_id)
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session '{session_id}' not found")
-
-        history_dict = []
-        for msg in session.history:
-            if isinstance(msg, ChatMessage):
-                # Skip hidden messages (e.g. compaction summaries for AI context)
-                if msg.metadata and msg.metadata.get("hidden"):
-                    continue
-                entry = {"role": msg.role, "content": msg.content}
-                if msg.metadata:
-                    entry["metadata"] = msg.metadata
-                history_dict.append(entry)
-            elif isinstance(msg, dict):
-                if msg.get("metadata", {}).get("hidden"):
-                    continue
-                entry = {
-                    "role": msg.get("role", ""),
-                    "content": msg.get("content", ""),
-                }
-                if msg.get("metadata"):
-                    entry["metadata"] = msg["metadata"]
-                history_dict.append(entry)
-
-        # Fallback: load from DB if in-memory is empty
-        if not history_dict:
-            db = SessionLocal()
-            try:
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.timestamp)
-                    .all()
-                )
-                import json as _json
-                history_dict = []
-                for m in db_messages:
-                    entry = {"role": m.role, "content": m.content}
-                    meta = {}
-                    if m.meta_data:
-                        try:
-                            meta = _json.loads(m.meta_data) or {}
-                        except (json.JSONDecodeError, ValueError):
-                            meta = {}
-                    if m.timestamp and "timestamp" not in meta:
-                        meta["timestamp"] = m.timestamp.isoformat() + "Z"
-                    if meta:
-                        entry["metadata"] = meta
-                    history_dict.append(entry)
-                if history_dict:
-                    session.history = [
-                        ChatMessage(role=m["role"], content=m["content"], metadata=m.get("metadata"))
-                        for m in history_dict
-                    ]
-            except Exception as e:
-                logger.error(f"DB fallback failed for {session_id}: {e}")
-            finally:
-                db.close()
-
-        return {
-            "history": history_dict,
-            "model": session.model,
-            "endpoint_url": session.endpoint_url,
-            "name": session.name,
-        }
-
-    @router.post("/api/session/{session_id}/truncate")
-    async def truncate_session(request: Request, session_id: str):
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-            result = session_manager.truncate_messages(session_id, keep_count)
-            return {"status": "ok", "kept": keep_count, "truncated": result}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Truncate error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/message")
-    async def add_message(request: Request, session_id: str):
-        """Add a message to a session (for slash command persistence)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            role = body.get("role", "assistant")
-            content = body.get("content", "")
-            if not content:
-                raise HTTPException(400, "content is required")
-            msg = ChatMessage(role=role, content=content, metadata=body.get("metadata"))
-            session_manager.add_message(session_id, msg)
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-
-    @router.post("/api/session/{session_id}/delete-messages")
-    async def delete_messages(request: Request, session_id: str):
-        """Delete specific messages by DB ID (or legacy index)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_ids = body.get("msg_ids", [])
-            indices = body.get("indices")  # legacy fallback
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                if msg_ids:
-                    # New ID-based delete
-                    deleted = 0
-                    for mid in msg_ids:
-                        db_msg = db.query(DbChatMessage).filter(
-                            DbChatMessage.id == mid,
-                            DbChatMessage.session_id == session_id,
-                        ).first()
-                        if db_msg:
-                            db.delete(db_msg)
-                            deleted += 1
-
-                    # Remove from in-memory history by matching _db_id
-                    def _get_db_id(m):
-                        meta = m.metadata if isinstance(m, ChatMessage) else (m.get('metadata') if isinstance(m, dict) else None)
-                        return meta.get('_db_id') if isinstance(meta, dict) else None
-                    session.history = [m for m in session.history if _get_db_id(m) not in msg_ids]
-                elif indices:
-                    # Legacy index-based delete
-                    indices = sorted(indices, reverse=True)
-                    db_messages = db.query(DbChatMessage).filter(
-                        DbChatMessage.session_id == session_id
-                    ).order_by(DbChatMessage.timestamp).all()
-
-                    deleted = 0
-                    for idx in indices:
-                        if 0 <= idx < len(db_messages):
-                            db.delete(db_messages[idx])
-                            deleted += 1
-                        if 0 <= idx < len(session.history):
-                            session.history.pop(idx)
-                else:
-                    return {"status": "ok", "deleted": 0}
-
-                session.message_count = len(session.history)
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    from datetime import datetime, timezone
-                    db_session.updated_at = datetime.now(timezone.utc)
-
-                db.commit()
-                return {"status": "ok", "deleted": deleted}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Delete messages error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/edit-message")
-    async def edit_message(request: Request, session_id: str):
-        """Edit the content of a message by its database ID."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            msg_id = body.get("msg_id")
-            content = body.get("content")
-            if not msg_id or content is None:
-                raise HTTPException(400, "msg_id and content are required")
-
-            session = session_manager.get_session(session_id)
-            db = SessionLocal()
-            try:
-                db_msg = db.query(DbChatMessage).filter(
-                    DbChatMessage.id == msg_id,
-                    DbChatMessage.session_id == session_id,
-                ).first()
-                if not db_msg:
-                    raise HTTPException(404, "Message not found")
-
-                db_msg.content = content
-                meta = {}
-                if db_msg.meta_data:
-                    try: meta = json.loads(db_msg.meta_data)
-                    except (json.JSONDecodeError, ValueError): pass
-                meta['edited'] = True
-                db_msg.meta_data = json.dumps(meta)
-
-                # Update in-memory history by matching _db_id
-                for hmsg in session.history:
-                    hmeta = hmsg.metadata if isinstance(hmsg, ChatMessage) else hmsg.get('metadata')
-                    if isinstance(hmeta, dict) and hmeta.get('_db_id') == msg_id:
-                        if isinstance(hmsg, ChatMessage):
-                            hmsg.content = content
-                            hmsg.metadata['edited'] = True
-                        elif isinstance(hmsg, dict):
-                            hmsg['content'] = content
-                            hmsg['metadata']['edited'] = True
-                        break
-
-                db.commit()
-                return {"status": "ok"}
-            finally:
-                db.close()
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Edit message error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/mark-stopped")
-    async def mark_stopped(request: Request, session_id: str):
-        """Mark the last assistant message as stopped by user."""
-        _verify_session_owner(request, session_id)
-        try:
-            session = session_manager.get_session(session_id)
-            # Find last assistant message and add stopped metadata
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata['stopped'] = True
-                        if not msg.metadata.get('model'):
-                            msg.metadata['model'] = session.model
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata']['stopped'] = True
-                        if not msg['metadata'].get('model'):
-                            msg['metadata']['model'] = session.model
-                    break
-            # Also update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.created_at.desc())
-                    .first()
-                )
-                if db_messages:
-                    meta = {}
-                    if db_messages.meta_data:
-                        try:
-                            meta = _json.loads(db_messages.meta_data)
-                        except (json.JSONDecodeError, ValueError):
-                            pass
-                    meta['stopped'] = True
-                    if not meta.get('model'):
-                        meta['model'] = session.model
-                    db_messages.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Mark stopped error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/update-last-meta")
-    async def update_last_meta(request: Request, session_id: str):
-        """Merge metadata into the last assistant message (e.g. save variants)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            meta_update = body.get("metadata", {})
-            session = session_manager.get_session(session_id)
-
-            # Update in-memory
-            for msg in reversed(session.history):
-                if (isinstance(msg, ChatMessage) and msg.role == 'assistant') or \
-                   (isinstance(msg, dict) and msg.get('role') == 'assistant'):
-                    if isinstance(msg, ChatMessage):
-                        if not msg.metadata:
-                            msg.metadata = {}
-                        msg.metadata.update(meta_update)
-                    else:
-                        if 'metadata' not in msg:
-                            msg['metadata'] = {}
-                        msg['metadata'].update(meta_update)
-                    break
-
-            # Update in DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_msg = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id, DbChatMessage.role == 'assistant')
-                    .order_by(DbChatMessage.created_at.desc())
-                    .first()
-                )
-                if db_msg:
-                    meta = {}
-                    if db_msg.meta_data:
-                        try: meta = _json.loads(db_msg.meta_data)
-                        except (json.JSONDecodeError, ValueError): pass
-                    meta.update(meta_update)
-                    db_msg.meta_data = _json.dumps(meta)
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok"}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Update last meta error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/merge-last-assistant")
-    async def merge_last_assistant(request: Request, session_id: str):
-        """Merge the last two assistant messages into one (for continue)."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            separator = body.get("separator", "\n\n")
-            session = session_manager.get_session(session_id)
-
-            # Find last two assistant messages in-memory
-            ai_indices = []
-            for i, msg in enumerate(session.history):
-                role = msg.role if isinstance(msg, ChatMessage) else msg.get('role', '')
-                if role == 'assistant':
-                    ai_indices.append(i)
-
-            if len(ai_indices) < 2:
-                return {"status": "ok", "merged": False}
-
-            idx1, idx2 = ai_indices[-2], ai_indices[-1]
-            msg1, msg2 = session.history[idx1], session.history[idx2]
-
-            content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
-            content2 = msg2.content if isinstance(msg2, ChatMessage) else msg2.get('content', '')
-            merged_content = content1 + separator + content2
-
-            # Merge metadata
-            meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
-            meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
-            merged_meta = {**meta1, **meta2}
-            merged_meta.pop('stopped', None)  # no longer stopped after continue
-
-            # Update first message, remove second
-            if isinstance(msg1, ChatMessage):
-                msg1.content = merged_content
-                msg1.metadata = merged_meta
-            else:
-                msg1['content'] = merged_content
-                msg1['metadata'] = merged_meta
-
-            # Also remove the hidden "continue" user message between them if present
-            # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
-
-            for ri in sorted(remove_indices, reverse=True):
-                session.history.pop(ri)
-
-            # Update DB
-            db = SessionLocal()
-            try:
-                import json as _json
-                db_messages = (
-                    db.query(DbChatMessage)
-                    .filter(DbChatMessage.session_id == session_id)
-                    .order_by(DbChatMessage.created_at)
-                    .all()
-                )
-                # Find last two assistant messages in DB
-                ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
-                if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
-
-                    # Remove the continue user message if between them
-                    db_idx2 = db_messages.index(db2)
-                    db_idx1 = db_messages.index(db1)
-                    for di in range(db_idx2, db_idx1, -1):
-                        db.delete(db_messages[di])
-
-                    db.commit()
-            finally:
-                db.close()
-            session_manager.save_sessions()
-            return {"status": "ok", "merged": True}
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-        except Exception as e:
-            logger.error(f"Merge assistant error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.post("/api/session/{session_id}/fork")
-    async def fork_session(request: Request, session_id: str):
-        """Create a new session with messages copied up to keep_count."""
-        _verify_session_owner(request, session_id)
-        try:
-            body = await request.json()
-            keep_count = body.get("keep_count", 0)
-
-            # Get the source session
-            source = session_manager.sessions.get(session_id)
-            if not source:
-                raise HTTPException(404, "Session not found")
-
-            # Create new session
-            new_id = str(uuid.uuid4())
-            fork_name = f"\u2ADD {source.name}"
-            new_session = session_manager.create_session(
-                session_id=new_id,
-                name=fork_name,
-                endpoint_url=source.endpoint_url,
-                model=source.model,
-                rag=False,
-                owner=getattr(source, 'owner', None),
-            )
-
-            # Copy messages up to keep_count
-            msgs_to_copy = source.history[:keep_count]
-            for msg in msgs_to_copy:
-                new_session.add_message(ChatMessage(msg.role, msg.content, msg.metadata))
-            try:
-                from src.event_bus import fire_event
-                fire_event("session_created", getattr(source, 'owner', None))
-            except Exception:
-                logger.debug("session_created event dispatch failed", exc_info=True)
-
-            return {
-                "status": "ok",
-                "id": new_id,
-                "name": fork_name,
-                "kept": len(msgs_to_copy),
-            }
-        except HTTPException:
-            raise
-        except Exception as e:
-            logger.error(f"Fork error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    @router.get("/api/conversations/topics")
-    async def get_conversation_topics(request: Request) -> Dict[str, Any]:
-        from src.auth_helpers import get_current_user
-        user = get_current_user(request)
-        try:
-            return analyze_topics(session_manager, owner=user)
-        except Exception as e:
-            raise HTTPException(500, f"Topic analysis failed: {e}")
-
-    @router.post("/api/session/{session_id}/compact")
-    async def compact_session(request: Request, session_id: str):
-        """Manually trigger context compaction for a session."""
-        _verify_session_owner(request, session_id)
-        try:
-            session = session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-
-        try:
-            from src.model_context import estimate_tokens, get_context_length
-            from src.llm_core import llm_call_async
-            from src.endpoint_resolver import resolve_endpoint
-
-            if len(session.history) < 6:
-                return {"status": "ok", "message": "Not enough messages to compact"}
-
-            ctx_len = get_context_length(session.endpoint_url, session.model)
-            messages_before = session.get_context_messages()
-            used_before = estimate_tokens(messages_before)
-            pct_before = round((used_before / ctx_len) * 100, 1) if ctx_len else 0
-            msg_count_before = len(session.history)
-
-            # Keep only last 4 messages, summarize the rest
-            keep_count = 4
-            older = session.history[:-keep_count]
-            recent = session.history[-keep_count:]
-
-            # Build text to summarize
-            convo_text = "\n".join(
-                f"{(m.role if isinstance(m, ChatMessage) else m.get('role', '')).upper()}: "
-                f"{(m.content if isinstance(m, ChatMessage) else m.get('content', ''))[:2000]}"
-                for m in older
-            )
-
-            # Use utility model if available
-            util_url, util_model, util_headers = resolve_endpoint("utility")
-            compact_url = util_url or session.endpoint_url
-            compact_model = util_model or session.model
-            compact_headers = util_headers if util_url else session.headers
-
-            from src.context_compactor import SELF_SUMMARY_SYSTEM_PROMPT
-            compaction_count = sum(1 for m in session.history if isinstance(m, ChatMessage) and "[Conversation summary" in (m.content or ""))
-            sys_prompt = SELF_SUMMARY_SYSTEM_PROMPT.replace("{count}", str(len(older))).replace("{n}", str(compaction_count + 1))
-            summary = await llm_call_async(
-                compact_url, compact_model,
-                [
-                    {"role": "system", "content": sys_prompt},
-                    {"role": "user", "content": convo_text},
-                ],
-                temperature=0.2, max_tokens=1024,
-                headers=compact_headers, timeout=30,
-            )
-
-            # Replace session history: summary as system message + recent messages
-            # System message holds the full summary for AI context
-            system_summary = ChatMessage(
-                role="system",
-                content=f"[Conversation summary — {len(older)} earlier messages were compacted]\n\n{summary}",
-                metadata={"compacted": True, "hidden": True},
-            )
-            # Visible assistant message just shows stats
-            summary_msg = ChatMessage(
-                role="assistant",
-                content=f"**Conversation compacted** — {len(older)} messages summarized, {len(recent)} kept.",
-                metadata={"compacted": True, "messages_removed": len(older)},
-            )
-            new_history = [system_summary, summary_msg] + list(recent)
-            session.history = new_history
-            session.message_count = len(session.history)
-            logger.info(f"Compact: session {session_id} history now has {len(session.history)} messages (was {msg_count_before})")
-
-            # Update DB: delete old messages, insert summary
-            db = SessionLocal()
-            try:
-                db_msgs = db.query(DbChatMessage).filter(
-                    DbChatMessage.session_id == session_id
-                ).order_by(DbChatMessage.timestamp).all()
-
-                # Delete all but the last keep_count
-                for m in db_msgs[:-keep_count]:
-                    db.delete(m)
-
-                # Insert system summary (hidden, for AI context) and visible summary
-                import json as _json
-                import uuid
-                from datetime import datetime, timezone
-                now = datetime.now(timezone.utc)
-                db_sys_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="system",
-                    content=system_summary.content,
-                    meta_data=_json.dumps(system_summary.metadata),
-                    timestamp=now,
-                )
-                db.add(db_sys_summary)
-                db_summary = DbChatMessage(
-                    id=str(uuid.uuid4()),
-                    session_id=session_id,
-                    role="assistant",
-                    content=summary_msg.content,
-                    meta_data=_json.dumps(summary_msg.metadata),
-                    timestamp=now,
-                )
-                db.add(db_summary)
-
-                # Update session record
-                db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-                if db_session:
-                    db_session.message_count = len(session.history)
-                    db_session.updated_at = datetime.now(timezone.utc)
-                db.commit()
-            finally:
-                db.close()
-
-            session_manager.save_sessions()
-
-            used_after = estimate_tokens(session.get_context_messages())
-            pct_after = round((used_after / ctx_len) * 100, 1) if ctx_len else 0
-
-            return {
-                "status": "ok",
-                "message": f"Compacted: {msg_count_before} msgs → {len(session.history)} msgs ({pct_before}% → {pct_after}%)",
-                "before": pct_before,
-                "after": pct_after,
-            }
-
-        except Exception as e:
-            logger.error(f"Manual compact error {session_id}: {e}")
-            raise HTTPException(500, str(e))
-
-    return router
+_sys.modules[__name__] = _canonical

routes/hwfit_routes.py

@@ -1,96 +1,215 @@
+import json
+import os
+import re
+import shlex
+import subprocess
 from copy import deepcopy
 
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
+
+from core.platform_compat import run_ssh_command
+from routes._validators import validate_remote_host, validate_ssh_port
+
+
+# Backends the manual hardware simulator accepts. Must stay a subset of what
+# services.hwfit.fit understands so a simulated box ranks like a real one:
+# "metal" routes through the Apple-Silicon path (GGUF-only, llama.cpp/Ollama),
+# the CPU backends through the RAM/offload path, cuda/rocm through vLLM.
+_MANUAL_BACKENDS = {"cuda", "rocm", "metal", "cpu_x86", "cpu_arm"}
+_OFFICIAL_NAMESPACES = {
+    "apple", "allenai", "black-forest-labs", "cohere", "deepseek-ai",
+    "google", "ibm", "lightricks", "meta-llama", "microsoft", "mistralai",
+    "nvidia", "openai", "qwen", "stabilityai", "tencent", "tiiuae",
+    "upstage", "zai-org", "runwayml",
+}
+
+
+def _is_official_model(model: dict) -> bool:
+    """Recognize first-party namespaces without maintaining model-name lists."""
+    # Image rows expose a friendly `name` without its namespace, while regular
+    # rows may use `name`. Prefer whichever field still contains `owner/repo`.
+    model_id = str(model.get("id") or model.get("name") or "")
+    namespace = model_id.split("/", 1)[0].strip().lower() if "/" in model_id else ""
+    # `provider` is a display label for image rows (for example, "Stability AI")
+    # and is not a stable repository namespace. The model id is the canonical
+    # source for this filter, so a recognized namespace is sufficient.
+    return namespace in _OFFICIAL_NAMESPACES
+
+
+def _validate_detection_target(host: str = "", ssh_port: str = "") -> tuple[str, str]:
+    host_value = validate_remote_host(host) or ""
+    port_value = validate_ssh_port(ssh_port) or ""
+    if port_value and not host_value:
+        raise HTTPException(400, "ssh_port requires host")
+    return host_value, port_value
+
+
+def _apply_manual_hardware(system, manual_mode="", manual_gpu_count="", manual_vram_gb="", manual_ram_gb="", manual_backend=""):
+    """Manual hardware is a "what if I had this setup" simulator —
+    REPLACES the detected hardware entirely instead of adding to it.
+
+    The previous additive behavior averaged the manual VRAM across
+    all GPUs (base + manual), which meant adding "1× 400 GB" on top
+    of "2× 70 GB" only nudged the per-GPU cap from 70 to 180 GB
+    (= 540 / 3), so GGUF models bigger than that still didn't surface
+    — exactly the "cap stuck at detected level" bug the user hit.
+    """
+    manual_mode = (manual_mode or "").lower()
+    if manual_mode not in {"gpu", "ram"}:
+        return system
+
+    try:
+        override_ram_gb = float(manual_ram_gb) if manual_ram_gb else 0
+    except ValueError:
+        override_ram_gb = 0
+    override_ram_gb = max(0.0, override_ram_gb)
+    if override_ram_gb:
+        # Replace RAM, don't add. The number in the field is the
+        # TOTAL system memory the user wants to simulate.
+        system["available_ram_gb"] = round(override_ram_gb, 1)
+        system["total_ram_gb"] = round(override_ram_gb, 1)
+    system["manual_hardware"] = True
+
+    if manual_mode == "ram":
+        # RAM-only simulation — wipe GPU entirely so the ranker uses
+        # CPU/RAM paths.
+        system["has_gpu"] = False
+        system["gpu_name"] = None
+        system["gpu_vram_gb"] = 0
+        system["gpu_count"] = 0
+        system["gpus"] = []
+        system["gpu_groups"] = []
+        system["backend"] = "cpu_x86"
+        system.pop("unified_memory", None)
+        return system
+
+    try:
+        count = int(manual_gpu_count) if manual_gpu_count else 1
+    except ValueError:
+        count = 1
+    try:
+        vram_each = float(manual_vram_gb) if manual_vram_gb else 8.0
+    except ValueError:
+        vram_each = 8.0
+    count = max(1, min(count, 16))
+    vram_each = max(1.0, vram_each)
+    backend = (manual_backend or system.get("backend") or "cuda").lower()
+    if backend not in _MANUAL_BACKENDS:
+        backend = "cuda"
+    total_vram = round(vram_each * count, 1)
+    gpu_name = f"Simulated {backend.upper()} GPU" + (f" × {count}" if count > 1 else "")
+    system["has_gpu"] = True
+    system["gpu_name"] = gpu_name
+    system["gpu_vram_gb"] = total_vram
+    system["gpu_count"] = count
+    system["gpus"] = [
+        {"index": i, "name": gpu_name, "vram_gb": vram_each}
+        for i in range(count)
+    ]
+    # Single homogeneous pool — vram_each here is the ACTUAL per-GPU
+    # VRAM the user entered, not an average. That's the whole point:
+    # raising vram_each lifts the per-GPU cap (GGUF, tensor-parallel
+    # math) all the way up, not just by a small fraction.
+    system["gpu_groups"] = [{
+        "name": gpu_name,
+        "vram_each": vram_each,
+        "count": count,
+        "indices": list(range(count)),
+        "vram_total": total_vram,
+    }]
+    system["homogeneous"] = True
+    system["backend"] = backend
+    # Apple Silicon shares one unified memory pool with the GPU; flag it so
+    # the API/UI report it the way real Metal detection does. Discrete GPUs
+    # (cuda/rocm) and the CPU backends carry separate VRAM, so clear any
+    # stale flag a previous detection left on the dict.
+    if backend == "metal":
+        system["unified_memory"] = True
+    else:
+        system.pop("unified_memory", None)
+    return system
+
+
+def _run_model_probe(host: str, ssh_port: str, cmd: str) -> str:
+    try:
+        if host:
+            r = run_ssh_command(
+                host,
+                ssh_port or None,
+                cmd,
+                timeout=15,
+                connect_timeout=5,
+                strict_host_key_checking=False,
+                text=True,
+            )
+        else:
+            r = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True, timeout=15)
+        if r.returncode == 0:
+            return (r.stdout or "").strip()
+    except Exception:
+        return ""
+    return ""
+
+
+def _inspect_model_path(model_path: str, host: str = "", ssh_port: str = "") -> dict:
+    """Read lightweight metadata from a local or SSH-visible HF model folder."""
+    path = (model_path or "").strip()
+    if not path or path.startswith(("http://", "https://")):
+        return {}
+    if not (path.startswith("/") or path.startswith("~")):
+        return {}
+
+    qpath = shlex.quote(path)
+    qconfig = shlex.quote(os.path.join(path, "config.json"))
+    out = {}
+    exists = _run_model_probe(host, ssh_port, f"test -d {qpath} && printf found || printf missing")
+    if exists != "found":
+        target = host or "local container"
+        out["model_probe_error"] = f"Model path is not visible on {target}: {path}"
+        return out
+    raw_config = _run_model_probe(host, ssh_port, f"test -f {qconfig} && sed -n '1,240p' {qconfig}")
+    if raw_config:
+        try:
+            cfg = json.loads(raw_config)
+        except Exception:
+            cfg = {}
+        for key in ("context_length", "max_position_embeddings", "n_ctx_train", "model_max_length", "max_seq_len"):
+            value = cfg.get(key)
+            if isinstance(value, (int, float)) and value > 0:
+                out["model_ctx_max"] = int(value)
+                break
+    else:
+        out["model_probe_error"] = f"config.json not found in model path: {path}"
+
+    size_cmd = (
+        f"find {qpath} -type f \\( -name '*.safetensors' -o -name '*.bin' -o -name '*.gguf' \\) "
+        "-printf '%s\\n' 2>/dev/null | awk '{s+=$1} END {if (s>0) printf \"%.6f\", s/1073741824}'"
+    )
+    weights = _run_model_probe(host, ssh_port, size_cmd)
+    try:
+        weights_gb = float(weights)
+    except Exception:
+        weights_gb = 0.0
+    if weights_gb > 0:
+        out["model_weights_gb"] = round(weights_gb, 3)
+    elif "model_probe_error" not in out:
+        out["model_probe_error"] = f"No model weight files found in: {path}"
+    return out
 
 
 def setup_hwfit_routes():
     router = APIRouter(prefix="/api/hwfit", tags=["hwfit"])
 
-    def _apply_manual_hardware(system, manual_mode="", manual_gpu_count="", manual_vram_gb="", manual_ram_gb="", manual_backend=""):
-        """Manual hardware is a "what if I had this setup" simulator —
-        REPLACES the detected hardware entirely instead of adding to it.
-
-        The previous additive behavior averaged the manual VRAM across
-        all GPUs (base + manual), which meant adding "1× 400 GB" on top
-        of "2× 70 GB" only nudged the per-GPU cap from 70 to 180 GB
-        (= 540 / 3), so GGUF models bigger than that still didn't surface
-        — exactly the "cap stuck at detected level" bug the user hit.
-        """
-        manual_mode = (manual_mode or "").lower()
-        if manual_mode not in {"gpu", "ram"}:
-            return system
-
-        try:
-            override_ram_gb = float(manual_ram_gb) if manual_ram_gb else 0
-        except ValueError:
-            override_ram_gb = 0
-        override_ram_gb = max(0.0, override_ram_gb)
-        if override_ram_gb:
-            # Replace RAM, don't add. The number in the field is the
-            # TOTAL system memory the user wants to simulate.
-            system["available_ram_gb"] = round(override_ram_gb, 1)
-            system["total_ram_gb"] = round(override_ram_gb, 1)
-        system["manual_hardware"] = True
-
-        if manual_mode == "ram":
-            # RAM-only simulation — wipe GPU entirely so the ranker uses
-            # CPU/RAM paths.
-            system["has_gpu"] = False
-            system["gpu_name"] = None
-            system["gpu_vram_gb"] = 0
-            system["gpu_count"] = 0
-            system["gpus"] = []
-            system["gpu_groups"] = []
-            system["backend"] = "cpu_x86"
-            return system
-
-        try:
-            count = int(manual_gpu_count) if manual_gpu_count else 1
-        except ValueError:
-            count = 1
-        try:
-            vram_each = float(manual_vram_gb) if manual_vram_gb else 8.0
-        except ValueError:
-            vram_each = 8.0
-        count = max(1, min(count, 16))
-        vram_each = max(1.0, vram_each)
-        backend = (manual_backend or system.get("backend") or "cuda").lower()
-        if backend not in {"cuda", "rocm", "cpu_x86", "cpu_arm"}:
-            backend = "cuda"
-        total_vram = round(vram_each * count, 1)
-        gpu_name = f"Simulated {backend.upper()} GPU" + (f" × {count}" if count > 1 else "")
-        system["has_gpu"] = True
-        system["gpu_name"] = gpu_name
-        system["gpu_vram_gb"] = total_vram
-        system["gpu_count"] = count
-        system["gpus"] = [
-            {"index": i, "name": gpu_name, "vram_gb": vram_each}
-            for i in range(count)
-        ]
-        # Single homogeneous pool — vram_each here is the ACTUAL per-GPU
-        # VRAM the user entered, not an average. That's the whole point:
-        # raising vram_each lifts the per-GPU cap (GGUF, tensor-parallel
-        # math) all the way up, not just by a small fraction.
-        system["gpu_groups"] = [{
-            "name": gpu_name,
-            "vram_each": vram_each,
-            "count": count,
-            "indices": list(range(count)),
-            "vram_total": total_vram,
-        }]
-        system["homogeneous"] = True
-        system["backend"] = backend
-        return system
-
     @router.get("/system")
     def get_system(host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False):
         """Detect and return current system hardware info. Pass host=user@server for remote.
         fresh=true bypasses the per-host cache (the Rescan button)."""
         from services.hwfit.hardware import detect_system
+        host, ssh_port = _validate_detection_target(host, ssh_port)
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "score", limit: int = 50, search: str = "", host: str = "", quant: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False, official_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -99,10 +218,17 @@ def setup_hwfit_routes():
         fresh=true bypasses the hardware-detection cache."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.fit import rank_models
-        from services.hwfit.models import get_models, model_catalog_path
+        from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
+        host, ssh_port = _validate_detection_target(host, ssh_port)
         system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
         if system.get("error"):
             return {"system": system, "models": [], "error": system["error"]}
+        catalog_refresh = None
+        if refresh_catalog:
+            try:
+                catalog_refresh = refresh_dynamic_catalogs(force=True)
+            except Exception as e:
+                catalog_refresh = {"error": str(e)}
         if not get_models():
             return {
                 "system": system,
@@ -147,8 +273,14 @@ def setup_hwfit_routes():
             system["gpu_name"] = g["name"]
             system["active_group"] = {**g, "use_count": n}
 
-        if gpu_count != "":
-            n = int(gpu_count)
+        # Parse the optional count defensively (matches the gpu_group guard
+        # above): a non-numeric query param previously raised ValueError ->
+        # HTTP 500. A malformed value is ignored, same as omitting it.
+        try:
+            n = int(gpu_count) if gpu_count != "" else None
+        except ValueError:
+            n = None
+        if n is not None:
             if n == 0:
                 # RAM-only mode: rank against system memory, offload allowed.
                 system["has_gpu"] = False
@@ -171,14 +303,138 @@ def setup_hwfit_routes():
             # gpu_only stays off here so the default view still surfaces offload.
             _apply_group(grp, grp["count"])
 
-        results = rank_models(system, use_case=use_case or None, limit=limit, search=search or None, sort=sort, quant=quant or None)
-        return {"system": system, "models": results}
+        try:
+            target_context = int(ctx) if ctx else None
+        except ValueError:
+            target_context = None
+        if target_context is not None:
+            target_context = max(1024, min(target_context, 1000000))
+
+        rank_kwargs = {
+            "use_case": use_case or None,
+            "limit": limit,
+            "search": search or None,
+            "sort": sort,
+            "quant": quant or None,
+            "fit_only": fit_only,
+        }
+        if target_context is not None:
+            rank_kwargs["target_context"] = target_context
+        try:
+            import inspect
+            supported = set(inspect.signature(rank_models).parameters)
+            rank_kwargs = {k: v for k, v in rank_kwargs.items() if k in supported}
+        except Exception:
+            rank_kwargs.pop("target_context", None)
+            rank_kwargs.pop("fit_only", None)
+        results = rank_models(system, **rank_kwargs)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
+        payload = {"system": system, "models": results}
+        if catalog_refresh is not None:
+            payload["catalog_refresh"] = catalog_refresh
+        return payload
+
+    @router.get("/profiles")
+    def get_serve_profiles(model: str = "", model_path: str = "", host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, serve_weights_gb: float = 0.0, serve_quant: str = ""):
+        """Compute llama.cpp serve profiles (Quality/Balanced/Speed) for `model`
+        against the detected hardware on `host` (or local). Returns concrete
+        flags (n_gpu_layers, n_cpu_moe, cache_type, ctx) the serve UI can apply.
+
+        `model` is matched against the catalog by name; if it's not in the
+        catalog (e.g. an ad-hoc HF repo), pass enough hints via a minimal synthetic
+        entry isn't possible here, so we return [] and the UI keeps manual flags.
+        """
+        from services.hwfit.hardware import detect_system
+        from services.hwfit.models import get_models
+        from services.hwfit.profiles import compute_serve_profiles
+        host, ssh_port = _validate_detection_target(host, ssh_port)
+        system = detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
+        if system.get("error"):
+            return {"system": system, "profiles": [], "error": system["error"]}
+        catalog = {m.get("name"): m for m in (get_models() or [])}
+
+        def _norm(s):
+            # Normalize for matching: drop org/ prefix, a trailing -GGUF/-gguf
+            # marker, and any quant tag, lowercase. So "DeepSeek-Coder-V2-Lite-
+            # Instruct-GGUF" (a local folder name) matches catalog entry
+            # "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct".
+            s = (s or "").lower().strip()
+            s = s.split("/")[-1]                     # drop org prefix
+            for suffix in ("-gguf", "_gguf", ".gguf", "gguf"):
+                if s.endswith(suffix):
+                    s = s[: -len(suffix)]
+                    break
+            cut_at = None
+            for idx, ch in enumerate(s):
+                if ch not in "-_." or idx + 1 >= len(s):
+                    continue
+                suffix = s[idx + 1:]
+                if (
+                    suffix in {"fp8", "bf16", "f16"}
+                    or suffix.startswith(("awq", "gptq", "iq"))
+                    or (suffix.startswith("q") and len(suffix) > 1 and suffix[1].isdigit())
+                ):
+                    cut_at = idx
+            if cut_at is not None:
+                s = s[:cut_at]
+            return s
+
+        m = catalog.get(model)
+        if m is None and model:
+            want = _norm(model)
+            for name, entry in catalog.items():
+                nn = _norm(name)
+                if nn and (nn == want or want.endswith(nn) or nn.endswith(want)):
+                    m = entry
+                    break
+        path_meta = _inspect_model_path(model_path or model, host=host, ssh_port=ssh_port)
+        if m is None:
+            return {
+                "system": system,
+                "profiles": [],
+                "error": "model not in catalog",
+                "model_ctx_max": int(path_meta.get("model_ctx_max") or 0),
+                "model_weights_gb": float(path_meta.get("model_weights_gb") or 0),
+                "model_probe_error": path_meta.get("model_probe_error") or "",
+            }
+        # Surface the model's trained context limit so the serve UI can clamp a
+        # user-typed context down to it (asking for ctx > n_ctx_train overflows
+        # and, with a quantized KV cache, can crash the GPU).
+        model_ctx_max = 0
+        for k in ("context_length", "max_position_embeddings", "n_ctx_train", "context"):
+            v = m.get(k)
+            if isinstance(v, (int, float)) and v > 0:
+                model_ctx_max = int(v)
+                break
+        path_ctx_max = int(path_meta.get("model_ctx_max") or 0)
+        if path_ctx_max > 0:
+            model_ctx_max = max(model_ctx_max, path_ctx_max)
+        model_weights_gb = float(path_meta.get("model_weights_gb") or 0)
+        if model_weights_gb <= 0:
+            for k in ("min_vram_gb", "required_gb", "size_gb", "recommended_ram_gb", "min_ram_gb"):
+                v = m.get(k)
+                if isinstance(v, (int, float)) and v > 0:
+                    model_weights_gb = float(v)
+                    break
+        return {
+            "system": system,
+            "profiles": compute_serve_profiles(
+                system, m,
+                serve_weights_gb=(serve_weights_gb or None),
+                serve_quant=(serve_quant or None),
+            ),
+            "model_ctx_max": model_ctx_max,
+            "model_weights_gb": model_weights_gb,
+            "model_probe_error": path_meta.get("model_probe_error") or "",
+        }
 
     @router.get("/image-models")
-    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False):
+    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, official_only: bool = False):
         """Rank image generation models against detected hardware."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.image_models import rank_image_models
+        host, ssh_port = _validate_detection_target(host, ssh_port)
         system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
         if system.get("error"):
             return {"system": system, "models": [], "error": system["error"]}
@@ -193,12 +449,30 @@ def setup_hwfit_routes():
             system["available_ram_gb"] = 0
             system["total_ram_gb"] = 0
         system = _apply_manual_hardware(system, manual_mode, manual_gpu_count, manual_vram_gb, manual_ram_gb, manual_backend)
-        # Image models use a single GPU — always use per-GPU VRAM
-        gpu_vrams = [float(g.get("vram_gb") or 0) for g in (system.get("gpus") or []) if isinstance(g, dict)]
-        single_vram = max(gpu_vrams) if gpu_vrams else ((system.get("gpu_vram_gb") or 0) / max(system.get("gpu_count") or 1, 1))
-        system["gpu_vram_gb"] = single_vram
-        system["gpu_count"] = 1 if single_vram > 0 else 0
+        try:
+            requested_gpu_count = int(gpu_count) if gpu_count != "" else None
+        except ValueError:
+            requested_gpu_count = None
+        if requested_gpu_count == 0:
+            # Respect the UI's RAM toggle. Before this route always rewrote the
+            # system to best-single-GPU VRAM, so image rows never changed when
+            # switching RAM/GPU.
+            system["has_gpu"] = False
+            system["gpu_vram_gb"] = 0
+            system["gpu_count"] = 0
+            system["gpu_only"] = False
+        else:
+            # Image diffusion backends generally use one device per pipeline,
+            # so rank GPU mode against the best single GPU rather than total
+            # multi-GPU VRAM.
+            gpu_vrams = [float(g.get("vram_gb") or 0) for g in (system.get("gpus") or []) if isinstance(g, dict)]
+            single_vram = max(gpu_vrams) if gpu_vrams else ((system.get("gpu_vram_gb") or 0) / max(system.get("gpu_count") or 1, 1))
+            system["gpu_vram_gb"] = single_vram
+            system["gpu_count"] = 1 if single_vram > 0 else 0
+            system["gpu_only"] = True if single_vram > 0 else False
         results = rank_image_models(system, search=search or None, sort=sort)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         return {"system": system, "models": results}
 
     return router

routes/mcp/__init__.py

@@ -0,0 +1,5 @@
+"""MCP route domain package (slice 2o, #4082/#4071).
+
+Contains mcp_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/mcp_routes.py re-exports from here.
+"""

routes/mcp/mcp_routes.py

@@ -0,0 +1,703 @@
+# routes/mcp_routes.py
+"""MCP (Model Context Protocol) server management routes."""
+import json
+import os
+import uuid
+import urllib.parse
+import html
+from pathlib import Path
+from fastapi import APIRouter, Form, HTTPException, Request
+from fastapi.responses import RedirectResponse, HTMLResponse
+import logging
+import httpx
+
+from core.database import McpServer, SessionLocal
+from core.middleware import require_admin
+from src.constants import DATA_DIR, MCP_OAUTH_DIR
+from src.mcp_manager import McpManager
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/mcp", tags=["mcp"])
+
+
+def _mcp_oauth_base_dir() -> Path:
+    """Directory that may contain OAuth files managed by Odysseus."""
+    return Path(MCP_OAUTH_DIR).resolve(strict=False)
+
+
+def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str:
+    """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth."""
+    raw = str(raw_path or "").strip()
+    if not raw:
+        return ""
+
+    base = _mcp_oauth_base_dir()
+    path = Path(os.path.expanduser(raw))
+    if not path.is_absolute():
+        path = base / path
+    resolved = path.resolve(strict=False)
+
+    try:
+        resolved.relative_to(base)
+    except ValueError as exc:
+        raise HTTPException(
+            400,
+            f"Invalid OAuth {field_name}: path must stay under {base}",
+        ) from exc
+    return str(resolved)
+
+
+def _sanitize_mcp_oauth_config(oauth_cfg):
+    """Return an OAuth config copy with file paths confined to mcp_oauth."""
+    if not oauth_cfg:
+        return oauth_cfg
+    if not isinstance(oauth_cfg, dict):
+        return {}
+    sanitized = dict(oauth_cfg)
+    for field_name in ("keys_file", "token_file"):
+        if sanitized.get(field_name):
+            sanitized[field_name] = _resolve_mcp_oauth_path(
+                sanitized[field_name],
+                field_name,
+            )
+    return sanitized
+
+
+def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool:
+    """Check token existence without letting legacy bad paths break listing."""
+    if not isinstance(oauth_cfg, dict):
+        return False
+    try:
+        token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file")
+    except HTTPException:
+        if strict:
+            raise
+        logger.warning("Ignoring MCP OAuth config with unsafe token_file")
+        return True
+    return bool(token_file and not os.path.exists(token_file))
+
+
+def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None:
+    """Pass sanitized Gmail package paths to MCP servers that honor them."""
+    if not oauth_cfg or not isinstance(env, dict):
+        return
+    keys_file = oauth_cfg.get("keys_file")
+    token_file = oauth_cfg.get("token_file")
+    if keys_file:
+        env["GMAIL_OAUTH_PATH"] = keys_file
+    if token_file:
+        env["GMAIL_CREDENTIALS_PATH"] = token_file
+
+
+def _load_disabled_map():
+    """Load per-server disabled tool sets from DB."""
+    db = SessionLocal()
+    try:
+        disabled_map = {}
+        for srv in db.query(McpServer).all():
+            if srv.disabled_tools:
+                try:
+                    names = json.loads(srv.disabled_tools)
+                    if names:
+                        disabled_map[srv.id] = set(names)
+                except (json.JSONDecodeError, TypeError):
+                    pass
+        return disabled_map
+    finally:
+        db.close()
+
+
+def _mcp_oauth_redirect_uri() -> str:
+    """Shared callback URL for legacy Google and generic MCP OAuth flows."""
+    from src.mcp_oauth import REDIRECT_URI
+    return REDIRECT_URI
+
+
+def setup_mcp_routes(mcp_manager: McpManager):
+    """Setup MCP routes with the provided manager."""
+
+    @router.get("/servers")
+    def list_servers(request: Request):
+        """List all configured MCP servers with connection status."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            servers = db.query(McpServer).all()
+            result = []
+            for srv in servers:
+                status = mcp_manager.get_server_status(srv.id)
+                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
+                needs_oauth = False
+                if oauth_cfg:
+                    needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False)
+                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+                total_tools = status.get("tool_count", 0)
+                result.append({
+                    "id": srv.id,
+                    "name": srv.name,
+                    "transport": srv.transport,
+                    "command": srv.command,
+                    "args": json.loads(srv.args) if srv.args else [],
+                    "env": json.loads(srv.env) if srv.env else {},
+                    "url": srv.url,
+                    "is_enabled": srv.is_enabled,
+                    "status": status.get("status", "disconnected"),
+                    "tool_count": total_tools,
+                    "disabled_tool_count": len(disabled_list),
+                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
+                    "error": status.get("error"),
+                    "auth_url": status.get("auth_url"),
+                    "has_oauth": oauth_cfg is not None,
+                    "needs_oauth": needs_oauth,
+                })
+            return result
+        finally:
+            db.close()
+
+    @router.post("/servers")
+    async def add_server(
+        request: Request,
+        name: str = Form(...),
+        transport: str = Form("stdio"),
+        command: str = Form(None),
+        args: str = Form("[]"),
+        env: str = Form("{}"),
+        url: str = Form(None),
+        oauth_file: str = Form(None),
+        oauth_config: str = Form(None),
+    ):
+        """Add a new MCP server config and attempt connection. Admin-only:
+        registering a stdio server is equivalent to executing arbitrary
+        binaries on the host."""
+        require_admin(request)
+        server_id = str(uuid.uuid4())[:8]
+
+        # Validate
+        if transport == "stdio" and not command:
+            raise HTTPException(400, "command is required for stdio transport")
+        if transport == "sse" and not url:
+            raise HTTPException(400, "url is required for SSE transport")
+        if transport == "http" and not url:
+            raise HTTPException(400, "url is required for HTTP transport")
+
+        # Parse JSON fields
+        try:
+            parsed_args = json.loads(args) if args else []
+        except json.JSONDecodeError:
+            parsed_args = []
+        try:
+            parsed_env = json.loads(env) if env else {}
+        except json.JSONDecodeError:
+            parsed_env = {}
+        if not isinstance(parsed_env, dict):
+            parsed_env = {}
+
+        # Parse OAuth config
+        parsed_oauth_config = None
+        if oauth_config:
+            try:
+                parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config))
+            except json.JSONDecodeError:
+                pass
+        _apply_mcp_oauth_env(parsed_env, parsed_oauth_config)
+
+        # Write OAuth credentials file if provided (for Google MCP servers)
+        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
+        if oauth_file:
+            try:
+                oauth_data = json.loads(oauth_file)
+                oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir")
+                oauth_filename = oauth_data.get("filename", "")
+                client_id = oauth_data.get("client_id", "")
+                client_secret = oauth_data.get("client_secret", "")
+                if oauth_dir and oauth_filename and client_id and client_secret:
+                    filepath = _resolve_mcp_oauth_path(
+                        Path(oauth_dir) / str(oauth_filename),
+                        "filename",
+                    )
+                    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+                    creds = {
+                        "installed": {
+                            "client_id": client_id,
+                            "client_secret": client_secret,
+                            "redirect_uris": ["http://localhost"],
+                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                            "token_uri": "https://accounts.google.com/o/oauth2/token",
+                        }
+                    }
+                    with open(filepath, "w", encoding="utf-8") as f:
+                        json.dump(creds, f, indent=2)
+                    logger.info(f"Wrote OAuth credentials to {filepath}")
+                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
+                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
+            except (json.JSONDecodeError, OSError) as e:
+                logger.warning(f"Failed to write OAuth file: {e}")
+
+        # Save to DB
+        db = SessionLocal()
+        try:
+            srv = McpServer(
+                id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=json.dumps(parsed_args),
+                env=json.dumps(parsed_env),
+                url=url,
+                is_enabled=True,
+                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
+            )
+            db.add(srv)
+            db.commit()
+        finally:
+            db.close()
+
+        # Check if OAuth token already exists — skip connection attempt if not
+        needs_oauth = False
+        if parsed_oauth_config:
+            needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config)
+
+        connected = False
+        if not needs_oauth:
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=name,
+                transport=transport,
+                command=command,
+                args=parsed_args,
+                env=parsed_env,
+                url=url,
+            )
+
+        status = mcp_manager.get_server_status(server_id)
+        needs_auth = status.get("status") == "needs_auth"
+        return {
+            "id": server_id,
+            "name": name,
+            "connected": connected,
+            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
+            "tool_count": status.get("tool_count", 0),
+            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
+            "needs_oauth": needs_oauth,
+            "needs_auth": needs_auth,
+            "auth_url": status.get("auth_url"),
+        }
+
+    @router.post("/servers/{server_id}/reconnect")
+    async def reconnect_server(server_id: str, request: Request):
+        """Reconnect to an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            status = mcp_manager.get_server_status(server_id)
+            return {
+                "connected": connected,
+                "status": status.get("status", "disconnected"),
+                "tool_count": status.get("tool_count", 0),
+                "error": status.get("error"),
+                "auth_url": status.get("auth_url"),
+                "needs_auth": status.get("status") == "needs_auth",
+            }
+        finally:
+            db.close()
+
+    @router.patch("/servers/{server_id}")
+    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
+        """Enable or disable an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            enabled = str(is_enabled).lower() == "true"
+            srv.is_enabled = enabled
+            db.commit()
+
+            if enabled:
+                args = json.loads(srv.args) if srv.args else []
+                env = json.loads(srv.env) if srv.env else {}
+                await mcp_manager.connect_server(
+                    server_id=server_id,
+                    name=srv.name,
+                    transport=srv.transport,
+                    command=srv.command,
+                    args=args,
+                    env=env,
+                    url=srv.url,
+                )
+            else:
+                await mcp_manager.disconnect_server(server_id)
+
+            return {"id": server_id, "is_enabled": enabled}
+        finally:
+            db.close()
+
+    @router.delete("/servers/{server_id}")
+    async def delete_server(server_id: str, request: Request):
+        """Remove an MCP server."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            await mcp_manager.disconnect_server(server_id)
+
+            db.delete(srv)
+            db.commit()
+            return {"status": "deleted"}
+        finally:
+            db.close()
+
+    @router.get("/tools")
+    def list_tools(request: Request):
+        """List all discovered MCP tools across all connected servers."""
+        require_admin(request)
+        disabled_map = _load_disabled_map()
+        return mcp_manager.get_all_tools(disabled_map)
+
+    @router.get("/servers/{server_id}/tools")
+    def list_server_tools(server_id: str, request: Request):
+        """List all tools for a specific MCP server with enabled/disabled state."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
+            disabled_set = set(disabled_list)
+        finally:
+            db.close()
+
+        all_tools = mcp_manager.get_all_tools()
+        server_tools = [t for t in all_tools if t["server_id"] == server_id]
+        for t in server_tools:
+            t["is_disabled"] = t["name"] in disabled_set
+        return server_tools
+
+    @router.patch("/servers/{server_id}/tools")
+    async def update_disabled_tools(server_id: str, request: Request):
+        """Bulk update disabled tools list for a server.
+
+        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
+        """
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+
+            body = await request.json()
+            disabled = body.get("disabled", [])
+            if not isinstance(disabled, list):
+                raise HTTPException(400, "disabled must be a list of tool names")
+
+            srv.disabled_tools = json.dumps(disabled) if disabled else None
+            db.commit()
+
+            return {"id": server_id, "disabled_count": len(disabled)}
+        finally:
+            db.close()
+
+    # ── OAuth flow for Google MCP servers ──────────────────────────
+
+    @router.get("/oauth/authorize/{server_id}")
+    def oauth_authorize(server_id: str, request: Request):
+        """Show OAuth authorization page with Google sign-in link."""
+        require_admin(request)
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                raise HTTPException(404, "Server not found")
+            if not srv.oauth_config:
+                raise HTTPException(400, "Server has no OAuth config")
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            if not keys_file or not os.path.exists(keys_file):
+                raise HTTPException(400, "OAuth keys file not found")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            if not keys:
+                raise HTTPException(400, "Invalid OAuth keys file format")
+
+            client_id = keys["client_id"]
+            scopes = oauth_cfg.get("scopes", [])
+
+            # For Desktop App creds, default to localhost — the user will
+            # paste the resulting URL back if they're on a different device.
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            params = {
+                "client_id": client_id,
+                "redirect_uri": redirect_uri,
+                "response_type": "code",
+                "scope": " ".join(scopes),
+                "access_type": "offline",
+                "prompt": "consent",
+                "state": server_id,
+            }
+            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
+
+            # Determine if user is accessing from the same machine
+            host = request.headers.get("host", "")
+            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
+
+            if is_local:
+                # Same machine — just redirect, callback will work directly
+                return RedirectResponse(auth_url)
+            else:
+                # Remote device — show paste-back page
+                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, redirect_uri))
+        finally:
+            db.close()
+
+    @router.get("/oauth/callback")
+    async def oauth_callback(code: str, state: str, request: Request):
+        """Handle OAuth callback. Generic MCP OAuth flows resolve via the
+        pending-state registry; Google flows fall through to the legacy path."""
+        require_admin(request)
+        from src.mcp_oauth import resolve_pending
+        if resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+        # Legacy Google path: state is the server_id
+        return await _exchange_and_connect(state, code, request)
+
+    @router.post("/oauth/exchange/{server_id}")
+    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
+        """Manual code exchange — user pastes the callback URL from their browser."""
+        require_admin(request)
+        try:
+            parsed = urllib.parse.urlparse(callback_url)
+            params = urllib.parse.parse_qs(parsed.query)
+            code = params.get("code", [None])[0]
+            if not code:
+                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
+        except Exception:
+            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
+
+        # Generic MCP OAuth: if the pasted URL carries a state we are waiting on,
+        # resolve it directly (the background connect finishes the handshake).
+        state = params.get("state", [None])[0]
+        from src.mcp_oauth import resolve_pending
+        if state and resolve_pending(state, code):
+            return HTMLResponse(_oauth_result_page(
+                "Authorization Successful",
+                "The MCP server is connecting. You can close this window and return to Odysseus.",
+                success=True,
+            ))
+
+        return await _exchange_and_connect(server_id, code, request)
+
+    async def _exchange_and_connect(server_id: str, code: str, request: Request):
+        """Exchange auth code for tokens and connect the MCP server."""
+        db = SessionLocal()
+        try:
+            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
+            if not srv:
+                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
+            if not srv.oauth_config:
+                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
+
+            oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config))
+            keys_file = oauth_cfg.get("keys_file", "")
+            token_file = oauth_cfg.get("token_file", "")
+            if not keys_file or not token_file:
+                raise HTTPException(400, "OAuth keys/token file not configured")
+
+            with open(keys_file, encoding="utf-8") as f:
+                keys_data = json.load(f)
+            keys = keys_data.get("installed") or keys_data.get("web")
+            client_id = keys["client_id"]
+            client_secret = keys["client_secret"]
+
+            redirect_uri = _mcp_oauth_redirect_uri()
+
+            async with httpx.AsyncClient() as client:
+                resp = await client.post(
+                    "https://oauth2.googleapis.com/token",
+                    data={
+                        "code": code,
+                        "client_id": client_id,
+                        "client_secret": client_secret,
+                        "redirect_uri": redirect_uri,
+                        "grant_type": "authorization_code",
+                    },
+                )
+
+            if resp.status_code != 200:
+                err = resp.text
+                logger.error(f"OAuth token exchange failed: {err}")
+                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
+
+            tokens = resp.json()
+            logger.info(f"OAuth tokens received for server {server_id}")
+
+            # Save tokens to the file the MCP package expects
+            os.makedirs(os.path.dirname(token_file), exist_ok=True)
+            with open(token_file, "w", encoding="utf-8") as f:
+                json.dump(tokens, f, indent=2)
+            logger.info(f"Saved OAuth tokens to {token_file}")
+
+            # Attempt to connect the MCP server now
+            args = json.loads(srv.args) if srv.args else []
+            env = json.loads(srv.env) if srv.env else {}
+            connected = await mcp_manager.connect_server(
+                server_id=server_id,
+                name=srv.name,
+                transport=srv.transport,
+                command=srv.command,
+                args=args,
+                env=env,
+                url=srv.url,
+            )
+
+            if connected:
+                status = mcp_manager.get_server_status(server_id)
+                tool_count = status.get("tool_count", 0)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorization Successful",
+                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
+                    success=True,
+                ))
+            else:
+                status = mcp_manager.get_server_status(server_id)
+                return HTMLResponse(_oauth_result_page(
+                    "Authorized but Connection Failed",
+                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
+                ))
+        except HTTPException as e:
+            logger.warning(f"OAuth callback rejected: {e.detail}")
+            return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code)
+        except Exception as e:
+            logger.exception(f"OAuth callback error: {e}")
+            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
+        finally:
+            db.close()
+
+    return router
+
+
+def _oauth_authorize_page(
+    auth_url: str,
+    server_id: str,
+    redirect_uri: str,
+) -> str:
+    """Page with Google sign-in link and URL paste-back form for remote access."""
+    # Escape values interpolated into the page: `server_id` comes from the OAuth
+    # state and is not trusted.
+    auth_url = html.escape(auth_url, quote=True)
+    server_id = html.escape(server_id, quote=True)
+    redirect_uri = html.escape(redirect_uri, quote=True)
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>Authorize — Odysseus</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 480px; text-align: center; }}
+  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
+  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
+  .step b {{ color: #e06c75; }}
+  a.auth-link {{
+    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
+    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
+    font-weight: 600; font-size: 0.9rem;
+  }}
+  a.auth-link:hover {{ background: #c55; }}
+  input[type=text] {{
+    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
+    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
+    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
+  }}
+  input:focus {{ outline: none; border-color: #e06c75; }}
+  button {{
+    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
+    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
+    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
+  }}
+  button:hover {{ background: #c55; }}
+  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
+</style></head>
+<body><div class="card">
+  <h2>Authorize Google Account</h2>
+  <div class="step">
+    <b>1.</b> Click the button below to sign in with Google<br>
+    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
+    <b>3.</b> Copy the full URL from your browser's address bar<br>
+    <b>4.</b> Paste it below and click Connect
+  </div>
+  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
+  <div class="divider"></div>
+  <!-- Relative action: the browser resolves it against the origin this page was
+       served from, so the form follows the user through any proxy without the
+       app having to know the scheme or the host. An absolute http:// action is
+       blocked as mixed content on exactly the HTTPS deployments that need
+       paste-back, and request.url.scheme cannot be trusted to spot them —
+       uvicorn only honours X-Forwarded-Proto from a peer in
+       --forwarded-allow-ips, which defaults to 127.0.0.1 and excludes a proxy
+       arriving over the Docker bridge. -->
+  <form method="POST" action="/api/mcp/oauth/exchange/{server_id}">
+    <p>Paste the URL from your browser after signing in:</p>
+    <input type="text" name="callback_url" placeholder="{redirect_uri}?code=..." required>
+    <br><button type="submit">Connect</button>
+  </form>
+</div></body></html>"""
+
+
+def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
+    """Generate a simple HTML page for the OAuth result."""
+    safe_title = html.escape(title)
+    safe_message = html.escape(message)
+    color = "#00661a" if success else "#e06c75"
+    icon = "&#10003;" if success else "&#10007;"
+    return f"""<!DOCTYPE html>
+<html><head>
+<meta charset="UTF-8"><title>{safe_title}</title>
+<style>
+  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
+    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
+  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
+    padding: 2rem; max-width: 420px; text-align: center; }}
+  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
+  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
+  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
+</style></head>
+<body><div class="card">
+  <div class="icon">{icon}</div>
+  <h2>{safe_title}</h2>
+  <p>{safe_message}</p>
+</div></body></html>"""

routes/mcp_routes.py

@@ -1,574 +1,18 @@
-# routes/mcp_routes.py
-"""MCP (Model Context Protocol) server management routes."""
-import json
-import os
-import uuid
-import urllib.parse
-import html
-from fastapi import APIRouter, Form, HTTPException, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-import logging
-import httpx
+"""Backward-compat shim — canonical location is routes/mcp/mcp_routes.py.
 
-from core.database import McpServer, SessionLocal
-from core.middleware import require_admin
-from src.mcp_manager import McpManager
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.mcp_routes``, ``from routes.mcp_routes import X``,
+``importlib.import_module("routes.mcp_routes")``, the
+``sys.modules.pop("routes.mcp_routes")`` + re-import pattern in
+test_security_regressions.py, and the ``monkeypatch.setattr(mcp_routes,
+"MCP_OAUTH_DIR", ...)`` pattern all operate on the *same* object. This also
+makes ``mcp_routes.__file__`` resolve to the canonical file (which the
+source-introspection at line 839 reads). Keeps existing import paths working
+after slice 2o (#4082/#4071).
+"""
 
-logger = logging.getLogger(__name__)
+import sys as _sys
 
-router = APIRouter(prefix="/api/mcp", tags=["mcp"])
+from routes.mcp import mcp_routes as _canonical  # noqa: F401
 
-
-def _load_disabled_map():
-    """Load per-server disabled tool sets from DB."""
-    db = SessionLocal()
-    try:
-        disabled_map = {}
-        for srv in db.query(McpServer).all():
-            if srv.disabled_tools:
-                try:
-                    names = json.loads(srv.disabled_tools)
-                    if names:
-                        disabled_map[srv.id] = set(names)
-                except (json.JSONDecodeError, TypeError):
-                    pass
-        return disabled_map
-    finally:
-        db.close()
-
-
-def setup_mcp_routes(mcp_manager: McpManager):
-    """Setup MCP routes with the provided manager."""
-
-    @router.get("/servers")
-    def list_servers(request: Request):
-        """List all configured MCP servers with connection status."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            servers = db.query(McpServer).all()
-            result = []
-            for srv in servers:
-                status = mcp_manager.get_server_status(srv.id)
-                oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None
-                needs_oauth = False
-                if oauth_cfg:
-                    token_file = os.path.expanduser(oauth_cfg.get("token_file", ""))
-                    needs_oauth = token_file and not os.path.exists(token_file)
-                disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-                total_tools = status.get("tool_count", 0)
-                result.append({
-                    "id": srv.id,
-                    "name": srv.name,
-                    "transport": srv.transport,
-                    "command": srv.command,
-                    "args": json.loads(srv.args) if srv.args else [],
-                    "env": json.loads(srv.env) if srv.env else {},
-                    "url": srv.url,
-                    "is_enabled": srv.is_enabled,
-                    "status": status.get("status", "disconnected"),
-                    "tool_count": total_tools,
-                    "disabled_tool_count": len(disabled_list),
-                    "enabled_tool_count": max(0, total_tools - len(disabled_list)),
-                    "error": status.get("error"),
-                    "has_oauth": oauth_cfg is not None,
-                    "needs_oauth": needs_oauth,
-                })
-            return result
-        finally:
-            db.close()
-
-    @router.post("/servers")
-    async def add_server(
-        request: Request,
-        name: str = Form(...),
-        transport: str = Form("stdio"),
-        command: str = Form(None),
-        args: str = Form("[]"),
-        env: str = Form("{}"),
-        url: str = Form(None),
-        oauth_file: str = Form(None),
-        oauth_config: str = Form(None),
-    ):
-        """Add a new MCP server config and attempt connection. Admin-only:
-        registering a stdio server is equivalent to executing arbitrary
-        binaries on the host."""
-        require_admin(request)
-        server_id = str(uuid.uuid4())[:8]
-
-        # Validate
-        if transport == "stdio" and not command:
-            raise HTTPException(400, "command is required for stdio transport")
-        if transport == "sse" and not url:
-            raise HTTPException(400, "url is required for SSE transport")
-
-        # Parse JSON fields
-        try:
-            parsed_args = json.loads(args) if args else []
-        except json.JSONDecodeError:
-            parsed_args = []
-        try:
-            parsed_env = json.loads(env) if env else {}
-        except json.JSONDecodeError:
-            parsed_env = {}
-
-        # Parse OAuth config
-        parsed_oauth_config = None
-        if oauth_config:
-            try:
-                parsed_oauth_config = json.loads(oauth_config)
-            except json.JSONDecodeError:
-                pass
-
-        # Write OAuth credentials file if provided (for Google MCP servers)
-        logger.info(f"MCP add_server: oauth_file={oauth_file!r}")
-        if oauth_file:
-            try:
-                oauth_data = json.loads(oauth_file)
-                oauth_dir = os.path.expanduser(oauth_data.get("dir", ""))
-                oauth_filename = oauth_data.get("filename", "")
-                client_id = oauth_data.get("client_id", "")
-                client_secret = oauth_data.get("client_secret", "")
-                if oauth_dir and oauth_filename and client_id and client_secret:
-                    os.makedirs(oauth_dir, exist_ok=True)
-                    creds = {
-                        "installed": {
-                            "client_id": client_id,
-                            "client_secret": client_secret,
-                            "redirect_uris": ["http://localhost"],
-                            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-                            "token_uri": "https://accounts.google.com/o/oauth2/token",
-                        }
-                    }
-                    filepath = os.path.join(oauth_dir, oauth_filename)
-                    with open(filepath, "w") as f:
-                        json.dump(creds, f, indent=2)
-                    logger.info(f"Wrote OAuth credentials to {filepath}")
-                    parsed_env.pop("GOOGLE_CLIENT_ID", None)
-                    parsed_env.pop("GOOGLE_CLIENT_SECRET", None)
-            except (json.JSONDecodeError, OSError) as e:
-                logger.warning(f"Failed to write OAuth file: {e}")
-
-        # Save to DB
-        db = SessionLocal()
-        try:
-            srv = McpServer(
-                id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=json.dumps(parsed_args),
-                env=json.dumps(parsed_env),
-                url=url,
-                is_enabled=True,
-                oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None,
-            )
-            db.add(srv)
-            db.commit()
-        finally:
-            db.close()
-
-        # Check if OAuth token already exists — skip connection attempt if not
-        needs_oauth = False
-        if parsed_oauth_config:
-            token_file = os.path.expanduser(parsed_oauth_config.get("token_file", ""))
-            if token_file and not os.path.exists(token_file):
-                needs_oauth = True
-
-        connected = False
-        if not needs_oauth:
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=name,
-                transport=transport,
-                command=command,
-                args=parsed_args,
-                env=parsed_env,
-                url=url,
-            )
-
-        status = mcp_manager.get_server_status(server_id)
-        return {
-            "id": server_id,
-            "name": name,
-            "connected": connected,
-            "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"),
-            "tool_count": status.get("tool_count", 0),
-            "error": "OAuth authorization required" if needs_oauth else status.get("error"),
-            "needs_oauth": needs_oauth,
-        }
-
-    @router.post("/servers/{server_id}/reconnect")
-    async def reconnect_server(server_id: str, request: Request):
-        """Reconnect to an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            status = mcp_manager.get_server_status(server_id)
-            return {
-                "connected": connected,
-                "status": status.get("status", "disconnected"),
-                "tool_count": status.get("tool_count", 0),
-                "error": status.get("error"),
-            }
-        finally:
-            db.close()
-
-    @router.patch("/servers/{server_id}")
-    async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)):
-        """Enable or disable an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            enabled = str(is_enabled).lower() == "true"
-            srv.is_enabled = enabled
-            db.commit()
-
-            if enabled:
-                args = json.loads(srv.args) if srv.args else []
-                env = json.loads(srv.env) if srv.env else {}
-                await mcp_manager.connect_server(
-                    server_id=server_id,
-                    name=srv.name,
-                    transport=srv.transport,
-                    command=srv.command,
-                    args=args,
-                    env=env,
-                    url=srv.url,
-                )
-            else:
-                await mcp_manager.disconnect_server(server_id)
-
-            return {"id": server_id, "is_enabled": enabled}
-        finally:
-            db.close()
-
-    @router.delete("/servers/{server_id}")
-    async def delete_server(server_id: str, request: Request):
-        """Remove an MCP server."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            await mcp_manager.disconnect_server(server_id)
-
-            db.delete(srv)
-            db.commit()
-            return {"status": "deleted"}
-        finally:
-            db.close()
-
-    @router.get("/tools")
-    def list_tools(request: Request):
-        """List all discovered MCP tools across all connected servers."""
-        require_admin(request)
-        disabled_map = _load_disabled_map()
-        return mcp_manager.get_all_tools(disabled_map)
-
-    @router.get("/servers/{server_id}/tools")
-    def list_server_tools(server_id: str, request: Request):
-        """List all tools for a specific MCP server with enabled/disabled state."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else []
-            disabled_set = set(disabled_list)
-        finally:
-            db.close()
-
-        all_tools = mcp_manager.get_all_tools()
-        server_tools = [t for t in all_tools if t["server_id"] == server_id]
-        for t in server_tools:
-            t["is_disabled"] = t["name"] in disabled_set
-        return server_tools
-
-    @router.patch("/servers/{server_id}/tools")
-    async def update_disabled_tools(server_id: str, request: Request):
-        """Bulk update disabled tools list for a server.
-
-        Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]}
-        """
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-
-            body = await request.json()
-            disabled = body.get("disabled", [])
-            if not isinstance(disabled, list):
-                raise HTTPException(400, "disabled must be a list of tool names")
-
-            srv.disabled_tools = json.dumps(disabled) if disabled else None
-            db.commit()
-
-            return {"id": server_id, "disabled_count": len(disabled)}
-        finally:
-            db.close()
-
-    # ── OAuth flow for Google MCP servers ──────────────────────────
-
-    @router.get("/oauth/authorize/{server_id}")
-    def oauth_authorize(server_id: str, request: Request):
-        """Show OAuth authorization page with Google sign-in link."""
-        require_admin(request)
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                raise HTTPException(404, "Server not found")
-            if not srv.oauth_config:
-                raise HTTPException(400, "Server has no OAuth config")
-
-            oauth_cfg = json.loads(srv.oauth_config)
-            keys_file = os.path.expanduser(oauth_cfg.get("keys_file", ""))
-            if not keys_file or not os.path.exists(keys_file):
-                raise HTTPException(400, "OAuth keys file not found")
-
-            with open(keys_file) as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            if not keys:
-                raise HTTPException(400, "Invalid OAuth keys file format")
-
-            client_id = keys["client_id"]
-            scopes = oauth_cfg.get("scopes", [])
-
-            # For Desktop App creds, redirect to localhost — the user will
-            # paste the resulting URL back if they're on a different device.
-            redirect_uri = "http://localhost:7000/api/mcp/oauth/callback"
-
-            params = {
-                "client_id": client_id,
-                "redirect_uri": redirect_uri,
-                "response_type": "code",
-                "scope": " ".join(scopes),
-                "access_type": "offline",
-                "prompt": "consent",
-                "state": server_id,
-            }
-            auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params)
-
-            # Determine if user is accessing from the same machine
-            host = request.headers.get("host", "")
-            is_local = host.startswith("localhost") or host.startswith("127.0.0.1")
-
-            if is_local:
-                # Same machine — just redirect, callback will work directly
-                return RedirectResponse(auth_url)
-            else:
-                # Remote device — show paste-back page
-                return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host))
-        finally:
-            db.close()
-
-    @router.get("/oauth/callback")
-    async def oauth_callback(code: str, state: str, request: Request):
-        """Handle OAuth callback from Google — exchange code for tokens."""
-        require_admin(request)
-        server_id = state
-        return await _exchange_and_connect(server_id, code, request)
-
-    @router.post("/oauth/exchange/{server_id}")
-    async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)):
-        """Manual code exchange — user pastes the callback URL from their browser."""
-        require_admin(request)
-        try:
-            parsed = urllib.parse.urlparse(callback_url)
-            params = urllib.parse.parse_qs(parsed.query)
-            code = params.get("code", [None])[0]
-            if not code:
-                return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400)
-        except Exception:
-            return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400)
-
-        return await _exchange_and_connect(server_id, code, request)
-
-    async def _exchange_and_connect(server_id: str, code: str, request: Request):
-        """Exchange auth code for tokens and connect the MCP server."""
-        db = SessionLocal()
-        try:
-            srv = db.query(McpServer).filter(McpServer.id == server_id).first()
-            if not srv:
-                return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404)
-            if not srv.oauth_config:
-                return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400)
-
-            oauth_cfg = json.loads(srv.oauth_config)
-            keys_file = os.path.expanduser(oauth_cfg.get("keys_file", ""))
-            token_file = os.path.expanduser(oauth_cfg.get("token_file", ""))
-
-            with open(keys_file) as f:
-                keys_data = json.load(f)
-            keys = keys_data.get("installed") or keys_data.get("web")
-            client_id = keys["client_id"]
-            client_secret = keys["client_secret"]
-
-            redirect_uri = "http://localhost:7000/api/mcp/oauth/callback"
-
-            async with httpx.AsyncClient() as client:
-                resp = await client.post(
-                    "https://oauth2.googleapis.com/token",
-                    data={
-                        "code": code,
-                        "client_id": client_id,
-                        "client_secret": client_secret,
-                        "redirect_uri": redirect_uri,
-                        "grant_type": "authorization_code",
-                    },
-                )
-
-            if resp.status_code != 200:
-                err = resp.text
-                logger.error(f"OAuth token exchange failed: {err}")
-                return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400)
-
-            tokens = resp.json()
-            logger.info(f"OAuth tokens received for server {server_id}")
-
-            # Save tokens to the file the MCP package expects
-            os.makedirs(os.path.dirname(token_file), exist_ok=True)
-            with open(token_file, "w") as f:
-                json.dump(tokens, f, indent=2)
-            logger.info(f"Saved OAuth tokens to {token_file}")
-
-            # Attempt to connect the MCP server now
-            args = json.loads(srv.args) if srv.args else []
-            env = json.loads(srv.env) if srv.env else {}
-            connected = await mcp_manager.connect_server(
-                server_id=server_id,
-                name=srv.name,
-                transport=srv.transport,
-                command=srv.command,
-                args=args,
-                env=env,
-                url=srv.url,
-            )
-
-            if connected:
-                status = mcp_manager.get_server_status(server_id)
-                tool_count = status.get("tool_count", 0)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorization Successful",
-                    f"{srv.name} connected with {tool_count} tools. You can close this window.",
-                    success=True,
-                ))
-            else:
-                status = mcp_manager.get_server_status(server_id)
-                return HTMLResponse(_oauth_result_page(
-                    "Authorized but Connection Failed",
-                    f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.",
-                ))
-        except Exception as e:
-            logger.exception(f"OAuth callback error: {e}")
-            return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500)
-        finally:
-            db.close()
-
-    return router
-
-
-def _oauth_authorize_page(auth_url: str, server_id: str, host: str) -> str:
-    """Page with Google sign-in link and URL paste-back form for remote access."""
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>Authorize — Odysseus</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 480px; text-align: center; }}
-  h2 {{ color: #e06c75; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.82rem; line-height: 1.6; margin: 0.8rem 0; }}
-  .step {{ text-align: left; color: #ccc; font-size: 0.82rem; line-height: 1.7; margin: 1rem 0; }}
-  .step b {{ color: #e06c75; }}
-  a.auth-link {{
-    display: inline-block; margin: 1rem 0; padding: 0.6rem 1.5rem;
-    background: #e06c75; color: #fff; text-decoration: none; border-radius: 6px;
-    font-weight: 600; font-size: 0.9rem;
-  }}
-  a.auth-link:hover {{ background: #c55; }}
-  input[type=text] {{
-    width: 100%; padding: 0.5rem; margin: 0.5rem 0;
-    background: #0f0f0f; border: 1px solid #333; border-radius: 6px;
-    color: #e0e0e0; font-family: 'Fira Code', monospace; font-size: 0.8rem;
-  }}
-  input:focus {{ outline: none; border-color: #e06c75; }}
-  button {{
-    padding: 0.5rem 1.5rem; border: none; border-radius: 6px;
-    background: #e06c75; color: #fff; font-weight: 600; cursor: pointer;
-    font-family: 'Fira Code', monospace; font-size: 0.85rem; margin-top: 0.3rem;
-  }}
-  button:hover {{ background: #c55; }}
-  .divider {{ border-top: 1px solid #333; margin: 1.2rem 0; }}
-</style></head>
-<body><div class="card">
-  <h2>Authorize Google Account</h2>
-  <div class="step">
-    <b>1.</b> Click the button below to sign in with Google<br>
-    <b>2.</b> After approving, your browser will show an error page — that's normal<br>
-    <b>3.</b> Copy the full URL from your browser's address bar<br>
-    <b>4.</b> Paste it below and click Connect
-  </div>
-  <a class="auth-link" href="{auth_url}" target="_blank" rel="noopener">Sign in with Google</a>
-  <div class="divider"></div>
-  <form method="POST" action="http://{host}/api/mcp/oauth/exchange/{server_id}">
-    <p>Paste the URL from your browser after signing in:</p>
-    <input type="text" name="callback_url" placeholder="http://localhost:7000/api/mcp/oauth/callback?code=..." required>
-    <br><button type="submit">Connect</button>
-  </form>
-</div></body></html>"""
-
-
-def _oauth_result_page(title: str, message: str, success: bool = False) -> str:
-    """Generate a simple HTML page for the OAuth result."""
-    safe_title = html.escape(title)
-    safe_message = html.escape(message)
-    color = "#00661a" if success else "#e06c75"
-    icon = "&#10003;" if success else "&#10007;"
-    return f"""<!DOCTYPE html>
-<html><head>
-<meta charset="UTF-8"><title>{safe_title}</title>
-<style>
-  body {{ font-family: 'Fira Code', monospace; background: #0f0f0f; color: #e0e0e0;
-    display: flex; justify-content: center; align-items: center; min-height: 100vh; }}
-  .card {{ background: #1a1a1a; border: 1px solid #333; border-radius: 12px;
-    padding: 2rem; max-width: 420px; text-align: center; }}
-  .icon {{ font-size: 3rem; color: {color}; margin-bottom: 1rem; }}
-  h2 {{ color: {color}; margin-bottom: 0.5rem; font-size: 1.1rem; }}
-  p {{ color: #aaa; font-size: 0.85rem; line-height: 1.5; }}
-</style></head>
-<body><div class="card">
-  <div class="icon">{icon}</div>
-  <h2>{safe_title}</h2>
-  <p>{safe_message}</p>
-</div></body></html>"""
+_sys.modules[__name__] = _canonical

routes/memory/__init__.py

@@ -0,0 +1,5 @@
+"""Memory route domain package (slice 2c, #4082/#4071).
+
+Contains memory_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/memory_routes.py re-exports from here.
+"""

routes/memory/memory_routes.py

@@ -0,0 +1,568 @@
+# routes/memory_routes.py
+from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
+from typing import Dict, Any, Optional, List
+import json
+import os
+import re
+import tempfile
+import time
+from datetime import datetime
+import logging
+
+# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
+# Strips one prefix per call so import-from-LLM-output doesn't leave the
+# numbering inside the saved memory text. Bullet markers (-, *, •) are
+# also peeled here for the same reason.
+_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+
+
+def _strip_list_prefix(text: str) -> str:
+    if not text:
+        return text
+    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+
+from services.memory import MemoryManager, MemoryStoreUnreadable
+from core.session_manager import SessionManager
+from src.request_models import MemoryAddRequest
+from core.database import SessionLocal
+from src.llm_core import llm_call_async
+from services.memory.memory_extractor import audit_memories
+from src.auth_helpers import get_current_user, require_user
+from src.endpoint_resolver import resolve_endpoint
+from src.task_endpoint import resolve_task_endpoint
+from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES
+
+logger = logging.getLogger(__name__)
+
+
+def _load_for_update(memory_manager) -> List[Dict[str, Any]]:
+    """Load the whole store for a read-modify-write cycle.
+
+    A transient read failure must not look like an empty store: the caller
+    would append to ``[]`` and save that back, atomically destroying every
+    existing memory (issue #5673). Surface it as a 503 and change nothing.
+    """
+    try:
+        return memory_manager.load_all_for_update()
+    except MemoryStoreUnreadable as e:
+        logger.error("Refusing to rewrite the memory store: %s", e)
+        raise HTTPException(
+            503, "Memory store is temporarily unreadable — no changes were made."
+        )
+
+
+def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
+    """Set up memory-related routes."""
+    router = APIRouter(prefix="/api/memory", tags=["memory"])
+
+    def _owner(request: Request) -> Optional[str]:
+        return get_current_user(request)
+
+    def _assert_session_owner(session_obj, user):
+        """SECURITY: 404 if the caller does not own this session.
+
+        SessionManager.get_session is NOT owner-scoped — it returns any
+        session by id. These routes accept a caller-supplied session id, so
+        without this gate a user could target another tenant's session and
+        leak their chat history, their session-scoped LLM credentials, or the
+        session title. Mirrors session_routes / webhook_routes ownership.
+        """
+        if user is not None and getattr(session_obj, "owner", None) != user:
+            raise HTTPException(404, "Session not found")
+
+    def _verify_memory_owner(memory: dict, user: Optional[str]):
+        """Raise 404 if user doesn't own this memory.
+
+        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
+        allowed any user to read/edit/delete memories with an empty/null owner
+        field, which leaked legacy data across the multi-user deploy.
+        """
+        if user is None:
+            return  # Auth disabled
+        if memory.get("owner") != user:
+            raise HTTPException(404, "Memory not found")
+
+    @router.post("/debug")
+    def debug_memory_relevance(request: Request, query: str = Form(...)):
+        """Debug which memories would be triggered for a query"""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
+
+        return {
+            "query": query,
+            "total_memories": len(memories),
+            "relevant_count": len(relevant),
+            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
+                                 for m in relevant]
+        }
+
+    @router.post("/add", response_model=Dict[str, Any])
+    async def api_add_memory(
+        request: Request,
+        memory_data: Optional[MemoryAddRequest] = None
+    ):
+        """Add a new memory entry with optional category, source, and session reference."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+        if memory_data is None:
+            form = await request.form()
+            memory_data = MemoryAddRequest(
+                text=form.get("text"),
+                category=form.get("category", "fact"),
+                source=form.get("source", "user"),
+                session_id=form.get("session_id")
+            )
+
+        user = _owner(request)
+        text = (memory_data.text or "").strip()
+        if not text:
+            raise HTTPException(400, "empty memory")
+        user_mem = memory_manager.load(owner=user)
+        if memory_manager.find_duplicates(text, user_mem):
+            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
+
+        if memory_data.session_id:
+            try:
+                session_obj = session_manager.get_session(memory_data.session_id)
+            except KeyError:
+                raise HTTPException(404, "Session not found")
+            _assert_session_owner(session_obj, user)
+
+        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
+        if memory_data.session_id:
+            new_entry["session_id"] = memory_data.session_id
+        all_mem = _load_for_update(memory_manager)
+        all_mem.append(new_entry)
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.add(new_entry["id"], text)
+        try:
+            from src.event_bus import fire_event
+            fire_event("memory_added", user)
+        except Exception:
+            logger.debug("memory_added event dispatch failed", exc_info=True)
+        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
+
+    @router.get("")
+    def api_get_memory(request: Request):
+        """Return all memory entries with their metadata."""
+        user = _owner(request)
+        return {"memory": memory_manager.load(owner=user)}
+
+    @router.post("/search")
+    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
+        """Search across all memories with optional filters."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+
+        if session_id:
+            memories = [m for m in memories if m.get("session_id") == session_id]
+
+        if category:
+            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
+
+        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
+
+        return {"memories": relevant, "total": len(relevant), "query": query}
+
+    @router.get("/timeline")
+    def memory_timeline(request: Request):
+        """Get memories in chronological order with source session information."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        results = []
+        for memory in sorted_memories:
+            if "timestamp" in memory:
+                try:
+                    dt = datetime.fromtimestamp(memory["timestamp"])
+                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
+                except (ValueError, OSError, OverflowError):
+                    memory["timestamp_str"] = "Unknown"
+            else:
+                memory["timestamp_str"] = "Unknown"
+
+            session_id = memory.get("session_id")
+            if session_id and session_id in session_manager.sessions:
+                try:
+                    session = session_manager.get_session(session_id)
+                    if session:
+                        _assert_session_owner(session, user)
+                    memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
+                except KeyError:
+                    memory["session_name"] = "Unknown"
+                except HTTPException as exc:
+                    if exc.status_code != 404:
+                        raise
+                    memory["session_name"] = "Unknown"
+            else:
+                memory["session_name"] = "Unknown"
+
+            results.append(memory)
+
+        return {"timeline": results, "total": len(results)}
+
+    @router.get("/by-session/{session_id}")
+    def get_memory_by_session(request: Request, session_id: str):
+        """Get all memories associated with a specific session."""
+        user = _owner(request)
+        try:
+            _session_obj = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, f"Session {session_id} not found")
+        _assert_session_owner(_session_obj, user)
+        memories = memory_manager.load(owner=user)
+        session_memories = [m for m in memories if m.get("session_id") == session_id]
+
+        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
+
+        try:
+            session = session_manager.get_session(session_id)
+            session_name = session.name if session else f"Session {session_id[:6]}"
+        except KeyError:
+            session_name = f"Session {session_id[:6]}"
+
+        for memory in session_memories:
+            memory["session_name"] = session_name
+
+        return {
+            "session_id": session_id,
+            "session_name": session_name,
+            "memory_count": len(session_memories),
+            "memories": session_memories
+        }
+
+    @router.post("/extract")
+    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
+        """Analyze a session's chat history and return memory suggestions."""
+        require_user(request)
+        try:
+            sess = session_manager.get_session(session)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        _assert_session_owner(sess, _owner(request))
+
+        system_msg = {
+            "role": "system",
+            "content": (
+                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
+                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
+                "might want to remember for future interactions. Return each piece of information as a JSON object "
+                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
+                "Only include information that is specific and likely to be useful later."
+            ),
+        }
+        messages = [system_msg] + sess.get_context_messages()
+
+        t_url, t_model, t_headers = resolve_task_endpoint(
+            sess.endpoint_url, sess.model, sess.headers, owner=_owner(request)
+        )
+
+        try:
+            suggestion_text = await llm_call_async(
+                t_url,
+                t_model,
+                messages,
+                temperature=0.2,
+                max_tokens=500,
+                headers=t_headers,
+            )
+            try:
+                suggestions = json.loads(suggestion_text)
+                if isinstance(suggestions, list):
+                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
+                else:
+                    suggestions = []
+            except json.JSONDecodeError:
+                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
+
+            return {"suggestions": [s for s in suggestions if s]}
+        except Exception as e:
+            logger.error(f"LLM memory extraction failed (session {session}): {e}")
+            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
+            return {"suggestions": [item["text"] for item in fallback]}
+
+    @router.post("/audit")
+    async def api_audit_memories(request: Request, session: str = Form(None)):
+        """Deduplicate and consolidate memories via LLM.
+
+        Uses task/utility/default settings through the shared resolver, with
+        the active session as fallback when no task or utility model is set.
+        Returns before and after memory counts.
+        """
+        user = _owner(request)
+        fallback_url = fallback_model = None
+        fallback_headers = None
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+                fallback_url = sess.endpoint_url
+                fallback_model = sess.model
+                fallback_headers = sess.headers
+            except KeyError:
+                pass
+
+        endpoint_url, model, headers = resolve_task_endpoint(
+            fallback_url, fallback_model, fallback_headers, owner=user
+        )
+
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No default model configured — set one in Settings")
+
+        result = await audit_memories(
+            memory_manager,
+            memory_vector,
+            endpoint_url,
+            model,
+            headers,
+            owner=user,
+        )
+
+        if "error" in result and "before" not in result:
+            raise HTTPException(502, f"Audit failed: {result['error']}")
+
+        return {
+            "ok": "error" not in result,
+            "before": result.get("before", 0),
+            "after": result.get("after", 0),
+            "removed": result.get("before", 0) - result.get("after", 0),
+            # True when the audit skipped the LLM because nothing changed
+            # since the last tidy. Frontend already says "Already clean"
+            # for removed==0, so this is here for future use / debugging.
+            "already_tidy": bool(result.get("already_tidy")),
+        }
+
+    @router.post("/import")
+    async def import_memories_from_file(
+        request: Request,
+        session: str | None = Form(None),
+        file: UploadFile = File(...)
+    ):
+        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
+        from src.auth_helpers import require_privilege
+        require_privilege(request, "can_manage_memory")
+
+        endpoint_url = None
+        model = None
+        headers = {}
+
+        user = _owner(request)
+
+        if session:
+            try:
+                sess = session_manager.get_session(session)
+                _assert_session_owner(sess, user)
+            except KeyError:
+                sess = None
+            except HTTPException as exc:
+                if exc.status_code != 404:
+                    raise
+                sess = None
+
+            if sess is None:
+                logger.warning("Session %s not found or inaccessible, falling back to utility endpoint", session)
+                endpoint_url, model, headers = resolve_endpoint("utility", owner=user)
+            else:
+                endpoint_url, model, headers = resolve_task_endpoint(
+                    sess.endpoint_url, sess.model, sess.headers, owner=user
+                )
+        else:
+            endpoint_url, model, headers = resolve_task_endpoint(owner=user)
+    
+        if not endpoint_url or not model:
+            raise HTTPException(400, "No LLM model configured. Set a default model in Settings.")
+
+        content = await read_upload_limited(file, MEMORY_IMPORT_MAX_BYTES, "Memory import")
+        filename = file.filename or "upload"
+        _, ext = os.path.splitext(filename.lower())
+
+        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
+        if ext not in allowed:
+            raise HTTPException(400, f"Unsupported file type: {ext}")
+
+        # Extract text based on file type
+        if ext == ".pdf":
+            from src.document_processor import _process_pdf
+            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                text = _process_pdf(tmp_path, owner=_owner(request))
+            finally:
+                os.unlink(tmp_path)
+        else:
+            try:
+                text = content.decode("utf-8")
+            except UnicodeDecodeError:
+                from charset_normalizer import detect
+                encoding = (detect(content) or {}).get("encoding") or "utf-8"
+                text = content.decode(encoding, errors="replace")
+
+        if not text.strip():
+            return {"suggestions": [], "message": "No readable content found"}
+
+        # Fast path: a .json upload that already looks like a memories export
+        # (list of {text, category, ...} dicts, or list of strings) round-trips
+        # directly without spending an LLM call to re-extract its own output.
+        # Without this, re-importing a memories.json from another account
+        # ran the file through the extractor, which often re-emitted the
+        # entries as a numbered list (and the numbering leaked into the
+        # `text` field).
+        if ext == ".json":
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list) and parsed:
+                direct = []
+                for item in parsed:
+                    if isinstance(item, dict) and item.get("text"):
+                        direct.append({
+                            "text": _strip_list_prefix(str(item["text"])),
+                            "category": item.get("category") or "fact",
+                        })
+                    elif isinstance(item, str) and item.strip():
+                        direct.append({
+                            "text": _strip_list_prefix(item.strip()),
+                            "category": "fact",
+                        })
+                if direct:
+                    return {"suggestions": direct, "filename": filename}
+
+        # Truncate very long documents
+        if len(text) > 15000:
+            text = text[:15000] + "\n[Truncated]"
+
+        # Send to LLM for memory extraction
+        import_prompt = (
+            "You are a memory extraction assistant. The user uploaded a document. "
+            "Analyze the text below and extract specific, useful facts — things like "
+            "names, preferences, jobs, locations, relationships, opinions, projects, "
+            "goals, contacts, or any other personal details worth remembering.\n\n"
+            "Rules:\n"
+            "- Each fact should be a short, self-contained statement\n"
+            "- Do NOT extract generic knowledge\n"
+            "- Focus on personal, memorable information\n"
+            "- If there are no useful facts, return an empty array\n\n"
+            "Return a JSON array of objects with 'text' and 'category' fields.\n"
+            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
+            "Return ONLY valid JSON, no markdown fences."
+        )
+
+        try:
+            raw = await llm_call_async(
+                endpoint_url,
+                model,
+                [
+                    {"role": "system", "content": import_prompt},
+                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
+                ],
+                temperature=0.2,
+                max_tokens=2000,
+                headers=headers,
+            )
+
+            # Parse JSON
+            raw = raw.strip()
+            if raw.startswith("```"):
+                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
+
+            suggestions = json.loads(raw)
+            if isinstance(suggestions, list):
+                normalized = []
+                for s in suggestions:
+                    if not s:
+                        continue
+                    if isinstance(s, dict):
+                        s = dict(s)
+                        if s.get("text"):
+                            s["text"] = _strip_list_prefix(str(s["text"]))
+                        normalized.append(s)
+                    else:
+                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
+                suggestions = normalized
+            else:
+                suggestions = []
+
+            return {"suggestions": suggestions, "filename": filename}
+
+        except json.JSONDecodeError:
+            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
+            # numbering the model added so saved memories don't keep the prefix.
+            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
+            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
+        except Exception as e:
+            logger.error(f"Memory import extraction failed: {e}")
+            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
+
+    @router.post("/{memory_id}/pin")
+    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
+        """Pin or unpin a memory. Pinned memories are always included in context."""
+        user = _owner(request)
+        all_mem = _load_for_update(memory_manager)
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["pinned"] = pinned
+                memory_manager.save(all_mem)
+                return {"ok": True, "pinned": pinned}
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
+    @router.get("/{memory_id}")
+    def get_memory_item(request: Request, memory_id: str):
+        """Get a specific memory item by ID."""
+        user = _owner(request)
+        memories = memory_manager.load(owner=user)
+        for memory in memories:
+            if memory["id"] == memory_id:
+                return {"memory": memory}
+
+        raise HTTPException(404, "Memory not found")
+
+    @router.put("/{memory_id}")
+    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
+        """Update an existing memory item with new text and optional category."""
+        user = _owner(request)
+        all_mem = _load_for_update(memory_manager)
+        for i, memory in enumerate(all_mem):
+            if memory["id"] == memory_id:
+                _verify_memory_owner(memory, user)
+                all_mem[i]["text"] = text.strip()
+                if category:
+                    all_mem[i]["category"] = category
+                all_mem[i]["timestamp"] = int(time.time())
+
+                memory_manager.save(all_mem)
+                # Sync vector index (remove old, add updated)
+                if memory_vector and memory_vector.healthy:
+                    memory_vector.remove(memory_id)
+                    memory_vector.add(memory_id, text.strip())
+                return {"ok": True, "message": "Memory updated successfully"}
+
+        raise HTTPException(404, f"Memory item {memory_id} not found")
+
+    @router.delete("/{memory_id}")
+    def delete_memory(request: Request, memory_id: str):
+        """Delete a memory item by its ID."""
+        user = _owner(request)
+        all_mem = _load_for_update(memory_manager)
+
+        # Find and verify ownership before deleting
+        target = next((m for m in all_mem if m["id"] == memory_id), None)
+        if not target:
+            raise HTTPException(404, f"Memory item {memory_id} not found")
+        _verify_memory_owner(target, user)
+
+        all_mem = [m for m in all_mem if m["id"] != memory_id]
+        memory_manager.save(all_mem)
+        # Sync vector index
+        if memory_vector and memory_vector.healthy:
+            memory_vector.remove(memory_id)
+        return {"ok": True, "message": "Memory deleted successfully"}
+
+    return router

routes/memory_routes.py

@@ -1,517 +1,18 @@
-# routes/memory_routes.py
-from fastapi import APIRouter, Form, HTTPException, Request, UploadFile, File
-from typing import Dict, Any, Optional, List
-import json
-import os
-import re
-import tempfile
-import time
-from datetime import datetime
-import logging
+"""Backward-compat shim — canonical location is routes/memory/memory_routes.py.
 
-# Leading list-marker like "1.", "12)", or "3:" plus surrounding whitespace.
-# Strips one prefix per call so import-from-LLM-output doesn't leave the
-# numbering inside the saved memory text. Bullet markers (-, *, •) are
-# also peeled here for the same reason.
-_LIST_PREFIX_RE = re.compile(r"^\s*(?:\d{1,3}[.):]\s+|[-*•]\s+)")
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.memory_routes``, ``from routes.memory_routes import X``,
+``importlib.import_module("routes.memory_routes")``, and
+``monkeypatch.setattr(routes.memory_routes, "ATTR", ...)`` (used by
+test_memory_routes_session_owner.py and test_memory_owner_isolation.py via
+``import ... as mr`` + ``setattr(mr, ...)``) all operate on the *same* object
+the application actually uses. Keeps existing import paths working after
+slice 2c (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
+import sys as _sys
 
-def _strip_list_prefix(text: str) -> str:
-    if not text:
-        return text
-    return _LIST_PREFIX_RE.sub("", text, count=1).strip()
+from routes.memory import memory_routes as _canonical  # noqa: F401
 
-from services.memory import MemoryManager
-from core.session_manager import SessionManager
-from src.request_models import MemoryAddRequest
-from core.database import SessionLocal
-from src.llm_core import llm_call_async
-from services.memory.memory_extractor import audit_memories
-from src.auth_helpers import get_current_user
-
-logger = logging.getLogger(__name__)
-
-def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None):
-    """Set up memory-related routes."""
-    router = APIRouter(prefix="/api/memory", tags=["memory"])
-
-    def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
-
-    def _verify_memory_owner(memory: dict, user: Optional[str]):
-        """Raise 404 if user doesn't own this memory.
-
-        SECURITY: strict ownership — previously `mem_owner and mem_owner != user`
-        allowed any user to read/edit/delete memories with an empty/null owner
-        field, which leaked legacy data across the multi-user deploy.
-        """
-        if user is None:
-            return  # Auth disabled
-        if memory.get("owner") != user:
-            raise HTTPException(404, "Memory not found")
-
-    @router.post("/debug")
-    def debug_memory_relevance(request: Request, query: str = Form(...)):
-        """Debug which memories would be triggered for a query"""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05)
-
-        return {
-            "query": query,
-            "total_memories": len(memories),
-            "relevant_count": len(relevant),
-            "relevant_memories": [{"text": m["text"], "category": m.get("category", "unknown")}
-                                 for m in relevant]
-        }
-
-    @router.post("/add", response_model=Dict[str, Any])
-    async def api_add_memory(
-        request: Request,
-        memory_data: Optional[MemoryAddRequest] = None
-    ):
-        """Add a new memory entry with optional category, source, and session reference."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-        if memory_data is None:
-            form = await request.form()
-            memory_data = MemoryAddRequest(
-                text=form.get("text"),
-                category=form.get("category", "fact"),
-                source=form.get("source", "user"),
-                session_id=form.get("session_id")
-            )
-
-        user = _owner(request)
-        text = (memory_data.text or "").strip()
-        if not text:
-            raise HTTPException(400, "empty memory")
-        user_mem = memory_manager.load(owner=user)
-        if memory_manager.find_duplicates(text, user_mem):
-            return {"ok": True, "count": len(user_mem), "message": "Memory already exists"}
-
-        new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user)
-        if memory_data.session_id:
-            new_entry["session_id"] = memory_data.session_id
-        all_mem = memory_manager.load_all()
-        all_mem.append(new_entry)
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.add(new_entry["id"], text)
-        try:
-            from src.event_bus import fire_event
-            fire_event("memory_added", user)
-        except Exception:
-            logger.debug("memory_added event dispatch failed", exc_info=True)
-        return {"ok": True, "count": len([m for m in all_mem if m.get("owner") == user])}
-
-    @router.get("")
-    def api_get_memory(request: Request):
-        """Return all memory entries with their metadata."""
-        user = _owner(request)
-        return {"memory": memory_manager.load(owner=user)}
-
-    @router.post("/search")
-    def search_memories(request: Request, query: str = Form(...), session_id: str = Form(None), category: str = Form(None)):
-        """Search across all memories with optional filters."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-
-        if session_id:
-            memories = [m for m in memories if m.get("session_id") == session_id]
-
-        if category:
-            memories = [m for m in memories if category in m.get("categories", [m.get("category", "")])]
-
-        relevant = memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
-
-        return {"memories": relevant, "total": len(relevant), "query": query}
-
-    @router.get("/timeline")
-    def memory_timeline(request: Request):
-        """Get memories in chronological order with source session information."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        sorted_memories = sorted(memories, key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        results = []
-        for memory in sorted_memories:
-            if "timestamp" in memory:
-                try:
-                    dt = datetime.fromtimestamp(memory["timestamp"])
-                    memory["timestamp_str"] = dt.strftime("%Y-%m-%d %H:%M:%S")
-                except (ValueError, OSError, OverflowError):
-                    memory["timestamp_str"] = "Unknown"
-            else:
-                memory["timestamp_str"] = "Unknown"
-
-            session_id = memory.get("session_id")
-            if session_id and session_id in session_manager.sessions:
-                session = session_manager.get_session(session_id)
-                memory["session_name"] = session.name if session else f"Session {session_id[:6]}"
-            else:
-                memory["session_name"] = "Unknown"
-
-            results.append(memory)
-
-        return {"timeline": results, "total": len(results)}
-
-    @router.get("/by-session/{session_id}")
-    def get_memory_by_session(request: Request, session_id: str):
-        """Get all memories associated with a specific session."""
-        try:
-            session_manager.get_session(session_id)
-        except KeyError:
-            raise HTTPException(404, f"Session {session_id} not found")
-
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        session_memories = [m for m in memories if m.get("session_id") == session_id]
-
-        session_memories.sort(key=lambda x: x.get("timestamp", 0), reverse=True)
-
-        try:
-            session = session_manager.get_session(session_id)
-            session_name = session.name if session else f"Session {session_id[:6]}"
-        except KeyError:
-            session_name = f"Session {session_id[:6]}"
-
-        for memory in session_memories:
-            memory["session_name"] = session_name
-
-        return {
-            "session_id": session_id,
-            "session_name": session_name,
-            "memory_count": len(session_memories),
-            "memories": session_memories
-        }
-
-    @router.post("/extract")
-    async def extract_memory(request: Request, session: str = Form(...)) -> Dict[str, List[str]]:
-        """Analyze a session's chat history and return memory suggestions."""
-        if not get_current_user(request):
-            raise HTTPException(401, "Not authenticated")
-        try:
-            sess = session_manager.get_session(session)
-        except KeyError:
-            raise HTTPException(404, "Session not found")
-
-        system_msg = {
-            "role": "system",
-            "content": (
-                "You are a helpful assistant. Analyze the entire conversation history provided and extract any "
-                "useful factual statements, contacts, addresses, phone numbers, or other information that the user "
-                "might want to remember for future interactions. Return each piece of information as a JSON object "
-                "with a 'text' field. For example: [{'text': 'Alice lives at 123 Main St'}, {'text': 'Bob works at Acme Corp'}]. "
-                "Only include information that is specific and likely to be useful later."
-            ),
-        }
-        messages = [system_msg] + sess.get_context_messages()
-
-        try:
-            suggestion_text = await llm_call_async(
-                sess.endpoint_url,
-                sess.model,
-                messages,
-                temperature=0.2,
-                max_tokens=500,
-                headers=sess.headers,
-            )
-            try:
-                suggestions = json.loads(suggestion_text)
-                if isinstance(suggestions, list):
-                    suggestions = [s if isinstance(s, str) else s.get("text", "") for s in suggestions]
-                else:
-                    suggestions = []
-            except json.JSONDecodeError:
-                suggestions = [line.strip() for line in suggestion_text.splitlines() if line.strip()]
-
-            return {"suggestions": [s for s in suggestions if s]}
-        except Exception as e:
-            logger.error(f"LLM memory extraction failed (session {session}): {e}")
-            fallback = memory_manager.extract_memory_from_chat(sess.history, session)
-            return {"suggestions": [item["text"] for item in fallback]}
-
-    @router.post("/audit")
-    async def api_audit_memories(request: Request, session: str = Form(None)):
-        """Deduplicate and consolidate memories via LLM.
-
-        Uses the default model from settings, or falls back to a session's model.
-        Returns before and after memory counts.
-        """
-        from routes.model_routes import _load_settings, _normalize_base, build_chat_url
-        from core.database import ModelEndpoint
-        import json as _json
-
-        endpoint_url = model = None
-        headers = {}
-
-        # Try default model from settings first
-        settings = _load_settings()
-        ep_id = settings.get("default_endpoint_id", "")
-        default_model = settings.get("default_model", "")
-        if ep_id:
-            db = SessionLocal()
-            try:
-                ep = db.query(ModelEndpoint).filter(
-                    ModelEndpoint.id == ep_id, ModelEndpoint.is_enabled == True
-                ).first()
-                if ep:
-                    base = _normalize_base(ep.base_url)
-                    endpoint_url = build_chat_url(base)
-                    model = default_model
-                    if not model and ep.models:
-                        try:
-                            models = _json.loads(ep.models) if isinstance(ep.models, str) else ep.models
-                            if models:
-                                model = models[0]
-                        except Exception:
-                            pass
-                    if ep.api_key:
-                        headers = {"Authorization": f"Bearer {ep.api_key}"}
-            finally:
-                db.close()
-
-        # Fall back to session model if no default configured
-        if not endpoint_url and session:
-            try:
-                sess = session_manager.get_session(session)
-                endpoint_url = sess.endpoint_url
-                model = sess.model
-                headers = sess.headers
-            except KeyError:
-                pass
-
-        if not endpoint_url or not model:
-            raise HTTPException(400, "No default model configured — set one in Settings")
-
-        user = _owner(request)
-        result = await audit_memories(
-            memory_manager,
-            memory_vector,
-            endpoint_url,
-            model,
-            headers,
-            owner=user,
-        )
-
-        if "error" in result and "before" not in result:
-            raise HTTPException(502, f"Audit failed: {result['error']}")
-
-        return {
-            "ok": "error" not in result,
-            "before": result.get("before", 0),
-            "after": result.get("after", 0),
-            "removed": result.get("before", 0) - result.get("after", 0),
-            # True when the audit skipped the LLM because nothing changed
-            # since the last tidy. Frontend already says "Already clean"
-            # for removed==0, so this is here for future use / debugging.
-            "already_tidy": bool(result.get("already_tidy")),
-        }
-
-    @router.post("/import")
-    async def import_memories_from_file(
-        request: Request,
-        session: str = Form(...),
-        file: UploadFile = File(...)
-    ):
-        """Extract memory suggestions from an uploaded file (PDF, TXT, MD, etc.)."""
-        from src.auth_helpers import require_privilege
-        require_privilege(request, "can_manage_memory")
-        try:
-            sess = session_manager.get_session(session)
-        except KeyError:
-            raise HTTPException(404, "Session not found — needed for LLM config")
-
-        # Read file content
-        content = await file.read()
-        filename = file.filename or "upload"
-        _, ext = os.path.splitext(filename.lower())
-
-        allowed = {".txt", ".md", ".pdf", ".csv", ".log", ".json", ".py", ".js", ".html"}
-        if ext not in allowed:
-            raise HTTPException(400, f"Unsupported file type: {ext}")
-
-        # Extract text based on file type
-        if ext == ".pdf":
-            from src.document_processor import _process_pdf
-            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
-                tmp.write(content)
-                tmp_path = tmp.name
-            try:
-                text = _process_pdf(tmp_path)
-            finally:
-                os.unlink(tmp_path)
-        else:
-            try:
-                text = content.decode("utf-8")
-            except UnicodeDecodeError:
-                from charset_normalizer import detect
-                encoding = (detect(content) or {}).get("encoding") or "utf-8"
-                text = content.decode(encoding, errors="replace")
-
-        if not text.strip():
-            return {"suggestions": [], "message": "No readable content found"}
-
-        # Fast path: a .json upload that already looks like a memories export
-        # (list of {text, category, ...} dicts, or list of strings) round-trips
-        # directly without spending an LLM call to re-extract its own output.
-        # Without this, re-importing a memories.json from another account
-        # ran the file through the extractor, which often re-emitted the
-        # entries as a numbered list (and the numbering leaked into the
-        # `text` field).
-        if ext == ".json":
-            try:
-                parsed = json.loads(text)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list) and parsed:
-                direct = []
-                for item in parsed:
-                    if isinstance(item, dict) and item.get("text"):
-                        direct.append({
-                            "text": _strip_list_prefix(str(item["text"])),
-                            "category": item.get("category") or "fact",
-                        })
-                    elif isinstance(item, str) and item.strip():
-                        direct.append({
-                            "text": _strip_list_prefix(item.strip()),
-                            "category": "fact",
-                        })
-                if direct:
-                    return {"suggestions": direct, "filename": filename}
-
-        # Truncate very long documents
-        if len(text) > 15000:
-            text = text[:15000] + "\n[Truncated]"
-
-        # Send to LLM for memory extraction
-        import_prompt = (
-            "You are a memory extraction assistant. The user uploaded a document. "
-            "Analyze the text below and extract specific, useful facts — things like "
-            "names, preferences, jobs, locations, relationships, opinions, projects, "
-            "goals, contacts, or any other personal details worth remembering.\n\n"
-            "Rules:\n"
-            "- Each fact should be a short, self-contained statement\n"
-            "- Do NOT extract generic knowledge\n"
-            "- Focus on personal, memorable information\n"
-            "- If there are no useful facts, return an empty array\n\n"
-            "Return a JSON array of objects with 'text' and 'category' fields.\n"
-            "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
-            "Return ONLY valid JSON, no markdown fences."
-        )
-
-        try:
-            raw = await llm_call_async(
-                sess.endpoint_url,
-                sess.model,
-                [
-                    {"role": "system", "content": import_prompt},
-                    {"role": "user", "content": f"Document: {filename}\n\n{text}"},
-                ],
-                temperature=0.2,
-                max_tokens=2000,
-                headers=sess.headers,
-            )
-
-            # Parse JSON
-            raw = raw.strip()
-            if raw.startswith("```"):
-                raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
-
-            suggestions = json.loads(raw)
-            if isinstance(suggestions, list):
-                normalized = []
-                for s in suggestions:
-                    if not s:
-                        continue
-                    if isinstance(s, dict):
-                        s = dict(s)
-                        if s.get("text"):
-                            s["text"] = _strip_list_prefix(str(s["text"]))
-                        normalized.append(s)
-                    else:
-                        normalized.append({"text": _strip_list_prefix(str(s)), "category": "fact"})
-                suggestions = normalized
-            else:
-                suggestions = []
-
-            return {"suggestions": suggestions, "filename": filename}
-
-        except json.JSONDecodeError:
-            # Fallback: split by lines, stripping any "1.", "2)" markdown-list
-            # numbering the model added so saved memories don't keep the prefix.
-            lines = [_strip_list_prefix(l.strip()) for l in raw.splitlines() if l.strip() and len(l.strip()) > 5]
-            return {"suggestions": [{"text": l, "category": "fact"} for l in lines[:20]], "filename": filename}
-        except Exception as e:
-            logger.error(f"Memory import extraction failed: {e}")
-            raise HTTPException(502, f"LLM extraction failed: {str(e)}")
-
-    @router.post("/{memory_id}/pin")
-    def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)):
-        """Pin or unpin a memory. Pinned memories are always included in context."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["pinned"] = pinned
-                memory_manager.save(all_mem)
-                return {"ok": True, "pinned": pinned}
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    # Wildcard routes MUST come last — otherwise they swallow /import, /search, etc.
-    @router.get("/{memory_id}")
-    def get_memory_item(request: Request, memory_id: str):
-        """Get a specific memory item by ID."""
-        user = _owner(request)
-        memories = memory_manager.load(owner=user)
-        for memory in memories:
-            if memory["id"] == memory_id:
-                return {"memory": memory}
-
-        raise HTTPException(404, "Memory not found")
-
-    @router.put("/{memory_id}")
-    def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)):
-        """Update an existing memory item with new text and optional category."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-        for i, memory in enumerate(all_mem):
-            if memory["id"] == memory_id:
-                _verify_memory_owner(memory, user)
-                all_mem[i]["text"] = text.strip()
-                if category:
-                    all_mem[i]["category"] = category
-                all_mem[i]["timestamp"] = int(time.time())
-
-                memory_manager.save(all_mem)
-                # Sync vector index (remove old, add updated)
-                if memory_vector and memory_vector.healthy:
-                    memory_vector.remove(memory_id)
-                    memory_vector.add(memory_id, text.strip())
-                return {"ok": True, "message": "Memory updated successfully"}
-
-        raise HTTPException(404, f"Memory item {memory_id} not found")
-
-    @router.delete("/{memory_id}")
-    def delete_memory(request: Request, memory_id: str):
-        """Delete a memory item by its ID."""
-        user = _owner(request)
-        all_mem = memory_manager.load_all()
-
-        # Find and verify ownership before deleting
-        target = next((m for m in all_mem if m["id"] == memory_id), None)
-        if not target:
-            raise HTTPException(404, f"Memory item {memory_id} not found")
-        _verify_memory_owner(target, user)
-
-        all_mem = [m for m in all_mem if m["id"] != memory_id]
-        memory_manager.save(all_mem)
-        # Sync vector index
-        if memory_vector and memory_vector.healthy:
-            memory_vector.remove(memory_id)
-        return {"ok": True, "message": "Memory deleted successfully"}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/note/__init__.py

@@ -0,0 +1,5 @@
+"""Note route domain package (slice 2f, #4082/#4071).
+
+Contains note_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/note_routes.py re-exports from here.
+"""

routes/note/note_routes.py

@@ -0,0 +1,943 @@
+# routes/note_routes.py
+"""Google Keep-style notes / checklists API."""
+
+import json
+import uuid
+import logging
+from typing import Dict, Any, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from core.database import SessionLocal, Note
+from core.middleware import INTERNAL_TOOL_USER
+from src.auth_helpers import require_user
+from src.constants import DATA_DIR
+from src.upload_handler import reserve_upload_references
+from sqlalchemy.orm.attributes import flag_modified
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Request models
+# ---------------------------------------------------------------------------
+
+class NoteCreate(BaseModel):
+    title: str = ""
+    content: Optional[str] = None
+    items: Optional[list] = None
+    note_type: str = "note"
+    color: Optional[str] = None
+    label: Optional[str] = None
+    pinned: bool = False
+    due_date: Optional[str] = None
+    source: str = "user"
+    session_id: Optional[str] = None
+    image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
+    repeat: Optional[str] = "none"
+    sort_order: Optional[int] = None
+
+
+class NoteUpdate(BaseModel):
+    title: Optional[str] = None
+    content: Optional[str] = None
+    items: Optional[list] = None
+    note_type: Optional[str] = None
+    color: Optional[str] = None
+    label: Optional[str] = None
+    pinned: Optional[bool] = None
+    archived: Optional[bool] = None
+    due_date: Optional[str] = None
+    image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
+    repeat: Optional[str] = None
+    sort_order: Optional[int] = None
+    agent_session_id: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _note_to_dict(note: Note) -> Dict[str, Any]:
+    items = None
+    if note.items:
+        try:
+            items = json.loads(note.items)
+        except (json.JSONDecodeError, TypeError):
+            items = None
+    ai_cls = None
+    raw_ai = getattr(note, "ai_classification", None)
+    if raw_ai:
+        try:
+            ai_cls = json.loads(raw_ai)
+        except (json.JSONDecodeError, TypeError):
+            ai_cls = None
+    return {
+        "id": note.id,
+        "owner": note.owner,
+        "title": note.title,
+        "content": note.content,
+        "items": items,
+        "note_type": note.note_type,
+        "color": note.color,
+        "label": note.label,
+        "pinned": note.pinned,
+        "archived": note.archived,
+        "due_date": note.due_date,
+        "source": note.source,
+        "session_id": note.session_id,
+        "sort_order": note.sort_order or 0,
+        "image_url": note.image_url,
+        "gallery_id": getattr(note, "gallery_id", None),
+        "repeat": note.repeat or "none",
+        "ai_classification": ai_cls,
+        "ai_content_hash": getattr(note, "ai_content_hash", None),
+        "agent_session_id": getattr(note, "agent_session_id", None),
+        "created_at": note.created_at.isoformat() if note.created_at else None,
+        "updated_at": note.updated_at.isoformat() if note.updated_at else None,
+    }
+
+
+def _reminder_text_from_note(note: Note) -> tuple[str, str]:
+    """Return the reminder title/body from a stored note row."""
+    title = (note.title or "Note reminder").strip() or "Note reminder"
+    if note.items:
+        try:
+            items = json.loads(note.items)
+        except (json.JSONDecodeError, TypeError):
+            items = None
+        if isinstance(items, list):
+            pending: list[str] = []
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                if item.get("done") or item.get("checked"):
+                    continue
+                text = str(item.get("text") or "").strip()
+                if text:
+                    pending.append(text)
+            if pending:
+                shown = "\n".join(f"- {text}" for text in pending[:8])
+                extra = f"\n...and {len(pending) - 8} more" if len(pending) > 8 else ""
+                return title, f"Pending ({len(pending)}):\n{shown}{extra}"
+            return title, f"{len(items)} item{'s' if len(items) != 1 else ''}"
+    return title, (note.content or "").strip()[:400]
+
+
+
+# ---------------------------------------------------------------------------
+# Reminder dispatch — module-level so background tasks (built-in actions)
+# can call it directly without an HTTP roundtrip + auth cookie. The route
+# version below is a thin wrapper that pulls `owner` from the request.
+# ---------------------------------------------------------------------------
+
+# Scheduler reference — set by setup_note_routes() so dispatch_reminder can
+# push a parallel in-app notification (frontend polls the scheduler's queue
+# and fires real browser Notification(...) popups). Optional; works without it.
+_scheduler_ref = None
+
+
+async def dispatch_reminder(
+    title: str,
+    note_body: str,
+    note_id: str,
+    owner: str = "",
+    queue_browser: bool = True,
+    settings_override: dict | None = None,
+) -> dict:
+    """Fire a reminder via the configured channel (browser/email/ntfy/webhook).
+
+    Args:
+        title: short headline shown to the user
+        note_body: longer body text
+        note_id: stable id (used as tag/dedupe in browser notifications)
+        owner: the user this reminder belongs to — scopes SMTP config to
+               their account so we don't cross-leak credentials
+
+    Returns: {synthesis, email_sent, ntfy_sent}. Browser channel is wired via
+    the in-memory notification queue picked up by the frontend poller, so
+    nothing is "sent" synchronously for it — the channel just routes there.
+    """
+    from src.settings import load_settings
+    settings = {**load_settings(), **(settings_override or {})}
+    channel = settings.get("reminder_channel", "browser")
+    llm_on = bool(settings.get("reminder_llm_synthesis", False))
+    title = (title or "").strip()
+    note_body = (note_body or "").strip()
+    cache_key = str(note_id) if note_id else ""
+    cache = {}
+    cache_path = None
+    if cache_key:
+        try:
+            import json as _json
+            from datetime import datetime as _dt, timezone as _tz, timedelta as _td
+            from pathlib import Path as _P
+            _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
+            cache_path = _P(DATA_DIR) / f"note_pings_{_slug}.json"
+            if cache_path.exists():
+                cache = _json.loads(cache_path.read_text(encoding="utf-8"))
+            last = cache.get(cache_key)
+            if last:
+                last_channel = None
+                if isinstance(last, dict):
+                    last_channel = last.get("channel")
+                    last = last.get("at")
+                last_dt = _dt.fromisoformat(str(last))
+                if last_dt.tzinfo is None:
+                    last_dt = last_dt.replace(tzinfo=_tz.utc)
+                # Legacy cache values were plain timestamps and could be
+                # written by the frontend even when the email/ntfy send failed.
+                # Treat those as browser-only dedupe so email reminders can be
+                # retried by the backend scanner after a failed frontend path.
+                should_skip = last_dt >= _dt.now(_tz.utc) - _td(minutes=25)
+                if should_skip and channel in ("email", "ntfy", "webhook"):
+                    should_skip = last_channel == channel
+                if should_skip:
+                    return {
+                        "synthesis": None,
+                        "email_sent": False,
+                        "ntfy_sent": False,
+                        "webhook_sent": False,
+                        "browser_sent": True,
+                        "skipped": True,
+                    }
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: cache read failed: {_e}")
+
+    synthesis = None
+    _SYNTH_FAILED_TAG = "[utility model unavailable — no summary generated]"
+    if llm_on:
+        try:
+            from src.endpoint_resolver import resolve_endpoint
+            from src.llm_core import llm_call_async
+            from src.reminder_personas import synthesis_system_prompt
+            url, model, headers = resolve_endpoint("utility", owner=owner or None)
+            if not url:
+                url, model, headers = resolve_endpoint("default", owner=owner or None)
+            if url and model:
+                persona_id = (settings.get("reminder_llm_persona") or "").strip()
+                sys_prompt = synthesis_system_prompt(persona_id)
+                raw = await llm_call_async(
+                    url=url, model=model,
+                    messages=[
+                        {"role": "system", "content": sys_prompt},
+                        {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()},
+                    ],
+                    temperature=0.7, max_tokens=200, headers=headers, timeout=30,
+                )
+                from src.text_helpers import strip_think as _strip_think
+                # prose=True strips untagged "The user wants me to…" chain-of-thought.
+                # prompt_echo=True strips Qwen-style "Thinking Process:" / leaked
+                # prompt prefixes. Both are safe here because this is a
+                # one-sentence LLM-only output, not user-pasted content.
+                synthesis = _strip_think(raw or "", prose=True, prompt_echo=True)
+                # Reminder synthesis is supposed to be ONE sentence. Strip-think's
+                # paragraph-based heuristic misses cases where the model puts
+                # reasoning + answer on consecutive lines inside one paragraph
+                # (e.g. "I should write... [\n] You have one task waiting...").
+                # Walk lines, drop reasoning/prompt-echo lines, then keep the
+                # last surviving line — that's the actual warm sentence.
+                if synthesis:
+                    import re as _re
+                    # Tightened: target ACTUAL self-talk (model narrating what
+                    # it'll do) rather than any first-person sentence. The old
+                    # pattern killed legit warm sentences like "I'll see you
+                    # tomorrow" or "I should be done by then". New rules:
+                    #  • "I (need|should|have|'ll|will) (write|draft|reply|…)"
+                    #    only matches when followed by a TASK verb taking an
+                    #    OBJECT (so first-person + intransitive verb passes).
+                    #  • Self-instructional patterns the model emits verbatim:
+                    #    "I should write something that reminds them…",
+                    #    "I need to draft…", "Let me think…".
+                    #  • Explicit instructions echoed back from the prompt:
+                    #    "Keep it under 25 words", "No greetings".
+                    _reasoning = _re.compile(
+                        r"^\s*(?:"
+                        # "I should write/draft/compose…" with a task-object follow
+                        r"i (?:need|should|have|'ll|will|am going|am)\s+to\s+"
+                        r"(?:write|draft|compose|craft|generate|produce|create|"
+                        r"summarize|answer|provide|note|address|remind|output)"
+                        r"\s+(?:a |an |the |something|this|that|here|them|him|her|"
+                        r"you|user|reply|response|sentence|message|line|warm)|"
+                        # The model literally narrating about the user
+                        r"the user (?:wants|is asking|asks|needs|wrote|said|requested) (?:me )?(?:to|for|that|about|something)|"
+                        # "Let me [think/write/draft/…] (about/for/the …)"
+                        r"let me (?:think|write|draft|consider|note|see|check)\b\s+(?:about|for|the|this|that|if|whether)|"
+                        # "Looking at the/this/that …"
+                        r"looking at (?:the|this|that)\b|"
+                        # "Based on the/this/what …"
+                        r"based on (?:the|this|what|context|that)\b|"
+                        # Prompt-echo of length / style instructions
+                        r"keep it under \d+ words\b|"
+                        r"(?:no greetings|no preamble|no hashtags|just output the)\b"
+                        r").*",
+                        _re.IGNORECASE,
+                    )
+                    # Echo of the prompt's "Pending:" / "<N> pending" tail.
+                    _echo = _re.compile(
+                        r"^\s*(?:pending\s*[:.]|(?:\d+|one|two|three|four|five)\s+pending\b)",
+                        _re.IGNORECASE,
+                    )
+                    lines = [ln for ln in synthesis.splitlines() if ln.strip()]
+                    cleaned = [ln for ln in lines if not _reasoning.match(ln) and not _echo.match(ln)]
+                    if cleaned:
+                        # The model's actual answer is normally the LAST surviving
+                        # line — reasoning leads, answer trails.
+                        synthesis = cleaned[-1].strip()
+            else:
+                synthesis = _SYNTH_FAILED_TAG
+        except Exception as e:
+            logger.warning(f"Reminder LLM synthesis failed: {e}")
+            synthesis = _SYNTH_FAILED_TAG
+        if synthesis:
+            _s = synthesis.strip(); _low = _s.lower()
+            if (not _s or _low.startswith("error:") or _low.startswith("[error")
+                    or "operation failed" in _low
+                    or ("upstream" in _low and "failed" in _low)) and synthesis != _SYNTH_FAILED_TAG:
+                logger.warning(f"Reminder synthesis looked like an error, replacing: {_s[:120]!r}")
+                synthesis = _SYNTH_FAILED_TAG
+
+    email_sent = False
+    email_error = ""
+    if channel == "email":
+        try:
+            from routes.email_routes import _get_email_config, _smtp_ready
+            from email.mime.text import MIMEText
+            from email.mime.multipart import MIMEMultipart
+            from datetime import datetime as _dt
+            # `reminder_email_account_id` lets the user pick WHICH email
+            # account to send reminders from (when they have several
+            # configured in Integrations). Falls back to the default
+            # account when no explicit choice is saved.
+            _acc_id = (settings.get("reminder_email_account_id") or "").strip() or None
+            cfg = _get_email_config(account_id=_acc_id, owner=owner or "")
+            if not _smtp_ready(cfg):
+                try:
+                    from core.database import SessionLocal as _SL, EmailAccount as _EA
+                    from sqlalchemy import and_, or_
+                    db = _SL()
+                    try:
+                        q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
+                        if owner:
+                            unowned = or_(_EA.owner == None, _EA.owner == "")  # noqa: E711
+                            same_mailbox = or_(_EA.imap_user == owner, _EA.from_address == owner)
+                            q = q.filter(or_(_EA.owner == owner, and_(unowned, same_mailbox)))
+                        for row in q.order_by(_EA.is_default.desc(), _EA.created_at.asc()).all():
+                            trial = _get_email_config(account_id=row.id, owner=owner or "")
+                            if _smtp_ready(trial):
+                                cfg = trial
+                                break
+                    finally:
+                        db.close()
+                except Exception as _fallback_error:
+                    logger.debug(f"Reminder SMTP fallback lookup failed: {_fallback_error}")
+            from_addr = (cfg.get("from_address") or cfg.get("smtp_user") or "").strip()
+            recipient = (settings.get("reminder_email_to") or "").strip() or from_addr
+            # Loud diagnostic so we can see WHY a reminder didn't send (the
+            # previous "silently no-op when cfg has no smtp_host" was invisible).
+            logger.info(
+                "dispatch_reminder[email] note_id=%s owner=%r "
+                "has_smtp_host=%s has_smtp_user=%s has_from=%s has_recipient=%s",
+                note_id, owner,
+                bool(cfg.get("smtp_host")), bool(cfg.get("smtp_user")),
+                bool(from_addr), bool(recipient),
+            )
+            missing = []
+            if not cfg.get("smtp_host"):
+                missing.append("SMTP host")
+            if not cfg.get("smtp_user"):
+                missing.append("SMTP user")
+            if not (cfg.get("smtp_password") or cfg.get("oauth_provider")):
+                missing.append("SMTP credentials")
+            if not from_addr:
+                missing.append("from address")
+            if not recipient:
+                missing.append("recipient")
+            if missing:
+                email_error = "Missing " + ", ".join(missing)
+                logger.warning(
+                    "Reminder email not sent for note_id=%s account=%r: %s",
+                    note_id, cfg.get("account_name"), email_error,
+                )
+            else:
+                msg = MIMEMultipart("alternative")
+                msg["From"] = from_addr
+                msg["To"] = recipient
+                _t = title or 'Note'
+                _t = _t[len('Reminder:'):].strip() if _t.lower().startswith('reminder:') else _t
+                msg["Subject"] = f"Reminder (Odysseus): {_t}"
+                msg["Date"] = _dt.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+                msg["X-Odysseus-Origin"] = "odysseus-ui"
+                msg["X-Odysseus-Kind"] = "reminder"
+                msg["X-Odysseus-Ref"] = str(note_id)
+                # Body shape: synthesis (warm sentence) → blank line → bold
+                # title header → note details. The title was previously only
+                # in the subject line, so the email read like a faceless
+                # to-do list with no anchor to which note triggered it.
+                _body_chunks = []
+                if synthesis:
+                    _body_chunks.append(synthesis)
+                if _t:
+                    _body_chunks.append(_t)
+                if note_body:
+                    _body_chunks.append(note_body)
+                plain = "\n\n".join(_body_chunks) if _body_chunks else title
+                msg.attach(MIMEText(plain, "plain", "utf-8"))
+
+                def _smtp_send():
+                    from routes.email_helpers import _send_smtp_message
+                    _send_smtp_message(cfg, from_addr, [recipient], msg.as_string())
+
+                import asyncio as _aio
+                await _aio.to_thread(_smtp_send)
+                email_sent = True
+        except Exception as e:
+            email_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder email send failed: {e}")
+
+    webhook_sent = False
+    webhook_error = ""
+    if channel == "webhook":
+        try:
+            import httpx
+            import json as _wjson
+            from src.integrations import load_integrations
+            # Built-in payload defaults for known presets so users don't have
+            # to configure a template just to use a standard service.
+            _PRESET_TEMPLATE_DEFAULTS = {
+                "discord_webhook": '{"embeds": [{"title": "{{title}}", "description": "{{message}}", "color": 5793266}]}',
+            }
+            intg_id = settings.get("reminder_webhook_integration_id", "").strip()
+            template = settings.get("reminder_webhook_payload_template", "").strip()
+            if not intg_id:
+                webhook_error = "No webhook integration selected"
+            else:
+                intg = next(
+                    (i for i in load_integrations()
+                     if i.get("id") == intg_id and i.get("base_url")),
+                    None,
+                )
+                if not intg:
+                    webhook_error = f"Integration {intg_id!r} not found or missing base URL"
+                else:
+                    # Fall back to a built-in default for known presets so
+                    # users don't have to configure a template for standard
+                    # services like Discord.
+                    if not template:
+                        template = _PRESET_TEMPLATE_DEFAULTS.get(intg.get("preset", ""), "")
+                    if not template:
+                        webhook_error = "No payload template configured"
+                    else:
+                        # Render template: JSON-escape the values so the result
+                        # is always valid JSON regardless of special characters.
+                        # dumps() returns `"value"` — strip outer quotes.
+                        msg = (synthesis or note_body or title or "Reminder")[:4000]
+                        _t = _wjson.dumps(title or "Reminder")[1:-1]
+                        _m = _wjson.dumps(msg)[1:-1]
+                        rendered = template.replace("{{title}}", _t).replace("{{message}}", _m)
+                        hdrs = {"Content-Type": "application/json"}
+                        api_key = intg.get("api_key", "")
+                        auth_type = (intg.get("auth_type") or "none").lower()
+                        if api_key:
+                            if auth_type == "bearer":
+                                hdrs["Authorization"] = f"Bearer {api_key}"
+                            elif auth_type == "header":
+                                hdrs[intg.get("auth_header") or "Authorization"] = api_key
+                        url = intg["base_url"].rstrip("/")
+                        # SSRF guard — matches the pattern used by webhook_routes,
+                        # CalDAV, search, and embeddings. Blocks link-local / metadata
+                        # addresses (169.254.x.x) by default; set
+                        # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true to also block
+                        # RFC-1918 ranges for locked-down deployments.
+                        import os as _os
+                        from src.url_safety import check_outbound_url as _chk
+                        _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
+                        _ok, _reason = _chk(url, block_private=_block)
+                        if not _ok:
+                            webhook_error = f"Webhook URL rejected: {_reason}"
+                        else:
+                            async with httpx.AsyncClient(timeout=10.0) as client:
+                                resp = await client.post(url, content=rendered.encode(), headers=hdrs)
+                                webhook_sent = resp.is_success
+                                if not webhook_sent:
+                                    webhook_error = f"Webhook returned HTTP {resp.status_code}"
+        except Exception as e:
+            webhook_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder webhook send failed: {e}")
+
+    ntfy_sent = False
+    ntfy_error = ""
+    if channel == "ntfy":
+        try:
+            from src.integrations import load_integrations
+            import httpx
+            intg = next(
+                (i for i in load_integrations()
+                 if i.get("preset") == "ntfy" and i.get("enabled", True) and i.get("base_url")),
+                None,
+            )
+            if intg:
+                base = intg["base_url"].rstrip("/")
+                topic = settings.get("reminder_ntfy_topic") or "reminders"
+                ntfy_body = synthesis or note_body or title
+                # ntfy Title is an ASCII HTTP header; sanitize Unicode and cap its length.
+                _clean_title = (title or "Reminder").encode("ascii", "replace").decode("ascii")[:200]
+                hdrs = {"Title": _clean_title, "Priority": "high", "Tags": "bell"}
+                api_key = intg.get("api_key", "")
+                if api_key:
+                    hdrs["Authorization"] = f"Bearer {api_key}"
+                # SSRF guard — same check (and env knob) as the webhook branch
+                # above: link-local / metadata addresses are always rejected;
+                # REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true also blocks RFC-1918
+                # so a ntfy base_url can't be pointed at internal services.
+                import os as _os
+                from src.url_safety import check_outbound_url as _chk
+                _block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
+                _ok, _reason = _chk(f"{base}/{topic}", block_private=_block)
+                if not _ok:
+                    ntfy_error = f"ntfy URL rejected: {_reason}"
+                else:
+                    async with httpx.AsyncClient(timeout=10.0) as client:
+                        resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
+                        ntfy_sent = resp.is_success
+                        if not ntfy_sent:
+                            ntfy_error = f"ntfy returned HTTP {resp.status_code}"
+            else:
+                ntfy_error = "No enabled ntfy integration"
+        except Exception as e:
+            ntfy_error = str(e) or e.__class__.__name__
+            logger.warning(f"Reminder ntfy send failed: {e}")
+
+    # In-app browser notification ALWAYS fires (regardless of channel). The
+    # frontend polls `/api/tasks/notifications` and turns any entry with a
+    # `body` into a real `Notification(...)` — same surface as task-success
+    # popups. Lets the user see reminders inside the app even when the
+    # primary channel is email/ntfy and the tab is open.
+    browser_sent = False
+    local_browser_sent = (not queue_browser and channel == "browser")
+    if queue_browser and _scheduler_ref is not None:
+        try:
+            _scheduler_ref.add_notification(
+                task_name=title or "Reminder",
+                status="success",
+                task_id=f"reminder-{note_id}",
+                owner=owner or None,
+                body=(synthesis or note_body or title or "").strip()[:500] or "Reminder",
+            )
+            browser_sent = True
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: in-app notif push failed: {_e}")
+
+    # Dedupe across paths: write to the same cache file `action_ping_notes`
+    # reads, so the background scanner's REPING_MIN window suppresses a
+    # second send for the same note within 25 min. Without this, a note
+    # whose due_date fires while the user has the app open got TWO emails
+    # (frontend-fired here + background-fired by ping_notes 0–5 min later).
+    if (email_sent or ntfy_sent or webhook_sent or browser_sent or local_browser_sent) and note_id:
+        try:
+            import json as _json
+            from datetime import datetime as _dt, timezone as _tz
+            from pathlib import Path as _P
+            # Per-owner cache so the scanner's prune step on user A's run
+            # doesn't drop user B's just-fired entry (review C4).
+            _STATE = cache_path
+            if _STATE is None:
+                _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
+                _STATE = _P(DATA_DIR) / f"note_pings_{_slug}.json"
+            _STATE.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                _cache = cache or (_json.loads(_STATE.read_text(encoding="utf-8")) if _STATE.exists() else {})
+            except Exception:
+                _cache = {}
+            sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "webhook" if webhook_sent else "browser"
+            _cache[cache_key or str(note_id)] = {
+                "at": _dt.now(_tz.utc).isoformat(),
+                "channel": sent_channel,
+            }
+            _STATE.write_text(_json.dumps(_cache), encoding="utf-8")
+        except Exception as _e:
+            logger.debug(f"dispatch_reminder: cache write failed: {_e}")
+
+    return {
+        "channel": channel,
+        "synthesis": synthesis,
+        "email_sent": email_sent,
+        "email_error": email_error,
+        "ntfy_sent": ntfy_sent,
+        "ntfy_error": ntfy_error,
+        "webhook_sent": webhook_sent,
+        "webhook_error": webhook_error,
+        "browser_sent": browser_sent or local_browser_sent,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Router factory
+# ---------------------------------------------------------------------------
+
+def setup_note_routes(task_scheduler=None, upload_handler=None):
+    # Expose the scheduler to module-level `dispatch_reminder` so reminders
+    # can also push to the in-app notification queue (the polling system
+    # turns each entry into a real browser Notification + the existing
+    # tasks-tab badge / dot system).
+    global _scheduler_ref
+    _scheduler_ref = task_scheduler
+
+    router = APIRouter(prefix="/api/notes", tags=["notes"])
+
+    def _owner(request: Request) -> Optional[str]:
+        # require_user, not bare get_current_user: a request that reaches
+        # these owner-scoped routes with NO identity (auth-middleware
+        # regression, SSRF from a sibling service) must fail closed (401)
+        # when auth is configured — not be treated as the single-user mode
+        # and handed blanket access to every account's notes. The documented
+        # anonymous modes (AUTH_ENABLED=false, LOCALHOST_BYPASS on loopback,
+        # unconfigured first-run) still resolve to None, the single-user
+        # path. fire_reminder below already gated this way; the CRUD routes
+        # did not.
+        return require_user(request) or None
+
+    def _reserve_note_uploads(owner: Optional[str], *values) -> None:
+        missing_id = reserve_upload_references(upload_handler, owner, *values)
+        if missing_id:
+            raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}")
+
+    def _is_admin_or_single_user(request: Request, user: str | None) -> bool:
+        if user == INTERNAL_TOOL_USER:
+            return True
+        if not user:
+            # require_user() already admitted this request, which only happens
+            # for auth-disabled, loopback-bypass, or unconfigured single-user
+            # modes. There is no separate non-admin account boundary there.
+            return True
+        try:
+            from core.auth import AuthManager
+            auth_mgr = getattr(request.app.state, "auth_manager", None) or AuthManager()
+            if not getattr(auth_mgr, "is_configured", True):
+                return True
+            return bool(auth_mgr.is_admin(user))
+        except Exception:
+            return False
+
+    # --- LIST ---
+    @router.get("")
+    def list_notes(
+        request: Request,
+        archived: Optional[bool] = None,
+        label: Optional[str] = None,
+    ):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            q = db.query(Note)
+            if user is not None:
+                q = q.filter(Note.owner == user)
+            if archived is not None:
+                q = q.filter(Note.archived == archived)
+            else:
+                q = q.filter(Note.archived == False)
+            if label:
+                q = q.filter(Note.label == label)
+            # Archived view: most recently archived first. Active view: pin + manual order.
+            if archived is True:
+                notes = q.order_by(Note.updated_at.desc()).all()
+            else:
+                notes = q.order_by(Note.pinned.desc(), Note.sort_order.asc(), Note.updated_at.desc()).all()
+            return {"notes": [_note_to_dict(n) for n in notes]}
+        finally:
+            db.close()
+
+    # --- CREATE ---
+    @router.post("")
+    def create_note(request: Request, body: NoteCreate):
+        user = _owner(request)
+        _reserve_note_uploads(
+            user,
+            body.image_url,
+            body.color,
+            body.content,
+            json.dumps(body.items) if body.items is not None else None,
+        )
+        db = SessionLocal()
+        try:
+            note = Note(
+                id=str(uuid.uuid4()),
+                owner=user,
+                title=body.title,
+                content=body.content,
+                items=json.dumps(body.items) if body.items is not None else None,
+                note_type=body.note_type,
+                color=body.color,
+                label=body.label,
+                pinned=body.pinned,
+                due_date=body.due_date,
+                source=body.source,
+                session_id=body.session_id,
+                image_url=body.image_url,
+                gallery_id=body.gallery_id,
+                repeat=body.repeat or "none",
+                sort_order=body.sort_order if body.sort_order is not None else 0,
+            )
+            db.add(note)
+            db.commit()
+            db.refresh(note)
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- GET ONE ---
+    @router.get("/{note_id}")
+    def get_note(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- UPDATE ---
+    @router.put("/{note_id}")
+    def update_note(request: Request, note_id: str, body: NoteUpdate):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+
+            _reserve_note_uploads(
+                user,
+                body.image_url,
+                body.color,
+                body.content,
+                json.dumps(body.items) if body.items is not None else None,
+            )
+            if body.title is not None:
+                note.title = body.title
+            if body.content is not None:
+                note.content = body.content
+            if body.items is not None:
+                note.items = json.dumps(body.items)
+                flag_modified(note, "items")
+            if body.note_type is not None:
+                note.note_type = body.note_type
+            if body.color is not None:
+                note.color = body.color
+            if body.label is not None:
+                note.label = body.label
+            if body.pinned is not None:
+                note.pinned = body.pinned
+            if body.archived is not None:
+                note.archived = body.archived
+            if body.due_date is not None:
+                note.due_date = body.due_date
+            if body.image_url is not None:
+                note.image_url = body.image_url
+            if body.gallery_id is not None:
+                note.gallery_id = body.gallery_id
+            if body.repeat is not None:
+                note.repeat = body.repeat
+            if body.sort_order is not None:
+                note.sort_order = body.sort_order
+            if body.agent_session_id is not None:
+                note.agent_session_id = body.agent_session_id
+
+            db.commit()
+            db.refresh(note)
+            return _note_to_dict(note)
+        finally:
+            db.close()
+
+    # --- DELETE ---
+    @router.delete("/{note_id}")
+    def delete_note(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            db.delete(note)
+            db.commit()
+            return {"ok": True}
+        finally:
+            db.close()
+
+    # --- TOGGLE PIN ---
+    @router.post("/{note_id}/pin")
+    def toggle_pin(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            note.pinned = not note.pinned
+            db.commit()
+            return {"ok": True, "pinned": note.pinned}
+        finally:
+            db.close()
+
+    # --- TOGGLE ARCHIVE ---
+    @router.post("/{note_id}/archive")
+    def toggle_archive(request: Request, note_id: str):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            note.archived = not note.archived
+            db.commit()
+            return {"ok": True, "archived": note.archived}
+        finally:
+            db.close()
+
+    # --- TOGGLE CHECKLIST ITEM ---
+    @router.post("/{note_id}/items/{index}/toggle")
+    def toggle_item(request: Request, note_id: str, index: int):
+        user = _owner(request)
+        db = SessionLocal()
+        try:
+            note = db.query(Note).filter(Note.id == note_id).first()
+            if not note:
+                raise HTTPException(404, "Note not found")
+            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
+            # let any user touch a row whose owner field was null/empty.
+            if user is not None and note.owner != user:
+                raise HTTPException(404, "Note not found")
+            if not note.items:
+                raise HTTPException(400, "Note has no checklist items")
+            items = json.loads(note.items)
+            if index < 0 or index >= len(items):
+                raise HTTPException(400, f"Item index {index} out of range")
+            items[index]["done"] = not items[index].get("done", False)
+            note.items = json.dumps(items)
+            flag_modified(note, "items")
+            db.commit()
+            return {"ok": True, "items": items}
+        finally:
+            db.close()
+
+    # --- FIRE REMINDER ---
+    @router.post("/fire-reminder")
+    async def fire_reminder(request: Request):
+        """Dispatch a reminder according to user settings.
+
+        Called by the frontend when a reminder fires. Optionally generates an
+        LLM synthesis line and/or sends an email through configured SMTP.
+        Returns {synthesis, email_sent}.
+        """
+        # Gate against anonymous callers — LLM synthesis can burn tokens.
+        user = require_user(request)
+        body = await request.json()
+        note_id = str(body.get("note_id") or "").strip()
+        if not note_id:
+            raise HTTPException(400, "note_id required")
+
+        caller = _owner(request)
+        is_test = note_id.startswith("test-")
+        is_admin = _is_admin_or_single_user(request, user or caller)
+        _override: dict = {}
+        if is_test:
+            if not is_admin:
+                raise HTTPException(403, "Admin only")
+            title = (body.get("title") or "Test Reminder").strip() or "Test Reminder"
+            note_body = (body.get("body") or "").strip()
+            # Optional overrides let the admin settings test button pass the
+            # current UI values directly so it never races a pending save.
+            if body.get("channel"):
+                _override["reminder_channel"] = body["channel"]
+            if body.get("webhook_integration_id"):
+                _override["reminder_webhook_integration_id"] = body["webhook_integration_id"]
+            if body.get("webhook_payload_template"):
+                _override["reminder_webhook_payload_template"] = body["webhook_payload_template"]
+            # Mirror the in-UI AI Synthesis toggle + persona so the test
+            # actually exercises the synthesis path before/without a Save.
+            if "llm_synthesis" in body:
+                _override["reminder_llm_synthesis"] = bool(body["llm_synthesis"])
+            if "llm_persona" in body:
+                _override["reminder_llm_persona"] = str(body["llm_persona"] or "")
+        else:
+            db = SessionLocal()
+            try:
+                note = db.query(Note).filter(Note.id == note_id).first()
+                if not note:
+                    raise HTTPException(404, "Note not found")
+                if caller is not None and note.owner != caller:
+                    raise HTTPException(404, "Note not found")
+                title, note_body = _reminder_text_from_note(note)
+            finally:
+                db.close()
+
+        return await dispatch_reminder(
+            title=title, note_body=note_body, note_id=note_id,
+            owner=caller or "",
+            queue_browser=False,
+            settings_override=_override or None,
+        )
+
+    # --- REORDER NOTES ---
+    @router.post("/reorder")
+    async def reorder_notes(request: Request):
+        """Update sort_order for a list of note IDs in the order provided."""
+        user = _owner(request)
+        body = await request.json()
+        ids = body.get("ids", [])
+        if not isinstance(ids, list):
+            raise HTTPException(400, "ids must be a list")
+        # v2 review HIGH-12: drop the legacy `(owner == user) | (owner ==
+        # None)` OR which let an authenticated user silently reorder
+        # every legacy-null-owner note belonging to other accounts. In
+        # an unconfigured (single-user) auth deploy the OR is still safe
+        # because there's no second user to attack; we keep that branch
+        # explicit and gated on AuthManager.is_configured.
+        try:
+            from core.auth import AuthManager
+            _allow_null = not AuthManager().is_configured
+        except Exception:
+            _allow_null = False
+        db = SessionLocal()
+        try:
+            for i, nid in enumerate(ids):
+                q = db.query(Note).filter(Note.id == nid)
+                if user is not None:
+                    if _allow_null:
+                        q = q.filter((Note.owner == user) | (Note.owner == None))  # noqa: E711
+                    else:
+                        q = q.filter(Note.owner == user)
+                note = q.first()
+                if note:
+                    note.sort_order = i
+            db.commit()
+            return {"ok": True, "count": len(ids)}
+        finally:
+            db.close()
+
+    return router

routes/note_routes.py

@@ -1,741 +1,18 @@
-# routes/note_routes.py
-"""Google Keep-style notes / checklists API."""
+"""Backward-compat shim — canonical location is routes/note/note_routes.py.
 
-import json
-import uuid
-import logging
-from typing import Dict, Any, Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.note_routes``, ``from routes.note_routes import X``,
+``importlib.import_module("routes.note_routes")``, and the
+``import ... as note_routes`` + ``monkeypatch.setattr(note_routes, "SessionLocal",
+...)`` pattern used by test_note_reminder_fire_scope.py /
+test_notes_fail_closed_auth.py all operate on the *same* object the
+application actually uses. Keeps existing import paths working after
+slice 2f (#4082/#4071). Source-introspection tests read the canonical file
+by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Request
-from pydantic import BaseModel
+import sys as _sys
 
-from core.database import SessionLocal, Note
-from src.auth_helpers import get_current_user
-from sqlalchemy.orm.attributes import flag_modified
+from routes.note import note_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Request models
-# ---------------------------------------------------------------------------
-
-class NoteCreate(BaseModel):
-    title: str = ""
-    content: Optional[str] = None
-    items: Optional[list] = None
-    note_type: str = "note"
-    color: Optional[str] = None
-    label: Optional[str] = None
-    pinned: bool = False
-    due_date: Optional[str] = None
-    source: str = "user"
-    session_id: Optional[str] = None
-    image_url: Optional[str] = None
-    repeat: Optional[str] = "none"
-    sort_order: Optional[int] = None
-
-
-class NoteUpdate(BaseModel):
-    title: Optional[str] = None
-    content: Optional[str] = None
-    items: Optional[list] = None
-    note_type: Optional[str] = None
-    color: Optional[str] = None
-    label: Optional[str] = None
-    pinned: Optional[bool] = None
-    archived: Optional[bool] = None
-    due_date: Optional[str] = None
-    image_url: Optional[str] = None
-    repeat: Optional[str] = None
-    sort_order: Optional[int] = None
-    agent_session_id: Optional[str] = None
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _note_to_dict(note: Note) -> Dict[str, Any]:
-    items = None
-    if note.items:
-        try:
-            items = json.loads(note.items)
-        except (json.JSONDecodeError, TypeError):
-            items = None
-    ai_cls = None
-    raw_ai = getattr(note, "ai_classification", None)
-    if raw_ai:
-        try:
-            ai_cls = json.loads(raw_ai)
-        except (json.JSONDecodeError, TypeError):
-            ai_cls = None
-    return {
-        "id": note.id,
-        "owner": note.owner,
-        "title": note.title,
-        "content": note.content,
-        "items": items,
-        "note_type": note.note_type,
-        "color": note.color,
-        "label": note.label,
-        "pinned": note.pinned,
-        "archived": note.archived,
-        "due_date": note.due_date,
-        "source": note.source,
-        "session_id": note.session_id,
-        "sort_order": note.sort_order or 0,
-        "image_url": note.image_url,
-        "repeat": note.repeat or "none",
-        "ai_classification": ai_cls,
-        "ai_content_hash": getattr(note, "ai_content_hash", None),
-        "agent_session_id": getattr(note, "agent_session_id", None),
-        "created_at": note.created_at.isoformat() if note.created_at else None,
-        "updated_at": note.updated_at.isoformat() if note.updated_at else None,
-    }
-
-
-
-# ---------------------------------------------------------------------------
-# Reminder dispatch — module-level so background tasks (built-in actions)
-# can call it directly without an HTTP roundtrip + auth cookie. The route
-# version below is a thin wrapper that pulls `owner` from the request.
-# ---------------------------------------------------------------------------
-
-# Scheduler reference — set by setup_note_routes() so dispatch_reminder can
-# push a parallel in-app notification (frontend polls the scheduler's queue
-# and fires real browser Notification(...) popups). Optional; works without it.
-_scheduler_ref = None
-
-
-async def dispatch_reminder(
-    title: str,
-    note_body: str,
-    note_id: str,
-    owner: str = "",
-    queue_browser: bool = True,
-) -> dict:
-    """Fire a reminder via the configured channel (browser/email/ntfy).
-
-    Args:
-        title: short headline shown to the user
-        note_body: longer body text
-        note_id: stable id (used as tag/dedupe in browser notifications)
-        owner: the user this reminder belongs to — scopes SMTP config to
-               their account so we don't cross-leak credentials
-
-    Returns: {synthesis, email_sent, ntfy_sent}. Browser channel is wired via
-    the in-memory notification queue picked up by the frontend poller, so
-    nothing is "sent" synchronously for it — the channel just routes there.
-    """
-    from src.settings import load_settings
-    settings = load_settings()
-    channel = settings.get("reminder_channel", "browser")
-    llm_on = bool(settings.get("reminder_llm_synthesis", False))
-    title = (title or "").strip()
-    note_body = (note_body or "").strip()
-    cache_key = str(note_id) if note_id else ""
-    cache = {}
-    cache_path = None
-    if cache_key:
-        try:
-            import json as _json
-            from datetime import datetime as _dt, timezone as _tz, timedelta as _td
-            from pathlib import Path as _P
-            _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
-            cache_path = _P(f"data/note_pings_{_slug}.json")
-            if cache_path.exists():
-                cache = _json.loads(cache_path.read_text())
-            last = cache.get(cache_key)
-            if last:
-                last_channel = None
-                if isinstance(last, dict):
-                    last_channel = last.get("channel")
-                    last = last.get("at")
-                last_dt = _dt.fromisoformat(str(last))
-                if last_dt.tzinfo is None:
-                    last_dt = last_dt.replace(tzinfo=_tz.utc)
-                # Legacy cache values were plain timestamps and could be
-                # written by the frontend even when the email/ntfy send failed.
-                # Treat those as browser-only dedupe so email reminders can be
-                # retried by the backend scanner after a failed frontend path.
-                should_skip = last_dt >= _dt.now(_tz.utc) - _td(minutes=25)
-                if should_skip and channel in ("email", "ntfy"):
-                    should_skip = last_channel == channel
-                if should_skip:
-                    return {
-                        "synthesis": None,
-                        "email_sent": False,
-                        "ntfy_sent": False,
-                        "browser_sent": True,
-                        "skipped": True,
-                    }
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: cache read failed: {_e}")
-
-    synthesis = None
-    _SYNTH_FAILED_TAG = "[utility model unavailable — no summary generated]"
-    if llm_on:
-        try:
-            from src.endpoint_resolver import resolve_endpoint
-            from src.llm_core import llm_call_async
-            url, model, headers = resolve_endpoint("utility")
-            if not url:
-                url, model, headers = resolve_endpoint("default")
-            if url and model:
-                raw = await llm_call_async(
-                    url=url, model=model,
-                    messages=[
-                        {"role": "system", "content": "You are a reminder assistant. Write a single short, warm, motivating sentence (max 25 words) reminding the user about the note below. Do not add greetings, preamble, or hashtags. Output only the sentence."},
-                        {"role": "user", "content": f"Title: {title}\n\n{note_body}".strip()},
-                    ],
-                    temperature=0.7, max_tokens=200, headers=headers, timeout=30,
-                )
-                from src.text_helpers import strip_think as _strip_think
-                # prose=True strips untagged "The user wants me to…" chain-of-thought.
-                # prompt_echo=True strips Qwen-style "Thinking Process:" / leaked
-                # prompt prefixes. Both are safe here because this is a
-                # one-sentence LLM-only output, not user-pasted content.
-                synthesis = _strip_think(raw or "", prose=True, prompt_echo=True)
-                # Reminder synthesis is supposed to be ONE sentence. Strip-think's
-                # paragraph-based heuristic misses cases where the model puts
-                # reasoning + answer on consecutive lines inside one paragraph
-                # (e.g. "I should write... [\n] You have one task waiting...").
-                # Walk lines, drop reasoning/prompt-echo lines, then keep the
-                # last surviving line — that's the actual warm sentence.
-                if synthesis:
-                    import re as _re
-                    # Tightened: target ACTUAL self-talk (model narrating what
-                    # it'll do) rather than any first-person sentence. The old
-                    # pattern killed legit warm sentences like "I'll see you
-                    # tomorrow" or "I should be done by then". New rules:
-                    #  • "I (need|should|have|'ll|will) (write|draft|reply|…)"
-                    #    only matches when followed by a TASK verb taking an
-                    #    OBJECT (so first-person + intransitive verb passes).
-                    #  • Self-instructional patterns the model emits verbatim:
-                    #    "I should write something that reminds them…",
-                    #    "I need to draft…", "Let me think…".
-                    #  • Explicit instructions echoed back from the prompt:
-                    #    "Keep it under 25 words", "No greetings".
-                    _reasoning = _re.compile(
-                        r"^\s*(?:"
-                        # "I should write/draft/compose…" with a task-object follow
-                        r"i (?:need|should|have|'ll|will|am going|am)\s+to\s+"
-                        r"(?:write|draft|compose|craft|generate|produce|create|"
-                        r"summarize|answer|provide|note|address|remind|output)"
-                        r"\s+(?:a |an |the |something|this|that|here|them|him|her|"
-                        r"you|user|reply|response|sentence|message|line|warm)|"
-                        # The model literally narrating about the user
-                        r"the user (?:wants|is asking|asks|needs|wrote|said|requested) (?:me )?(?:to|for|that|about|something)|"
-                        # "Let me [think/write/draft/…] (about/for/the …)"
-                        r"let me (?:think|write|draft|consider|note|see|check)\b\s+(?:about|for|the|this|that|if|whether)|"
-                        # "Looking at the/this/that …"
-                        r"looking at (?:the|this|that)\b|"
-                        # "Based on the/this/what …"
-                        r"based on (?:the|this|what|context|that)\b|"
-                        # Prompt-echo of length / style instructions
-                        r"keep it under \d+ words\b|"
-                        r"(?:no greetings|no preamble|no hashtags|just output the)\b"
-                        r").*",
-                        _re.IGNORECASE,
-                    )
-                    # Echo of the prompt's "Pending:" / "<N> pending" tail.
-                    _echo = _re.compile(
-                        r"^\s*(?:pending\s*[:.]|(?:\d+|one|two|three|four|five)\s+pending\b)",
-                        _re.IGNORECASE,
-                    )
-                    lines = [ln for ln in synthesis.splitlines() if ln.strip()]
-                    cleaned = [ln for ln in lines if not _reasoning.match(ln) and not _echo.match(ln)]
-                    if cleaned:
-                        # The model's actual answer is normally the LAST surviving
-                        # line — reasoning leads, answer trails.
-                        synthesis = cleaned[-1].strip()
-            else:
-                synthesis = _SYNTH_FAILED_TAG
-        except Exception as e:
-            logger.warning(f"Reminder LLM synthesis failed: {e}")
-            synthesis = _SYNTH_FAILED_TAG
-        if synthesis:
-            _s = synthesis.strip(); _low = _s.lower()
-            if (not _s or _low.startswith("error:") or _low.startswith("[error")
-                    or "operation failed" in _low
-                    or ("upstream" in _low and "failed" in _low)) and synthesis != _SYNTH_FAILED_TAG:
-                logger.warning(f"Reminder synthesis looked like an error, replacing: {_s[:120]!r}")
-                synthesis = _SYNTH_FAILED_TAG
-
-    email_sent = False
-    email_error = ""
-    if channel == "email":
-        try:
-            from routes.email_routes import _get_email_config
-            from email.mime.text import MIMEText
-            from email.mime.multipart import MIMEMultipart
-            from datetime import datetime as _dt
-            # `reminder_email_account_id` lets the user pick WHICH email
-            # account to send reminders from (when they have several
-            # configured in Integrations). Falls back to the default
-            # account when no explicit choice is saved.
-            _acc_id = (settings.get("reminder_email_account_id") or "").strip() or None
-            cfg = _get_email_config(account_id=_acc_id, owner=owner or "")
-            if not (cfg.get("smtp_host") and cfg.get("smtp_user") and cfg.get("smtp_password")):
-                try:
-                    from core.database import SessionLocal as _SL, EmailAccount as _EA
-                    from sqlalchemy import and_, or_
-                    db = _SL()
-                    try:
-                        q = db.query(_EA).filter(_EA.enabled == True)  # noqa: E712
-                        if owner:
-                            unowned = or_(_EA.owner == None, _EA.owner == "")  # noqa: E711
-                            same_mailbox = or_(_EA.imap_user == owner, _EA.from_address == owner)
-                            q = q.filter(or_(_EA.owner == owner, and_(unowned, same_mailbox)))
-                        for row in q.order_by(_EA.is_default.desc(), _EA.created_at.asc()).all():
-                            trial = _get_email_config(account_id=row.id, owner=owner or "")
-                            if trial.get("smtp_host") and trial.get("smtp_user") and trial.get("smtp_password"):
-                                cfg = trial
-                                break
-                    finally:
-                        db.close()
-                except Exception as _fallback_error:
-                    logger.debug(f"Reminder SMTP fallback lookup failed: {_fallback_error}")
-            from_addr = (cfg.get("from_address") or cfg.get("smtp_user") or "").strip()
-            recipient = (settings.get("reminder_email_to") or "").strip() or from_addr
-            # Loud diagnostic so we can see WHY a reminder didn't send (the
-            # previous "silently no-op when cfg has no smtp_host" was invisible).
-            logger.info(
-                f"dispatch_reminder[email] note_id={note_id} owner={owner!r} "
-                f"smtp_host={cfg.get('smtp_host')!r} smtp_user={cfg.get('smtp_user')!r} "
-                f"from={from_addr!r} recipient={recipient!r} "
-                f"account_name={cfg.get('account_name')!r}"
-            )
-            missing = []
-            if not cfg.get("smtp_host"):
-                missing.append("SMTP host")
-            if not cfg.get("smtp_user"):
-                missing.append("SMTP user")
-            if not cfg.get("smtp_password"):
-                missing.append("SMTP password")
-            if not from_addr:
-                missing.append("from address")
-            if not recipient:
-                missing.append("recipient")
-            if missing:
-                email_error = "Missing " + ", ".join(missing)
-                logger.warning(
-                    "Reminder email not sent for note_id=%s account=%r: %s",
-                    note_id, cfg.get("account_name"), email_error,
-                )
-            else:
-                msg = MIMEMultipart("alternative")
-                msg["From"] = from_addr
-                msg["To"] = recipient
-                _t = title or 'Note'
-                _t = _t[len('Reminder:'):].strip() if _t.lower().startswith('reminder:') else _t
-                msg["Subject"] = f"Reminder (Odysseus): {_t}"
-                msg["Date"] = _dt.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-                msg["X-Odysseus-Origin"] = "odysseus-ui"
-                msg["X-Odysseus-Kind"] = "reminder"
-                msg["X-Odysseus-Ref"] = str(note_id)
-                # Body shape: synthesis (warm sentence) → blank line → bold
-                # title header → note details. The title was previously only
-                # in the subject line, so the email read like a faceless
-                # to-do list with no anchor to which note triggered it.
-                _body_chunks = []
-                if synthesis:
-                    _body_chunks.append(synthesis)
-                if _t:
-                    _body_chunks.append(_t)
-                if note_body:
-                    _body_chunks.append(note_body)
-                plain = "\n\n".join(_body_chunks) if _body_chunks else title
-                msg.attach(MIMEText(plain, "plain", "utf-8"))
-
-                def _smtp_send():
-                    from routes.email_helpers import _send_smtp_message
-                    _send_smtp_message(cfg, from_addr, [recipient], msg.as_string())
-
-                import asyncio as _aio
-                await _aio.to_thread(_smtp_send)
-                email_sent = True
-        except Exception as e:
-            email_error = str(e) or e.__class__.__name__
-            logger.warning(f"Reminder email send failed: {e}")
-
-    ntfy_sent = False
-    ntfy_error = ""
-    if channel == "ntfy":
-        try:
-            from src.integrations import load_integrations
-            import httpx
-            intg = next(
-                (i for i in load_integrations()
-                 if i.get("preset") == "ntfy" and i.get("enabled", True) and i.get("base_url")),
-                None,
-            )
-            if intg:
-                base = intg["base_url"].rstrip("/")
-                topic = settings.get("reminder_ntfy_topic") or "reminders"
-                ntfy_body = synthesis or note_body or title
-                hdrs = {"Title": title or "Reminder", "Priority": "high", "Tags": "bell"}
-                api_key = intg.get("api_key", "")
-                if api_key:
-                    hdrs["Authorization"] = f"Bearer {api_key}"
-                async with httpx.AsyncClient(timeout=10.0) as client:
-                    resp = await client.post(f"{base}/{topic}", content=ntfy_body, headers=hdrs)
-                    ntfy_sent = resp.is_success
-                    if not ntfy_sent:
-                        ntfy_error = f"ntfy returned HTTP {resp.status_code}"
-            else:
-                ntfy_error = "No enabled ntfy integration"
-        except Exception as e:
-            ntfy_error = str(e) or e.__class__.__name__
-            logger.warning(f"Reminder ntfy send failed: {e}")
-
-    # In-app browser notification ALWAYS fires (regardless of channel). The
-    # frontend polls `/api/tasks/notifications` and turns any entry with a
-    # `body` into a real `Notification(...)` — same surface as task-success
-    # popups. Lets the user see reminders inside the app even when the
-    # primary channel is email/ntfy and the tab is open.
-    browser_sent = False
-    local_browser_sent = (not queue_browser and channel == "browser")
-    if queue_browser and _scheduler_ref is not None:
-        try:
-            _scheduler_ref.add_notification(
-                task_name=title or "Reminder",
-                status="success",
-                task_id=f"reminder-{note_id}",
-                owner=owner or None,
-                body=(synthesis or note_body or title or "").strip()[:500] or "Reminder",
-            )
-            browser_sent = True
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: in-app notif push failed: {_e}")
-
-    # Dedupe across paths: write to the same cache file `action_ping_notes`
-    # reads, so the background scanner's REPING_MIN window suppresses a
-    # second send for the same note within 25 min. Without this, a note
-    # whose due_date fires while the user has the app open got TWO emails
-    # (frontend-fired here + background-fired by ping_notes 0–5 min later).
-    if (email_sent or ntfy_sent or browser_sent or local_browser_sent) and note_id:
-        try:
-            import json as _json
-            from datetime import datetime as _dt, timezone as _tz
-            from pathlib import Path as _P
-            # Per-owner cache so the scanner's prune step on user A's run
-            # doesn't drop user B's just-fired entry (review C4).
-            _STATE = cache_path
-            if _STATE is None:
-                _slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default"))
-                _STATE = _P(f"data/note_pings_{_slug}.json")
-            _STATE.parent.mkdir(parents=True, exist_ok=True)
-            try:
-                _cache = cache or (_json.loads(_STATE.read_text()) if _STATE.exists() else {})
-            except Exception:
-                _cache = {}
-            sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "browser"
-            _cache[cache_key or str(note_id)] = {
-                "at": _dt.now(_tz.utc).isoformat(),
-                "channel": sent_channel,
-            }
-            _STATE.write_text(_json.dumps(_cache))
-        except Exception as _e:
-            logger.debug(f"dispatch_reminder: cache write failed: {_e}")
-
-    return {
-        "synthesis": synthesis,
-        "email_sent": email_sent,
-        "email_error": email_error,
-        "ntfy_sent": ntfy_sent,
-        "ntfy_error": ntfy_error,
-        "browser_sent": browser_sent or local_browser_sent,
-    }
-
-
-# ---------------------------------------------------------------------------
-# Router factory
-# ---------------------------------------------------------------------------
-
-def setup_note_routes(task_scheduler=None):
-    # Expose the scheduler to module-level `dispatch_reminder` so reminders
-    # can also push to the in-app notification queue (the polling system
-    # turns each entry into a real browser Notification + the existing
-    # tasks-tab badge / dot system).
-    global _scheduler_ref
-    _scheduler_ref = task_scheduler
-
-    router = APIRouter(prefix="/api/notes", tags=["notes"])
-
-    def _owner(request: Request) -> Optional[str]:
-        return get_current_user(request)
-
-    # --- LIST ---
-    @router.get("")
-    def list_notes(
-        request: Request,
-        archived: Optional[bool] = None,
-        label: Optional[str] = None,
-    ):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            q = db.query(Note)
-            if user is not None:
-                q = q.filter(Note.owner == user)
-            if archived is not None:
-                q = q.filter(Note.archived == archived)
-            else:
-                q = q.filter(Note.archived == False)
-            if label:
-                q = q.filter(Note.label == label)
-            # Archived view: most recently archived first. Active view: pin + manual order.
-            if archived is True:
-                notes = q.order_by(Note.updated_at.desc()).all()
-            else:
-                notes = q.order_by(Note.pinned.desc(), Note.sort_order.asc(), Note.updated_at.desc()).all()
-            return {"notes": [_note_to_dict(n) for n in notes]}
-        finally:
-            db.close()
-
-    # --- CREATE ---
-    @router.post("")
-    def create_note(request: Request, body: NoteCreate):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = Note(
-                id=str(uuid.uuid4()),
-                owner=user,
-                title=body.title,
-                content=body.content,
-                items=json.dumps(body.items) if body.items is not None else None,
-                note_type=body.note_type,
-                color=body.color,
-                label=body.label,
-                pinned=body.pinned,
-                due_date=body.due_date,
-                source=body.source,
-                session_id=body.session_id,
-                image_url=body.image_url,
-                repeat=body.repeat or "none",
-                sort_order=body.sort_order if body.sort_order is not None else 0,
-            )
-            db.add(note)
-            db.commit()
-            db.refresh(note)
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- GET ONE ---
-    @router.get("/{note_id}")
-    def get_note(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- UPDATE ---
-    @router.put("/{note_id}")
-    def update_note(request: Request, note_id: str, body: NoteUpdate):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-
-            if body.title is not None:
-                note.title = body.title
-            if body.content is not None:
-                note.content = body.content
-            if body.items is not None:
-                note.items = json.dumps(body.items)
-                flag_modified(note, "items")
-            if body.note_type is not None:
-                note.note_type = body.note_type
-            if body.color is not None:
-                note.color = body.color
-            if body.label is not None:
-                note.label = body.label
-            if body.pinned is not None:
-                note.pinned = body.pinned
-            if body.archived is not None:
-                note.archived = body.archived
-            if body.due_date is not None:
-                note.due_date = body.due_date
-            if body.image_url is not None:
-                note.image_url = body.image_url
-            if body.repeat is not None:
-                note.repeat = body.repeat
-            if body.sort_order is not None:
-                note.sort_order = body.sort_order
-            if body.agent_session_id is not None:
-                note.agent_session_id = body.agent_session_id
-
-            db.commit()
-            db.refresh(note)
-            return _note_to_dict(note)
-        finally:
-            db.close()
-
-    # --- DELETE ---
-    @router.delete("/{note_id}")
-    def delete_note(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            db.delete(note)
-            db.commit()
-            return {"ok": True}
-        finally:
-            db.close()
-
-    # --- TOGGLE PIN ---
-    @router.post("/{note_id}/pin")
-    def toggle_pin(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            note.pinned = not note.pinned
-            db.commit()
-            return {"ok": True, "pinned": note.pinned}
-        finally:
-            db.close()
-
-    # --- TOGGLE ARCHIVE ---
-    @router.post("/{note_id}/archive")
-    def toggle_archive(request: Request, note_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            note.archived = not note.archived
-            db.commit()
-            return {"ok": True, "archived": note.archived}
-        finally:
-            db.close()
-
-    # --- TOGGLE CHECKLIST ITEM ---
-    @router.post("/{note_id}/items/{index}/toggle")
-    def toggle_item(request: Request, note_id: str, index: int):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            note = db.query(Note).filter(Note.id == note_id).first()
-            if not note:
-                raise HTTPException(404, "Note not found")
-            # SECURITY: strict ownership — previously `note.owner and note.owner != user`
-            # let any user touch a row whose owner field was null/empty.
-            if user is not None and note.owner != user:
-                raise HTTPException(404, "Note not found")
-            if not note.items:
-                raise HTTPException(400, "Note has no checklist items")
-            items = json.loads(note.items)
-            if index < 0 or index >= len(items):
-                raise HTTPException(400, f"Item index {index} out of range")
-            items[index]["done"] = not items[index].get("done", False)
-            note.items = json.dumps(items)
-            flag_modified(note, "items")
-            db.commit()
-            return {"ok": True, "items": items}
-        finally:
-            db.close()
-
-    # --- FIRE REMINDER ---
-    @router.post("/fire-reminder")
-    async def fire_reminder(request: Request):
-        """Dispatch a reminder according to user settings.
-
-        Called by the frontend when a reminder fires. Optionally generates an
-        LLM synthesis line and/or sends an email through configured SMTP.
-        Returns {synthesis, email_sent}.
-        """
-        # Gate against anonymous callers — LLM synthesis can burn tokens.
-        from src.auth_helpers import get_current_user as _gcu
-        if not _gcu(request):
-            raise HTTPException(401, "Not authenticated")
-        body = await request.json()
-        note_id = body.get("note_id")
-        title = (body.get("title") or "").strip()
-        note_body = (body.get("body") or "").strip()
-        if not note_id:
-            raise HTTPException(400, "note_id required")
-
-        # Delegate to the module-level helper so background tasks can reuse
-        # the same dispatch without an HTTP roundtrip + auth cookie.
-        return await dispatch_reminder(
-            title=title, note_body=note_body, note_id=note_id,
-            owner=_gcu(request) or "",
-            queue_browser=False,
-        )
-
-    # --- REORDER NOTES ---
-    @router.post("/reorder")
-    async def reorder_notes(request: Request):
-        """Update sort_order for a list of note IDs in the order provided."""
-        user = _owner(request)
-        body = await request.json()
-        ids = body.get("ids", [])
-        if not isinstance(ids, list):
-            raise HTTPException(400, "ids must be a list")
-        # v2 review HIGH-12: drop the legacy `(owner == user) | (owner ==
-        # None)` OR which let an authenticated user silently reorder
-        # every legacy-null-owner note belonging to other accounts. In
-        # an unconfigured (single-user) auth deploy the OR is still safe
-        # because there's no second user to attack; we keep that branch
-        # explicit and gated on AuthManager.is_configured.
-        try:
-            from core.auth import AuthManager
-            _allow_null = not AuthManager().is_configured
-        except Exception:
-            _allow_null = False
-        db = SessionLocal()
-        try:
-            for i, nid in enumerate(ids):
-                q = db.query(Note).filter(Note.id == nid)
-                if user is not None:
-                    if _allow_null:
-                        q = q.filter((Note.owner == user) | (Note.owner == None))  # noqa: E711
-                    else:
-                        q = q.filter(Note.owner == user)
-                note = q.first()
-                if note:
-                    note.sort_order = i
-            db.commit()
-            return {"ok": True, "count": len(ids)}
-        finally:
-            db.close()
-
-    return router
+_sys.modules[__name__] = _canonical

routes/personal_routes.py

@@ -1,20 +1,133 @@
 # routes/personal_routes.py
 """Routes for personal documents management."""
+import asyncio
 import os
 import logging
-from typing import List
+import shutil
+import uuid
+from typing import Any, Dict, List, Tuple
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Depends
+from fastapi.concurrency import run_in_threadpool
 from src.request_models import DirectoryRequest
-from core.constants import BASE_DIR, PERSONAL_DIR
+from core.constants import BASE_DIR, PERSONAL_DIR, PERSONAL_UPLOADS_DIR
 from src.rag_singleton import get_rag_manager
-from src.auth_helpers import get_current_user, require_user
+from src.auth_helpers import require_privilege, require_user
 from core.middleware import require_admin
 from src.upload_handler import secure_filename
+from src.upload_limits import PERSONAL_UPLOAD_MAX_BYTES
 
-UPLOADS_DIR = os.path.join(BASE_DIR, "data", "personal_uploads")
+UPLOADS_DIR = PERSONAL_UPLOADS_DIR
 
 logger = logging.getLogger(__name__)
 
+def _personal_upload_dir_for_owner(owner: str | None, *, create: bool = True) -> str:
+    """Return the per-owner upload directory used for direct RAG uploads."""
+    owner_segment = secure_filename((owner or "local").strip())[:80] or "local"
+    upload_dir = os.path.abspath(os.path.join(UPLOADS_DIR, owner_segment))
+    base_abs = os.path.abspath(UPLOADS_DIR)
+    if os.path.commonpath([upload_dir, base_abs]) != base_abs:
+        raise ValueError("Unsafe upload owner path")
+    if create:
+        os.makedirs(upload_dir, exist_ok=True)
+    return upload_dir
+
+
+def _unique_personal_upload_path(upload_dir: str, original_name: str | None) -> Tuple[str, str, str]:
+    """Build a collision-resistant upload path while preserving a display name."""
+    safe_name = secure_filename(os.path.basename(original_name or "upload"))
+    if not safe_name or safe_name.startswith("."):
+        safe_name = "upload"
+
+    stem, ext = os.path.splitext(safe_name)
+    stem = (stem or "upload")[:80]
+    filename = f"{stem}-{uuid.uuid4().hex[:10]}{ext.lower()}"
+    file_path = os.path.abspath(os.path.join(upload_dir, filename))
+    upload_abs = os.path.abspath(upload_dir)
+    if os.path.commonpath([file_path, upload_abs]) != upload_abs:
+        raise ValueError("Unsafe upload filename")
+    return file_path, filename, safe_name
+
+
+def _unique_existing_target(path: str) -> str:
+    """Return a non-existing sibling path for rename collision handling."""
+    if not os.path.exists(path):
+        return path
+    stem, ext = os.path.splitext(path)
+    while True:
+        candidate = f"{stem}-{uuid.uuid4().hex[:10]}{ext}"
+        if not os.path.exists(candidate):
+            return candidate
+
+
+def _remove_empty_tree(path: str) -> None:
+    """Best-effort removal of empty directories under ``path``."""
+    if not os.path.isdir(path):
+        return
+    for root, dirs, _files in os.walk(path, topdown=False):
+        for dirname in dirs:
+            candidate = os.path.join(root, dirname)
+            try:
+                os.rmdir(candidate)
+            except OSError:
+                pass
+    try:
+        os.rmdir(path)
+    except OSError:
+        pass
+
+
+def rename_personal_upload_owner(
+    old_owner: str,
+    new_owner: str,
+    *,
+    personal_docs_manager: Any = None,
+    rag_manager: Any = None,
+) -> Dict[str, Any]:
+    """Move direct personal uploads and rewrite RAG owner metadata on user rename."""
+    old_dir = _personal_upload_dir_for_owner(old_owner, create=False)
+    new_dir = _personal_upload_dir_for_owner(new_owner, create=False)
+    path_map: Dict[str, str] = {}
+    moved_files = 0
+
+    if os.path.isdir(old_dir) and old_dir != new_dir:
+        os.makedirs(new_dir, exist_ok=True)
+        for root, _dirs, files in os.walk(old_dir):
+            rel_root = os.path.relpath(root, old_dir)
+            target_root = new_dir if rel_root == "." else os.path.join(new_dir, rel_root)
+            os.makedirs(target_root, exist_ok=True)
+            for filename in files:
+                source = os.path.abspath(os.path.join(root, filename))
+                target = _unique_existing_target(os.path.abspath(os.path.join(target_root, filename)))
+                shutil.move(source, target)
+                path_map[source] = target
+                moved_files += 1
+        _remove_empty_tree(old_dir)
+
+    if personal_docs_manager is not None:
+        rename_directory = getattr(personal_docs_manager, "rename_directory", None)
+        if callable(rename_directory):
+            rename_directory(old_dir, new_dir, path_map=path_map)
+
+    rag_result = None
+    if rag_manager is not None:
+        rename_owner = getattr(rag_manager, "rename_owner", None)
+        if callable(rename_owner):
+            rag_result = rename_owner(
+                old_owner,
+                new_owner,
+                path_map=path_map,
+                path_prefixes=[(old_dir, new_dir)],
+            )
+
+    return {
+        "old_dir": old_dir,
+        "new_dir": new_dir,
+        "moved_files": moved_files,
+        "path_map": path_map,
+        "rag_result": rag_result,
+    }
+
+
 def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     """
     Setup personal documents related routes.
@@ -29,6 +142,22 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     """
     router = APIRouter(prefix="/api/personal")
 
+    # Serializes directory index jobs across requests. Indexing runs in the
+    # threadpool (#5558), so concurrent requests would otherwise run in parallel
+    # and race PersonalDocsManager's unsynchronized list mutations and file
+    # writes; before the threadpool move they serialized on the blocked event
+    # loop, so one-at-a-time is behavior parity.
+    #
+    # An asyncio.Lock acquired in the async handler BEFORE offloading: a waiting
+    # request parks on the event loop instead of pinning a threadpool worker (an
+    # earlier threading.Lock taken INSIDE the worker meant queued jobs held pool
+    # tokens while blocked, starving every other run_in_threadpool caller).
+    # add/remove/reload all take this lock, so their mutations never interleave.
+    # Per-router (not module-global) so each app binds it to its own event loop.
+    # Scope is the single process: multi-worker deployments would need a shared
+    # lock (out of scope for #5558).
+    _index_job_lock = asyncio.Lock()
+
     def _rag():
         """Get the current RAG manager, retrying init if needed."""
         return get_rag_manager()
@@ -38,9 +167,12 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         if not directory:
             raise HTTPException(400, "Directory path is required")
 
-        base_abs = os.path.abspath(PERSONAL_DIR)
+        # realpath (not abspath) so a symlink inside PERSONAL_DIR that points
+        # outside it is resolved before the commonpath confinement check below;
+        # abspath only normalises `..` and would let such a symlink escape.
+        base_abs = os.path.realpath(PERSONAL_DIR)
         candidate = directory if os.path.isabs(directory) else os.path.join(base_abs, directory)
-        resolved = os.path.abspath(candidate)
+        resolved = os.path.realpath(candidate)
         try:
             in_base = os.path.commonpath([resolved, base_abs]) == base_abs
         except ValueError:
@@ -57,8 +189,12 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
         return {"files": files, "directories": directories}
     
     @router.post("/reload")
-    def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
-        personal_docs_manager.refresh_index()
+    async def api_personal_reload(owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
+        # refresh_index() re-extracts text across every tracked directory —
+        # blocking work. Take the shared job lock (so it cannot race an add /
+        # remove) and run it off the event loop.
+        async with _index_job_lock:
+            await run_in_threadpool(personal_docs_manager.refresh_index)
         return {"ok": True, "count": len(personal_docs_manager.index)}
     
     @router.post("/add_directory")
@@ -92,12 +228,26 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             # Use the RAGManager to index the directory
             rag = _rag()
             if rag:
-                result = rag.index_personal_documents(directory, owner=owner)
-                
+                def _index_directory():
+                    result = rag.index_personal_documents(directory, owner=owner)
+                    if result["success"]:
+                        # Also update the personal_docs_manager to track this
+                        # directory. Kept inside the offloaded call: it triggers
+                        # refresh_index(), which re-extracts text across tracked
+                        # directories.
+                        personal_docs_manager.add_directory(directory, index=False)
+                    return result
+
+                # Indexing walks, embeds, and stores the whole tree — minutes
+                # on a real directory. The handler is async, so calling it
+                # inline runs it on the event loop and every other request
+                # queues behind it until it finishes (#5558). Serialize on the
+                # async job lock BEFORE offloading so a queued request parks on
+                # the loop instead of pinning a threadpool worker.
+                async with _index_job_lock:
+                    result = await run_in_threadpool(_index_directory)
+
                 if result["success"]:
-                    # Also update the personal_docs_manager to track this directory
-                    personal_docs_manager.add_directory(directory, index=False)
-                    
                     return {
                         "success": True,
                         "message": f"Successfully indexed {result['indexed_count']} chunks from {directory}",
@@ -128,22 +278,33 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
             JSON response confirming removal
         """
         try:
-            if not directory:
-                raise HTTPException(400, "Directory path is required")
+            # Confine to PERSONAL_DIR — parity with add_directory_to_rag (which
+            # resolves the path the same way). Without this, an arbitrary or
+            # `..`-escaping path is passed straight to
+            # personal_docs_manager.remove_directory / rag.remove_directory.
+            directory = _resolve_allowed_personal_dir(directory)
 
             logger.info(f"Removing directory from RAG: {directory}")
 
-            # Always remove from personal_docs_manager tracking
-            if hasattr(personal_docs_manager, 'remove_directory'):
-                personal_docs_manager.remove_directory(directory)
-
-            # Remove from RAG vector store (best-effort)
             rag = _rag()
-            if rag:
-                try:
-                    rag.remove_directory(directory)
-                except Exception as e:
-                    logger.warning(f"RAG removal failed for directory {directory}: {e}")
+
+            def _remove_directory():
+                # Always remove from personal_docs_manager tracking. This
+                # mutates the same unsynchronized list/index an add job touches
+                # and re-extracts text (refresh_index), so it is blocking work.
+                if hasattr(personal_docs_manager, 'remove_directory'):
+                    personal_docs_manager.remove_directory(directory)
+                # Remove from RAG vector store (best-effort).
+                if rag:
+                    try:
+                        rag.remove_directory(directory)
+                    except Exception as e:
+                        logger.warning(f"RAG removal failed for directory {directory}: {e}")
+
+            # Same job lock as add/reload so remove cannot interleave with an
+            # in-flight add; offloaded off the event loop.
+            async with _index_job_lock:
+                await run_in_threadpool(_remove_directory)
 
             return {
                 "success": True,
@@ -160,70 +321,84 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     @router.post("/upload")
     async def upload_files_to_rag(request: Request, files: List[UploadFile] = File(...)):
         """Upload files directly into RAG. Supports text and PDF."""
-        user = get_current_user(request)
+        user = require_privilege(request, "can_use_documents")
         rag = _rag()
         if not rag:
             raise HTTPException(503, "RAG system is not available — is the embedding service running?")
 
-        os.makedirs(UPLOADS_DIR, exist_ok=True)
+        upload_dir = _personal_upload_dir_for_owner(user)
 
         total_indexed = 0
         total_failed = 0
         uploaded_files = []
 
-        for upload in files:
-            try:
-                # Sanitize filename — strip directory components and unsafe chars
-                safe_name = secure_filename(os.path.basename(upload.filename or "upload"))
-                if not safe_name or safe_name.startswith("."):
-                    safe_name = f"upload_{total_indexed + total_failed}"
-                file_path = os.path.join(UPLOADS_DIR, safe_name)
-                # Defense-in-depth: ensure resolved path stays under UPLOADS_DIR
-                base_abs = os.path.abspath(UPLOADS_DIR)
-                if os.path.commonpath([os.path.abspath(file_path), base_abs]) != base_abs:
-                    logger.warning(f"Rejected unsafe upload path: {upload.filename!r}")
-                    total_failed += 1
-                    continue
-                content_bytes = await upload.read()
-                with open(file_path, "wb") as f:
-                    f.write(content_bytes)
-
-                ext = os.path.splitext(safe_name)[1].lower()
-                if ext == ".pdf":
-                    from src.personal_docs import extract_pdf_text
-                    text = extract_pdf_text(file_path)
-                else:
-                    text = content_bytes.decode("utf-8", errors="replace")
-
-                if not text or not text.strip():
-                    total_failed += 1
-                    continue
-
-                # Chunk and index
-                chunks = rag._split_into_chunks(text, chunk_size=500)
-                for i, chunk in enumerate(chunks):
-                    metadata = {
-                        "source": file_path,
-                        "filename": safe_name,
-                        "directory": UPLOADS_DIR,
-                        "type": ext,
-                        "chunk_id": i,
-                    }
-                    if user:
-                        metadata["owner"] = user
-                    if rag.add_document(chunk, metadata):
-                        total_indexed += 1
-                    else:
+        # Chunking, embedding and the tracking update are blocking work over the
+        # same vector/tracking state add_directory mutates (#5634). Take the
+        # shared job lock BEFORE offloading so a queued request parks on the loop
+        # instead of pinning a threadpool worker, matching add_directory.
+        # Read and process one capped payload at a time so a multi-file request
+        # cannot retain len(files) * PERSONAL_UPLOAD_MAX_BYTES in memory.
+        async with _index_job_lock:
+            for upload in files:
+                try:
+                    file_path, stored_name, safe_name = _unique_personal_upload_path(
+                        upload_dir, upload.filename
+                    )
+                    content_bytes = await upload.read(PERSONAL_UPLOAD_MAX_BYTES + 1)
+                    if len(content_bytes) > PERSONAL_UPLOAD_MAX_BYTES:
+                        logger.warning(f"Rejected oversized personal upload: {upload.filename!r}")
                         total_failed += 1
+                        continue
 
-                uploaded_files.append(safe_name)
-            except Exception as e:
-                logger.error(f"Failed to upload/index {upload.filename}: {e}")
-                total_failed += 1
+                    def _index_upload():
+                        with open(file_path, "wb") as f:
+                            f.write(content_bytes)
 
-        # Track uploads directory
-        if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
-            personal_docs_manager.add_directory(UPLOADS_DIR, index=False)
+                        ext = os.path.splitext(safe_name)[1].lower()
+                        if ext == ".pdf":
+                            from src.personal_docs import extract_pdf_text
+                            text = extract_pdf_text(file_path)
+                        else:
+                            text = content_bytes.decode("utf-8", errors="replace")
+
+                        if not text or not text.strip():
+                            return 0, 1, None
+
+                        indexed = 0
+                        failed = 0
+                        chunks = rag._split_into_chunks(text, chunk_size=500)
+                        for i, chunk in enumerate(chunks):
+                            metadata = {
+                                "source": file_path,
+                                "filename": safe_name,
+                                "stored_filename": stored_name,
+                                "directory": upload_dir,
+                                "type": ext,
+                                "chunk_id": i,
+                            }
+                            if user:
+                                metadata["owner"] = user
+                            if rag.add_document(chunk, metadata):
+                                indexed += 1
+                            else:
+                                failed += 1
+                        return indexed, failed, safe_name
+
+                    indexed, failed, uploaded_name = await run_in_threadpool(_index_upload)
+                    total_indexed += indexed
+                    total_failed += failed
+                    if uploaded_name:
+                        uploaded_files.append(uploaded_name)
+                except Exception as e:
+                    logger.error(f"Failed to upload/index {upload.filename}: {e}")
+                    total_failed += 1
+
+            # Same transition, same lock: the tracking update must not land
+            # while another job is mid-write over the same state.
+            if uploaded_files and hasattr(personal_docs_manager, "add_directory"):
+                await run_in_threadpool(
+                    personal_docs_manager.add_directory, upload_dir, index=False
+                )
 
         return {
             "success": True,
@@ -236,33 +411,47 @@ def setup_personal_routes(personal_docs_manager, rag_manager, rag_available):
     async def delete_file_from_rag(filepath: str = Query(...), owner: str = Depends(require_user), _admin: None = Depends(require_admin)):
         """Delete a specific file from RAG index and optionally from disk."""
         try:
-            # Remove chunks from RAG vector store (best-effort)
-            removed = 0
-            rag = _rag()
-            if rag:
+            def _delete_file():
+                # Remove chunks from RAG vector store (best-effort)
+                removed = 0
+                rag = _rag()
+                if rag:
+                    try:
+                        removed = rag.delete_by_source(filepath)
+                    except Exception as e:
+                        logger.warning(f"RAG removal failed for {filepath}: {e}")
+
+                # Delete file from disk if it's in the caller's own uploads dir.
+                # Scope to the per-owner subdir, not the shared uploads root, so one
+                # admin can't delete another user's personal files by path.
+                deleted_from_disk = False
                 try:
-                    removed = rag.delete_by_source(filepath)
-                except Exception as e:
-                    logger.warning(f"RAG removal failed for {filepath}: {e}")
+                    abs_target = os.path.realpath(filepath)
+                    base_abs = os.path.realpath(_personal_upload_dir_for_owner(owner, create=False))
+                    in_uploads = (
+                        abs_target == base_abs
+                        or os.path.commonpath([abs_target, base_abs]) == base_abs
+                    )
+                except ValueError:
+                    # commonpath raises on mixed drives / non-comparable paths
+                    in_uploads = False
+                if in_uploads and abs_target != base_abs:
+                    try:
+                        os.remove(abs_target)
+                        deleted_from_disk = True
+                    except FileNotFoundError:
+                        pass  # already gone — race with another request or cleanup
 
-            # Delete file from disk if it's in uploads dir
-            deleted_from_disk = False
-            try:
-                abs_target = os.path.abspath(filepath)
-                base_abs = os.path.abspath(UPLOADS_DIR)
-                in_uploads = (
-                    abs_target == base_abs
-                    or os.path.commonpath([abs_target, base_abs]) == base_abs
-                )
-            except ValueError:
-                # commonpath raises on mixed drives / non-comparable paths
-                in_uploads = False
-            if in_uploads and abs_target != base_abs and os.path.exists(abs_target):
-                os.remove(abs_target)
-                deleted_from_disk = True
+                # Exclude the file from the listing (persists across restarts)
+                personal_docs_manager.exclude_file(filepath)
+                return removed, deleted_from_disk
 
-            # Exclude the file from the listing (persists across restarts)
-            personal_docs_manager.exclude_file(filepath)
+            # Vector removal, the disk unlink and the exclusion write are one
+            # transition over the same state add_directory mutates (#5634), and
+            # all three block. Take the shared job lock BEFORE offloading, as
+            # add_directory does.
+            async with _index_job_lock:
+                removed, deleted_from_disk = await run_in_threadpool(_delete_file)
 
             return {
                 "success": True,

routes/prefs_routes.py

@@ -1,50 +1,101 @@
 """User preferences API — per-user key/value store backed by a JSON file."""
 import json
-import os
 from typing import Optional
 from fastapi import APIRouter, Request
+from core.atomic_io import atomic_write_json
 from src.auth_helpers import get_current_user
+from src.constants import USER_PREFS_FILE
 
-PREFS_FILE = os.path.join("data", "user_prefs.json")
+PREFS_FILE = USER_PREFS_FILE
+_FOREGROUND_POLICY_KEYS = (
+    "foreground_fallback_enabled",
+    "foreground_model_fallbacks",
+)
 
 
 def _load():
     """Load the raw prefs file (internal use only)."""
     try:
-        with open(PREFS_FILE, "r") as f:
-            return json.load(f)
+        with open(PREFS_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            return data if isinstance(data, dict) else {}
     except (FileNotFoundError, json.JSONDecodeError):
         return {}
 
 
 def _save(prefs):
-    os.makedirs(os.path.dirname(PREFS_FILE), exist_ok=True)
-    with open(PREFS_FILE, "w") as f:
-        json.dump(prefs, f, indent=2)
+    atomic_write_json(PREFS_FILE, prefs, indent=2)
 
 
 def _load_for_user(user: Optional[str] = None) -> dict:
     """Load preferences for a specific user."""
     all_prefs = _load()
-    if "_users" in all_prefs:
+    users = all_prefs.get("_users")
+    if isinstance(users, dict):
         if user is None:
             # Auth disabled — return first user's prefs for backward compat
-            users = all_prefs["_users"]
-            return dict(next(iter(users.values()), {}))
-        return dict(all_prefs["_users"].get(user, {}))
-    # Legacy flat format — return as-is
-    return dict(all_prefs)
+            prefs = dict(next(iter(users.values()), {}))
+            # Foreground fallback consent is never borrowed from a named
+            # owner. Auth-disabled operation has a separate flat/root opt-in
+            # that remains inert when authentication is enabled again.
+            for key in _FOREGROUND_POLICY_KEYS:
+                prefs.pop(key, None)
+                if key in all_prefs:
+                    prefs[key] = all_prefs[key]
+            return prefs
+        prefs = users.get(user, {})
+        return dict(prefs) if isinstance(prefs, dict) else {}
+    # A legacy flat store belongs only to auth-disabled single-user mode.
+    # Copying it into the first named user's new `_users` record during an
+    # auth transition would silently transfer another user's preferences and,
+    # critically, foreground fallback consent. Named owners therefore start
+    # with an empty record and must write their own preferences explicitly.
+    return dict(all_prefs) if user is None else {}
 
 
 def _save_for_user(user: Optional[str], prefs: dict):
     """Save preferences for a specific user."""
     all_prefs = _load()
     if user is None:
-        # Auth disabled — save flat
+        # Auth disabled. If the store is already multi-user (e.g. auth was
+        # turned off on a deployment that previously ran multi-user), writing
+        # `prefs` flat would overwrite the whole `_users` map and destroy every
+        # other user's preferences. Instead write back into the same (first)
+        # slot _load_for_user(None) reads from, preserving the others.
+        users = all_prefs.get("_users")
+        if isinstance(users, dict):
+            first_key = next(iter(users), None)
+            if first_key is not None:
+                existing_named = users.get(first_key)
+                existing_named = (
+                    dict(existing_named)
+                    if isinstance(existing_named, dict)
+                    else {}
+                )
+                named_foreground = {
+                    key: existing_named[key]
+                    for key in _FOREGROUND_POLICY_KEYS
+                    if key in existing_named
+                }
+                users[first_key] = {
+                    key: value
+                    for key, value in prefs.items()
+                    if key not in _FOREGROUND_POLICY_KEYS
+                }
+                users[first_key].update(named_foreground)
+                for key in _FOREGROUND_POLICY_KEYS:
+                    if key in prefs:
+                        all_prefs[key] = prefs[key]
+                _save(all_prefs)
+                return
         _save(prefs)
         return
-    if "_users" not in all_prefs:
-        all_prefs = {"_users": {}}
+    if not isinstance(all_prefs.get("_users"), dict):
+        # Preserve the flat single-user object as inert legacy data while
+        # creating the first named-owner namespace. In particular, historical
+        # fallback values must not be deleted or copied into the new owner.
+        all_prefs = dict(all_prefs)
+        all_prefs["_users"] = {}
     all_prefs["_users"][user] = prefs
     _save(all_prefs)
 

routes/preset_routes.py

@@ -1,5 +1,6 @@
 """Preset routes — /api/presets GET, /api/presets/custom POST, user templates CRUD."""
 
+import asyncio
 import logging
 import uuid
 from typing import Dict, Any, List
@@ -9,6 +10,7 @@ from pydantic import BaseModel, Field
 
 from src.request_models import PresetUpdateRequest
 from core.middleware import require_admin
+from src.auth_helpers import effective_user
 
 logger = logging.getLogger(__name__)
 
@@ -19,6 +21,8 @@ class UserTemplateRequest(BaseModel):
     system_prompt: str = Field("", max_length=10000)
     temperature: float = Field(1.0, ge=0.0, le=2.0)
     max_tokens: int = Field(0, ge=0, le=65536)
+    persona_memory: str = Field("", max_length=6000)
+    persona_memory_schema: str = Field("general", pattern="^(general|health)$")
 
 
 def setup_preset_routes(preset_manager) -> APIRouter:
@@ -39,6 +43,10 @@ def setup_preset_routes(preset_manager) -> APIRouter:
                 preset_update.enabled,
                 preset_update.inject_prefix,
                 preset_update.inject_suffix,
+                preset_update.persona_memory,
+                preset_update.persona_memory_schema,
+                preset_update.thinking_mode,
+                preset_update.show_persona_name,
             )
             if success:
                 return {"success": True, "message": "Custom preset updated"}
@@ -100,7 +108,8 @@ def setup_preset_routes(preset_manager) -> APIRouter:
 
         try:
             model_spec = data.get("model") or ""
-            url, model, headers = _resolve_model(model_spec)
+            user = effective_user(request)
+            url, model, headers = await asyncio.to_thread(_resolve_model, model_spec, owner=user)
             result = await llm_call_async(url, model, messages, temperature=0.8, max_tokens=500, headers=headers)
             return {"success": True, "prompt": result.strip()}
         except Exception as e:

routes/research/__init__.py

@@ -0,0 +1,5 @@
+"""Research route domain package (slice 2b, #4082/#4071).
+
+Contains research_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/research_routes.py re-exports from here.
+"""

routes/research/research_routes.py

@@ -0,0 +1,897 @@
+"""Research background task routes — /api/research/*."""
+
+import asyncio
+import json
+import logging
+import re
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Literal, Optional
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import HTMLResponse, StreamingResponse
+from pydantic import BaseModel, Field
+from core.middleware import INTERNAL_TOOL_USER
+from src.endpoint_resolver import resolve_endpoint
+from src.auth_helpers import _auth_disabled, get_current_user
+from src.owner_identity import REQUEST_SENTINEL_OWNERS
+from src.constants import DEEP_RESEARCH_DIR
+
+_SESSION_ID_RE = re.compile(r"^[a-zA-Z0-9-]{1,128}$")
+
+
+def _validate_session_id(session_id: str) -> str:
+    if not _SESSION_ID_RE.fullmatch(session_id):
+        raise HTTPException(400, "Invalid session ID format")
+    return session_id
+
+
+def _research_storage_root() -> Path:
+    return Path(DEEP_RESEARCH_DIR).resolve()
+
+
+def _find_research_path(session_id: str) -> Path | None:
+    """Find a persisted research file without deriving its path from input."""
+    expected_name = f"{_validate_session_id(session_id)}.json"
+    root = _research_storage_root()
+    for stored_path in root.glob("*.json"):
+        if stored_path.name != expected_name:
+            continue
+        resolved = stored_path.resolve()
+        try:
+            resolved.relative_to(root)
+        except ValueError:
+            return None
+        if not resolved.is_file():
+            return None
+        return resolved
+    return None
+
+
+def _require_research_path(session_id: str) -> Path:
+    path = _find_research_path(session_id)
+    if path is None:
+        raise HTTPException(404, "Research not found")
+    return path
+
+
+def _find_owned_research_path(session_id: str, user: str) -> Path | None:
+    path = _find_research_path(session_id)
+    if path is None:
+        return None
+    try:
+        owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+    except Exception:
+        return None
+    if owner != user:
+        return None
+    return path
+
+
+logger = logging.getLogger(__name__)
+
+# Model-name substrings that are NOT chat/generation models — research must
+# never pick these as its model. An OpenAI-style endpoint often lists
+# `text-embedding-ada-002` etc. first in its model list, which is why research
+# was failing with "Cannot reach model 'text-embedding-ada-002'".
+_NON_CHAT_MODEL = (
+    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
+    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
+)
+
+_RESEARCH_IMAGE_BLOCKLIST = {
+    "cdn.shopify.com/s/files/1/0179/4388/7926/files/icon.png",
+}
+
+
+def _is_research_icon_or_logo_url(url: str) -> bool:
+    path = url.lower().split("?")[0]
+    return any(token in path for token in (
+        "/logo", "logo_", "-logo", "favicon", "apple-touch-icon",
+        "sprite", "icon-", "_icon", "/icons/", "badge",
+    ))
+
+
+def _research_thumbnail(data: dict) -> str:
+    """Pick the same first visible image the visual report uses as hero."""
+    hidden = set(data.get("hidden_images") or [])
+    seen = set()
+
+    def usable(image: str) -> bool:
+        image = str(image or "").strip()
+        if not image or image in seen or image in hidden:
+            return False
+        if not image.startswith("https://"):
+            return False
+        if image.endswith((".svg", ".ico", ".gif")):
+            return False
+        if any(blocked in image for blocked in _RESEARCH_IMAGE_BLOCKLIST):
+            return False
+        if _is_research_icon_or_logo_url(image):
+            return False
+        return True
+
+    for source in data.get("sources") or []:
+        if not isinstance(source, dict):
+            continue
+        image = str(source.get("image") or source.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    for finding in data.get("raw_findings") or data.get("findings") or []:
+        if not isinstance(finding, dict):
+            continue
+        image = str(finding.get("image") or finding.get("og_image") or "").strip()
+        if usable(image):
+            seen.add(image)
+            return image
+    return ""
+
+
+def _first_chat_model(models) -> str:
+    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
+    for m in (models or []):
+        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
+            return m
+    return (models[0] if models else "")
+
+
+def _resolve_research_endpoint(sess, owner: Optional[str] = None) -> tuple:
+    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
+    owner = owner or getattr(sess, "owner", None) or None
+    url, model, headers = resolve_endpoint(
+        "research",
+        fallback_url=sess.endpoint_url,
+        fallback_model=sess.model,
+        fallback_headers=sess.headers,
+        owner=owner,
+    )
+    return url, model, headers
+
+
+def _owned_enabled_endpoint(db, owner, endpoint_id=None):
+    """An enabled ModelEndpoint VISIBLE to `owner` (their own rows + legacy
+    null-owner "shared" rows), optionally narrowed to a specific endpoint_id;
+    None if nothing visible matches.
+
+    Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null
+    owner = private, "the model picker only shows the endpoint to that user") and
+    holds a decrypted `api_key`. /api/research/start feeds the resolved row's
+    api_key + base_url into research_handler.start_research(llm_endpoint=,
+    llm_headers=), so an UNSCOPED lookup — by the caller-supplied endpoint_id, or
+    via the bare first-enabled fallback — would let a research-privileged user
+    spend ANOTHER user's API key/quota and reach whatever internal base_url they
+    configured. Mirrors webhook_routes._first_enabled_endpoint and
+    session_routes._owned_endpoint. A null/empty owner is a no-op (single-user /
+    legacy mode).
+    """
+    from src.database import ModelEndpoint
+    from src.auth_helpers import owner_filter
+    q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if endpoint_id:
+        q = q.filter(ModelEndpoint.id == endpoint_id)
+    return owner_filter(q, ModelEndpoint, owner).first()
+
+
+def _resolve_endpoint_runtime(ep, owner=None, model: Optional[str] = None):
+    """Resolve a ModelEndpoint row into (chat_url, model, headers).
+
+    Mirrors endpoint_resolver.resolve_endpoint's provider-auth handling for
+    panel-selected research endpoints. ChatGPT Subscription endpoints keep
+    OAuth tokens in ProviderAuthSession, so ep.api_key is intentionally empty.
+    """
+    from src.endpoint_resolver import (
+        build_chat_url,
+        build_headers,
+        resolve_endpoint_runtime as resolve_model_endpoint_runtime,
+    )
+
+    try:
+        base, api_key = resolve_model_endpoint_runtime(ep, owner=owner)
+    except Exception as e:
+        logger.warning("Could not resolve endpoint credentials for research: %s", e)
+        return None
+
+    ep_model = (model or "").strip()
+    if not ep_model:
+        try:
+            models = json.loads(ep.cached_models) if ep.cached_models else []
+            if models:
+                ep_model = _first_chat_model(models)
+        except Exception:
+            pass
+    if not ep_model:
+        return None
+    return build_chat_url(base), ep_model, build_headers(api_key, base)
+
+
+def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
+    router = APIRouter(tags=["research"])
+
+    def _require_user(request: Request) -> str:
+        """All research endpoints require an authenticated user. Research
+        data isn't owner-scoped in the on-disk JSON yet, so we at least
+        block anonymous access. Multi-tenant deploys should additionally
+        verify the session belongs to this user."""
+        user = get_current_user(request)
+        if not user:
+            if _auth_disabled():
+                return ""
+            raise HTTPException(401, "Not authenticated")
+        return user
+
+    def _owns_in_memory(session_id: str, user: str) -> bool:
+        """Ownership check for an in-flight (in-memory) research task.
+        Falls back to the on-disk JSON if the task has already finished."""
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            return entry.get("owner", "") == user
+        # Task no longer in memory — check the persisted JSON.
+        try:
+            return _find_owned_research_path(session_id, user) is not None
+        except HTTPException:
+            return False
+
+    def _require_owned_or_active_research_path(session_id: str, user: str) -> Path | None:
+        """Validate ownership once and return the completed on-disk path.
+
+        Active running research has no completed disk path yet. Completed
+        tasks can remain in _active_tasks after persistence, so prefer their
+        owned disk path when available. Completed disk lookups still reuse the
+        path after the ownership gate.
+        """
+        entry = research_handler._active_tasks.get(session_id)
+        if entry is not None:
+            if entry.get("owner", "") != user:
+                raise HTTPException(404, "No research found for this session")
+            if entry.get("status") != "running":
+                path = _find_owned_research_path(session_id, user)
+                if path is not None:
+                    return path
+            return None
+
+        path = _find_owned_research_path(session_id, user)
+        if path is None:
+            raise HTTPException(404, "No research found for this session")
+        return path
+
+    @router.get("/api/research/active")
+    async def research_active(request: Request):
+        """List all currently active (running) research tasks."""
+        user = _require_user(request)
+        active = []
+        for sid, entry in research_handler._active_tasks.items():
+            # SECURITY: only show this user's running tasks.
+            if entry.get("owner", "") != user:
+                continue
+            if entry.get("status") == "running":
+                active.append({
+                    "session_id": sid,
+                    "query": entry.get("query", ""),
+                    "status": "running",
+                    "progress": entry.get("progress", {}),
+                    "source_state": research_handler.get_source_state(sid),
+                    "source_coverage": research_handler.get_source_coverage(sid),
+                    "navigation_trace": research_handler.get_navigation_trace(sid),
+                    "action_trace": research_handler.get_action_trace(sid),
+                    "started_at": entry.get("started_at", 0),
+                    "category": research_handler.get_category(sid),
+                    "mode": research_handler.get_mode(sid),
+                })
+        return {"active": active}
+
+    @router.get("/api/research/status/{session_id}")
+    async def research_status(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        status = research_handler.get_status(session_id)
+        if status is None:
+            raise HTTPException(404, "No research found for this session")
+        try:
+            source_state = research_handler.get_source_state(session_id)
+            if isinstance(source_state, str) and source_state:
+                status["source_state"] = source_state
+            source_coverage = research_handler.get_source_coverage(session_id)
+            if isinstance(source_coverage, dict) and source_coverage:
+                status["source_coverage"] = source_coverage
+        except Exception:
+            pass
+        try:
+            navigation_trace = research_handler.get_navigation_trace(session_id)
+            if isinstance(navigation_trace, list) and navigation_trace:
+                status["navigation_trace"] = navigation_trace
+            action_trace = research_handler.get_action_trace(session_id)
+            if isinstance(action_trace, list) and action_trace:
+                status["action_trace"] = action_trace
+        except Exception:
+            pass
+        return status
+
+    @router.post("/api/research/cancel/{session_id}")
+    async def research_cancel(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        cancelled = research_handler.cancel_research(session_id)
+        return {"cancelled": cancelled}
+
+    @router.post("/api/research/result/{session_id}")
+    async def research_result(session_id: str, request: Request):
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research result available")
+        result = research_handler.get_result(session_id)
+        if result is None:
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        category = research_handler.get_category(session_id)
+        mode = research_handler.get_mode(session_id)
+        research_handler.clear_result(session_id)
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": category,
+            "mode": mode,
+        }
+
+    def _assert_owns_research(session_id: str, user: str) -> None:
+        """404-not-403 ownership gate for a research session's on-disk JSON.
+        Use BEFORE returning any data or mutating the file."""
+        path = _require_research_path(session_id)
+        try:
+            owner = json.loads(path.read_text(encoding="utf-8")).get("owner")
+        except Exception:
+            raise HTTPException(404, "Research not found")
+        if owner != user:
+            raise HTTPException(404, "Research not found")
+
+    @router.get("/api/research/report/{session_id}")
+    async def research_report(session_id: str, request: Request):
+        """Serve the visual HTML report for a completed research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        logger.info(f"Visual report requested for session {session_id}")
+        try:
+            html_content = research_handler.get_report_html(session_id)
+        except Exception as e:
+            logger.error(f"Visual report generation error: {e}", exc_info=True)
+            raise HTTPException(500, f"Report generation failed: {e}")
+        if html_content is None:
+            logger.warning(f"No report data found for session {session_id}")
+            raise HTTPException(404, "No visual report available for this session")
+        return HTMLResponse(content=html_content)
+
+    class HideImageRequest(BaseModel):
+        url: str
+
+    @router.post("/api/research/{session_id}/hide-image")
+    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
+        """Mark an image URL as hidden for this research's visual report.
+        Persisted to the research JSON so subsequent /report renders skip it."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.hide_image(session_id, body.url)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.post("/api/research/{session_id}/unhide-images")
+    async def research_unhide_images(session_id: str, request: Request):
+        """Clear the hidden-images list for a research session."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        _assert_owns_research(session_id, user)
+        ok = research_handler.unhide_all_images(session_id)
+        if not ok:
+            raise HTTPException(404, "Research not found")
+        return {"ok": True}
+
+    @router.get("/api/research/library")
+    async def research_library(
+        request: Request,
+        search: Optional[str] = Query(None),
+        sort: str = Query("recent"),
+        limit: int = Query(50),
+        archived: bool = Query(False),
+    ):
+        user = _require_user(request)
+        """List all completed research for the Library panel."""
+        data_dir = Path(DEEP_RESEARCH_DIR)
+        items = []
+        for p in data_dir.glob("*.json"):
+            try:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                # SECURITY: only show research belonging to this user. Legacy
+                # JSONs without an `owner` field are hidden — auth was the only
+                # gate before, so every user saw every other user's reports.
+                if d.get("owner") != user:
+                    continue
+                # Archived view shows ONLY archived reports; default hides them.
+                if bool(d.get("archived")) != archived:
+                    continue
+                query = d.get("query", "")
+                if search and search.lower() not in query.lower():
+                    continue
+                sources = d.get("sources", [])
+                items.append({
+                    "id": p.stem,
+                    "query": query,
+                    "category": d.get("category") or "",
+                    "mode": d.get("mode") or "research",
+                    "mode": d.get("mode") or "research",
+                    "source_count": len(sources),
+                    "status": d.get("status", "done"),
+                    "duration": d.get("stats", {}).get("Duration", ""),
+                    "rounds": d.get("stats", {}).get("Rounds", ""),
+                    "started_at": d.get("started_at", 0),
+                    "completed_at": d.get("completed_at", 0),
+                    "archived": bool(d.get("archived")),
+                    "thumbnail": _research_thumbnail(d),
+                })
+            except Exception:
+                continue
+
+        # Sort
+        if sort == "recent":
+            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
+        elif sort == "oldest":
+            items.sort(key=lambda x: x["completed_at"] or 0)
+        elif sort == "most-messages":
+            items.sort(key=lambda x: x["source_count"], reverse=True)
+        elif sort == "alpha":
+            items.sort(key=lambda x: x["query"].lower())
+
+        return {"research": items[:limit], "total": len(items)}
+
+    @router.get("/api/research/detail/{session_id}")
+    async def research_detail(session_id: str, request: Request):
+        """Return the full JSON for a single research result — sources,
+        summary, stats — used by the Library preview panel."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _require_research_path(session_id)
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as e:
+            raise HTTPException(500, f"Failed to read research: {e}")
+        # SECURITY: 404 (not 403) so we don't leak that the report exists.
+        if data.get("owner") != user:
+            raise HTTPException(404, "Research not found")
+        return data
+
+    @router.post("/api/research/{session_id}/archive")
+    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
+        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        path = _require_research_path(session_id)
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("owner") != user:
+                raise HTTPException(404, "Research not found")
+            data["archived"] = bool(archived)
+            path.write_text(json.dumps(data), encoding="utf-8")
+        except HTTPException:
+            raise
+        except Exception as e:
+            raise HTTPException(500, f"Failed to update research: {e}")
+        return {"ok": True, "id": session_id, "archived": bool(archived)}
+
+    @router.delete("/api/research/{session_id}")
+    async def research_delete(session_id: str, request: Request):
+        """Delete a research result from disk."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        json_path = _find_research_path(session_id)
+        deleted = False
+        if json_path is not None:
+            # SECURITY: verify ownership before letting the caller delete it.
+            try:
+                data = json.loads(json_path.read_text(encoding="utf-8"))
+                if data.get("owner") != user:
+                    raise HTTPException(404, "Research not found")
+            except HTTPException:
+                raise
+            except Exception:
+                raise HTTPException(404, "Research not found")
+            json_path.unlink()
+            deleted = True
+        return {"deleted": deleted}
+
+    # ------------------------------------------------------------------
+    # Panel endpoints — launch research without a chat session
+    # ------------------------------------------------------------------
+
+    class ResearchStartRequest(BaseModel):
+        query: str
+        origin_chat_id: Optional[str] = None
+        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
+        max_rounds: int = Field(default=0, ge=0, le=20)
+        search_provider: Optional[str] = None
+        endpoint_id: Optional[str] = None
+        model: Optional[str] = None
+        max_time: int = Field(default=300, ge=60, le=1800)
+        extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
+        extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
+        category: Optional[Literal["product", "comparison", "howto", "factcheck"]] = None
+
+    @router.post("/api/research/start")
+    async def research_start(body: ResearchStartRequest, request: Request):
+        """Launch a research job from the dedicated panel."""
+        from src.auth_helpers import require_privilege
+        user = require_privilege(request, "can_use_research")
+        if user == INTERNAL_TOOL_USER:
+            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
+            if tool_owner and tool_owner not in REQUEST_SENTINEL_OWNERS:
+                auth_mgr = getattr(request.app.state, "auth_manager", None)
+                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
+                    try:
+                        privs = auth_mgr.get_privileges(tool_owner) or {}
+                        if not privs.get("can_use_research", True):
+                            raise HTTPException(403, f"Your account is not allowed to can use research.")
+                    except HTTPException:
+                        raise
+                    except Exception:
+                        pass
+                user = tool_owner
+        session_id = f"rp-{uuid.uuid4().hex[:12]}"
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        if body.origin_chat_id:
+            from core.database import SessionLocal, Session as DbSession
+            with SessionLocal() as db:
+                origin = db.get(DbSession, body.origin_chat_id)
+                if origin is None or origin.owner != user:
+                    raise HTTPException(404, 'Origin chat not found')
+            if delivery is None:
+                raise HTTPException(503, 'Background chat delivery is unavailable')
+
+        if body.endpoint_id:
+            from src.database import SessionLocal
+            db = SessionLocal()
+            try:
+                # Owner-scoped: never resolve another user's private endpoint
+                # (and its decrypted api_key / internal base_url). A scoped miss
+                # reads as 404 so the endpoint's existence isn't revealed.
+                ep = _owned_enabled_endpoint(db, user, body.endpoint_id)
+                if not ep:
+                    raise HTTPException(404, "Endpoint not found or disabled")
+                resolved = _resolve_endpoint_runtime(ep, owner=user, model=body.model)
+                if not resolved:
+                    raise HTTPException(400, "Endpoint is not configured with a usable model.")
+                ep_url, ep_model, ep_headers = resolved
+            finally:
+                db.close()
+        else:
+            ep_url, ep_model, ep_headers = resolve_endpoint("research", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("utility", owner=user)
+            # When neither research nor utility is configured, use the user's
+            # configured DEFAULT model (default_endpoint_id/default_model) rather
+            # than arbitrarily grabbing the first enabled endpoint's first model
+            # (which surfaced gpt-3.5). "Default" should mean the default model.
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("default", owner=user)
+            if not ep_url:
+                ep_url, ep_model, ep_headers = resolve_endpoint("chat", owner=user)
+            if not ep_url:
+                from src.database import SessionLocal
+                db = SessionLocal()
+                try:
+                    # Owner-scoped first-enabled fallback: the caller's own rows
+                    # + legacy null-owner shared rows only — never borrow another
+                    # user's private endpoint/api_key. Same fix as the
+                    # /api/v1/chat fallback (webhook_routes._first_enabled_endpoint).
+                    ep = _owned_enabled_endpoint(db, user)
+                    if ep:
+                        resolved = _resolve_endpoint_runtime(ep, owner=user)
+                        if resolved:
+                            ep_url, ep_model, ep_headers = resolved
+                finally:
+                    db.close()
+            if not ep_url:
+                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
+            if body.model:
+                ep_model = body.model
+
+        # 0 = auto research capped at 20.
+        effective_max_rounds = body.max_rounds if body.max_rounds != 0 else 20
+        if body.origin_chat_id and 'max_rounds' not in body.model_fields_set:
+            effective_max_rounds = 2
+        if body.origin_chat_id:
+            delivery.register(session_id, body.origin_chat_id, user, 'research', body.query, effective_max_rounds)
+        research_handler.start_research(
+            session_id=session_id,
+            query=body.query,
+            llm_endpoint=ep_url,
+            llm_model=ep_model,
+            max_time=body.max_time,
+            llm_headers=ep_headers,
+            max_rounds=effective_max_rounds,
+            search_provider=body.search_provider or None,
+            category=body.category or None,
+            extraction_timeout=body.extraction_timeout,
+            extraction_concurrency=body.extraction_concurrency,
+            owner=user,
+            on_complete=(lambda sid, result, sources, findings: delivery.complete(sid, result, sources))
+                if body.origin_chat_id else None,
+        )
+        return {
+            "session_id": session_id,
+            "status": "running",
+            "query": body.query,
+            "category": body.category or "",
+            "mode": "research",
+        }
+
+    @router.get('/api/research/chat-jobs/{chat_id}')
+    async def chat_research_jobs(chat_id: str, request: Request):
+        user = _require_user(request)
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        return {'jobs': delivery.list_for_chat(chat_id, user) if delivery else []}
+
+    @router.get("/api/research/stream/{session_id}")
+    async def research_stream(session_id: str, request: Request):
+        """SSE stream of research progress events."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        if not _owns_in_memory(session_id, user):
+            raise HTTPException(404, "No research found for this session")
+        async def _generate():
+            last_payload = None
+            while True:
+                status = research_handler.get_status(session_id)
+                if status is None:
+                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
+                    return
+                st = status.get("status", "")
+                progress = status.get("progress", {})
+                payload = {
+                    **progress,
+                    'status': st,
+                    'category': research_handler.get_category(session_id),
+                    'mode': research_handler.get_mode(session_id),
+                }
+                try:
+                    source_state = research_handler.get_source_state(session_id)
+                    if source_state:
+                        payload["source_state"] = source_state
+                    source_coverage = research_handler.get_source_coverage(session_id)
+                    if source_coverage:
+                        payload["source_coverage"] = source_coverage
+                    navigation_trace = research_handler.get_navigation_trace(session_id)
+                    if navigation_trace:
+                        payload["navigation_trace"] = navigation_trace
+                    action_trace = research_handler.get_action_trace(session_id)
+                    if action_trace:
+                        payload["action_trace"] = action_trace
+                except Exception:
+                    pass
+                if payload != last_payload:
+                    last_payload = payload
+                    yield f"data: {json.dumps(payload)}\n\n"
+                if st != "running":
+                    final = {'status': st, 'final': True}
+                    task = research_handler._active_tasks.get(session_id, {})
+                    if st == "error" and task.get("result"):
+                        final['error'] = str(task["result"])[:500]
+                    yield f"data: {json.dumps(final)}\n\n"
+                    return
+                await asyncio.sleep(1.5)
+
+        return StreamingResponse(
+            _generate(),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+    @router.post("/api/research/result-peek/{session_id}")
+    async def research_result_peek(session_id: str, request: Request):
+        """Get research result without clearing it (for panel use)."""
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
+        result = research_handler.get_result(session_id)
+        if result is None:
+            p = owned_disk_path
+            if p is not None:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                return {
+                    "result": d.get("result", ""),
+                    "sources": d.get("sources", []),
+                    "raw_findings": d.get("raw_findings", []),
+                    "analyzed_urls": d.get("analyzed_urls", []),
+                    "source_state": d.get("source_state", ""),
+                    "source_coverage": d.get("source_coverage", {}),
+                    "navigation_trace": d.get("navigation_trace", []),
+                    "action_trace": d.get("action_trace", []),
+                    "category": d.get("category") or "",
+                }
+            raise HTTPException(404, "No research result available")
+        sources = research_handler.get_sources(session_id) or []
+        raw_findings = research_handler.get_raw_findings(session_id) or []
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": research_handler.get_category(session_id),
+            "mode": research_handler.get_mode(session_id),
+        }
+
+    @router.post("/api/research/spinoff/{session_id}")
+    async def research_spinoff(session_id: str, request: Request):
+        """Create a new chat session pre-seeded with this research as context.
+
+        Reads the persisted research result + sources for `session_id`, creates
+        a fresh session (inheriting endpoint/model/headers from the source
+        session if available, otherwise from the resolved chat endpoint), and
+        injects a single system message containing the report and sources so
+        the user can ask follow-up questions in a clean conversation.
+        """
+        user = _require_user(request)
+        _validate_session_id(session_id)
+        # SECURITY: gate on ownership before reading the persisted research —
+        # otherwise any authenticated user could spin off (and thereby read)
+        # another user's report by guessing its session ID. Mirrors every other
+        # endpoint in this file (see result_peek above).
+        owned_disk_path = _require_owned_or_active_research_path(session_id, user)
+        if session_manager is None:
+            raise HTTPException(500, "session_manager not configured")
+
+        # Load research data — prefer in-memory result, fall back to disk
+        result = research_handler.get_result(session_id)
+        sources = research_handler.get_sources(session_id) or []
+        query = ""
+
+        path = owned_disk_path
+        if path is not None:
+            try:
+                disk = json.loads(path.read_text(encoding="utf-8"))
+                if not result:
+                    result = disk.get("result")
+                if not sources:
+                    sources = disk.get("sources", []) or []
+                query = disk.get("query", "") or ""
+            except Exception as e:
+                logger.warning(f"Could not read research JSON for spinoff: {e}")
+
+        if not result:
+            raise HTTPException(404, "No research result available for this session")
+
+        # Inherit endpoint/model/headers from the source session when possible.
+        # For panel-launched research (rp-* IDs), there is no chat session, so
+        # fall back through the same chain as /api/research/start: research →
+        # utility → first enabled endpoint in the DB.
+        ep_url, ep_model, ep_headers = "", "", {}
+        try:
+            src_sess = session_manager.get_session(session_id)
+            ep_url = src_sess.endpoint_url or ""
+            ep_model = src_sess.model or ""
+            ep_headers = dict(src_sess.headers or {})
+        except KeyError:
+            pass
+
+        def _merge(r_url, r_model, r_headers):
+            nonlocal ep_url, ep_model, ep_headers
+            if not ep_url and r_url:
+                ep_url = r_url
+            if not ep_model and r_model:
+                ep_model = r_model
+            if not ep_headers and r_headers:
+                ep_headers = dict(r_headers)
+
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("chat", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("research", owner=user))
+        if not ep_url or not ep_model:
+            _merge(*resolve_endpoint("utility", owner=user))
+        if not ep_url or not ep_model:
+            # Last resort: this user's enabled endpoint, plus legacy shared rows.
+            from src.database import SessionLocal
+            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
+            db = SessionLocal()
+            try:
+                ep = _owned_enabled_endpoint(db, user)
+                if ep:
+                    base = normalize_base(ep.base_url)
+                    fallback_url = build_chat_url(base)
+                    fallback_headers = build_headers(ep.api_key, base)
+                    fallback_model = ""
+                    if ep.cached_models:
+                        try:
+                            models = json.loads(ep.cached_models)
+                            if models:
+                                fallback_model = _first_chat_model(models)
+                        except Exception:
+                            pass
+                    _merge(fallback_url, fallback_model, fallback_headers)
+            finally:
+                db.close()
+
+        if not ep_url or not ep_model:
+            raise HTTPException(400, "No endpoint configured — add one in Settings first")
+
+        # Create new session
+        new_sid = str(uuid.uuid4())
+
+        title_query = (query or "research").strip()
+        if len(title_query) > 60:
+            title_query = title_query[:57] + "…"
+        new_name = f"Follow-up: {title_query}"
+
+        new_sess = session_manager.create_session(
+            session_id=new_sid,
+            name=new_name,
+            endpoint_url=ep_url,
+            model=ep_model,
+            rag=False,
+            owner=user,
+        )
+        if ep_headers:
+            new_sess.headers = ep_headers
+            session_manager.save_sessions()
+        try:
+            from src.event_bus import fire_event
+            fire_event("session_created", user)
+        except Exception:
+            logger.debug("session_created event dispatch failed", exc_info=True)
+
+        # Build the priming system message — report only, no sources injected.
+        # The user can open the visual report for source details; keeping sources
+        # out of the chat context saves tokens and avoids the AI fabricating
+        # citations.
+        date_str = datetime.utcnow().strftime("%Y-%m-%d")
+        primer = (
+            f"[Research context — {date_str}]\n\n"
+            f"The user previously ran a deep research investigation. Use the "
+            f"report below as your primary knowledge base when answering "
+            f"follow-up questions. If the user asks something not covered, "
+            f"say so plainly rather than guessing.\n\n"
+            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
+            f"=== REPORT ===\n{result}"
+        )
+
+        from core.models import ChatMessage
+        new_sess.add_message(ChatMessage(
+            role="system",
+            content=primer,
+            metadata={"research_spinoff_from": session_id},
+        ))
+        session_manager.save_sessions()
+
+        return {
+            "session_id": new_sid,
+            "name": new_name,
+            "source_count": len(sources),
+        }
+
+    return router

routes/research_routes.py

@@ -1,607 +1,17 @@
-"""Research background task routes — /api/research/*."""
+"""Backward-compat shim — canonical location is routes/research/research_routes.py.
 
-import asyncio
-import json
-import logging
-import uuid
-from datetime import datetime
-from pathlib import Path
-from typing import Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.research_routes``, ``from routes.research_routes import X``,
+``importlib.import_module("routes.research_routes")``, and
+``monkeypatch.setattr("routes.research_routes.ATTR", ...)`` (string-targeted
+patch used by ``test_research_owner_scope_routes.py``) all operate on the
+*same* object the application actually uses. Keeps existing import paths
+working after slice 2b (#4082/#4071). Source-introspection tests read the
+canonical file by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Query, Request
-from fastapi.responses import HTMLResponse, StreamingResponse
-from pydantic import BaseModel, Field
-from src.endpoint_resolver import resolve_endpoint
-from src.auth_helpers import get_current_user
+import sys as _sys
 
-logger = logging.getLogger(__name__)
+from routes.research import research_routes as _canonical  # noqa: F401
 
-# Model-name substrings that are NOT chat/generation models — research must
-# never pick these as its model. An OpenAI-style endpoint often lists
-# `text-embedding-ada-002` etc. first in its model list, which is why research
-# was failing with "Cannot reach model 'text-embedding-ada-002'".
-_NON_CHAT_MODEL = (
-    "text-embedding", "embedding", "tts-", "whisper", "dall-e",
-    "moderation", "rerank", "reranker", "clip", "stable-diffusion",
-)
-
-
-def _first_chat_model(models) -> str:
-    """First model that isn't an embedding/tts/etc. — falls back to models[0]."""
-    for m in (models or []):
-        if not any(p in str(m).lower() for p in _NON_CHAT_MODEL):
-            return m
-    return (models[0] if models else "")
-
-
-def _resolve_research_endpoint(sess) -> tuple:
-    """Return (endpoint_url, model, headers) for Deep Research, checking admin overrides."""
-    url, model, headers = resolve_endpoint(
-        "research",
-        fallback_url=sess.endpoint_url,
-        fallback_model=sess.model,
-        fallback_headers=sess.headers,
-    )
-    return url, model, headers
-
-
-def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
-    router = APIRouter(tags=["research"])
-
-    def _require_user(request: Request) -> str:
-        """All research endpoints require an authenticated user. Research
-        data isn't owner-scoped in the on-disk JSON yet, so we at least
-        block anonymous access. Multi-tenant deploys should additionally
-        verify the session belongs to this user."""
-        user = get_current_user(request)
-        if not user:
-            raise HTTPException(401, "Not authenticated")
-        return user
-
-    def _owns_in_memory(session_id: str, user: str) -> bool:
-        """Ownership check for an in-flight (in-memory) research task.
-        Falls back to the on-disk JSON if the task has already finished."""
-        entry = research_handler._active_tasks.get(session_id)
-        if entry is not None:
-            return entry.get("owner", "") == user
-        # Task no longer in memory — check the persisted JSON.
-        path = Path("data/deep_research") / f"{session_id}.json"
-        if not path.exists():
-            return False
-        try:
-            return json.loads(path.read_text()).get("owner") == user
-        except Exception:
-            return False
-
-    @router.get("/api/research/active")
-    async def research_active(request: Request):
-        """List all currently active (running) research tasks."""
-        user = _require_user(request)
-        active = []
-        for sid, entry in research_handler._active_tasks.items():
-            # SECURITY: only show this user's running tasks.
-            if entry.get("owner", "") != user:
-                continue
-            if entry.get("status") == "running":
-                active.append({
-                    "session_id": sid,
-                    "query": entry.get("query", ""),
-                    "status": "running",
-                    "progress": entry.get("progress", {}),
-                    "started_at": entry.get("started_at", 0),
-                })
-        return {"active": active}
-
-    @router.get("/api/research/status/{session_id}")
-    async def research_status(session_id: str, request: Request):
-        user = _require_user(request)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        status = research_handler.get_status(session_id)
-        if status is None:
-            raise HTTPException(404, "No research found for this session")
-        return status
-
-    @router.post("/api/research/cancel/{session_id}")
-    async def research_cancel(session_id: str, request: Request):
-        user = _require_user(request)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        cancelled = research_handler.cancel_research(session_id)
-        return {"cancelled": cancelled}
-
-    @router.post("/api/research/result/{session_id}")
-    async def research_result(session_id: str, request: Request):
-        user = _require_user(request)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research result available")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
-
-    def _assert_owns_research(session_id: str, user: str) -> None:
-        """404-not-403 ownership gate for a research session's on-disk JSON.
-        Use BEFORE returning any data or mutating the file."""
-        path = Path("data/deep_research") / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            owner = json.loads(path.read_text()).get("owner")
-        except Exception:
-            raise HTTPException(404, "Research not found")
-        if owner != user:
-            raise HTTPException(404, "Research not found")
-
-    @router.get("/api/research/report/{session_id}")
-    async def research_report(session_id: str, request: Request):
-        """Serve the visual HTML report for a completed research session."""
-        user = _require_user(request)
-        _assert_owns_research(session_id, user)
-        logger.info(f"Visual report requested for session {session_id}")
-        try:
-            html_content = research_handler.get_report_html(session_id)
-        except Exception as e:
-            logger.error(f"Visual report generation error: {e}", exc_info=True)
-            raise HTTPException(500, f"Report generation failed: {e}")
-        if html_content is None:
-            logger.warning(f"No report data found for session {session_id}")
-            raise HTTPException(404, "No visual report available for this session")
-        return HTMLResponse(content=html_content)
-
-    class HideImageRequest(BaseModel):
-        url: str
-
-    @router.post("/api/research/{session_id}/hide-image")
-    async def research_hide_image(session_id: str, body: HideImageRequest, request: Request):
-        """Mark an image URL as hidden for this research's visual report.
-        Persisted to the research JSON so subsequent /report renders skip it."""
-        user = _require_user(request)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.hide_image(session_id, body.url)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.post("/api/research/{session_id}/unhide-images")
-    async def research_unhide_images(session_id: str, request: Request):
-        """Clear the hidden-images list for a research session."""
-        user = _require_user(request)
-        _assert_owns_research(session_id, user)
-        ok = research_handler.unhide_all_images(session_id)
-        if not ok:
-            raise HTTPException(404, "Research not found")
-        return {"ok": True}
-
-    @router.get("/api/research/library")
-    async def research_library(
-        request: Request,
-        search: Optional[str] = Query(None),
-        sort: str = Query("recent"),
-        limit: int = Query(50),
-        archived: bool = Query(False),
-    ):
-        user = _require_user(request)
-        """List all completed research for the Library panel."""
-        data_dir = Path("data/deep_research")
-        items = []
-        for p in data_dir.glob("*.json"):
-            try:
-                d = json.loads(p.read_text())
-                # SECURITY: only show research belonging to this user. Legacy
-                # JSONs without an `owner` field are hidden — auth was the only
-                # gate before, so every user saw every other user's reports.
-                if d.get("owner") != user:
-                    continue
-                # Archived view shows ONLY archived reports; default hides them.
-                if bool(d.get("archived")) != archived:
-                    continue
-                query = d.get("query", "")
-                if search and search.lower() not in query.lower():
-                    continue
-                sources = d.get("sources", [])
-                items.append({
-                    "id": p.stem,
-                    "query": query,
-                    "category": d.get("category") or "",
-                    "source_count": len(sources),
-                    "status": d.get("status", "done"),
-                    "duration": d.get("stats", {}).get("Duration", ""),
-                    "rounds": d.get("stats", {}).get("Rounds", ""),
-                    "started_at": d.get("started_at", 0),
-                    "completed_at": d.get("completed_at", 0),
-                    "archived": bool(d.get("archived")),
-                })
-            except Exception:
-                continue
-
-        # Sort
-        if sort == "recent":
-            items.sort(key=lambda x: x["completed_at"] or 0, reverse=True)
-        elif sort == "oldest":
-            items.sort(key=lambda x: x["completed_at"] or 0)
-        elif sort == "most-messages":
-            items.sort(key=lambda x: x["source_count"], reverse=True)
-        elif sort == "alpha":
-            items.sort(key=lambda x: x["query"].lower())
-
-        return {"research": items[:limit], "total": len(items)}
-
-    @router.get("/api/research/detail/{session_id}")
-    async def research_detail(session_id: str, request: Request):
-        """Return the full JSON for a single research result — sources,
-        summary, stats — used by the Library preview panel."""
-        user = _require_user(request)
-        path = Path("data/deep_research") / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text())
-        except Exception as e:
-            raise HTTPException(500, f"Failed to read research: {e}")
-        # SECURITY: 404 (not 403) so we don't leak that the report exists.
-        if data.get("owner") != user:
-            raise HTTPException(404, "Research not found")
-        return data
-
-    @router.post("/api/research/{session_id}/archive")
-    async def research_archive(session_id: str, request: Request, archived: bool = Query(True)):
-        """Soft-archive / restore a research report (sets `archived` in its JSON)."""
-        user = _require_user(request)
-        path = Path("data/deep_research") / f"{session_id}.json"
-        if not path.exists():
-            raise HTTPException(404, "Research not found")
-        try:
-            data = json.loads(path.read_text())
-            if data.get("owner") != user:
-                raise HTTPException(404, "Research not found")
-            data["archived"] = bool(archived)
-            path.write_text(json.dumps(data))
-        except HTTPException:
-            raise
-        except Exception as e:
-            raise HTTPException(500, f"Failed to update research: {e}")
-        return {"ok": True, "id": session_id, "archived": bool(archived)}
-
-    @router.delete("/api/research/{session_id}")
-    async def research_delete(session_id: str, request: Request):
-        """Delete a research result from disk."""
-        user = _require_user(request)
-        data_dir = Path("data/deep_research")
-        json_path = data_dir / f"{session_id}.json"
-        deleted = False
-        if json_path.exists():
-            # SECURITY: verify ownership before letting the caller delete it.
-            try:
-                data = json.loads(json_path.read_text())
-                if data.get("owner") != user:
-                    raise HTTPException(404, "Research not found")
-            except HTTPException:
-                raise
-            except Exception:
-                raise HTTPException(404, "Research not found")
-            json_path.unlink()
-            deleted = True
-        return {"deleted": deleted}
-
-    # ------------------------------------------------------------------
-    # Panel endpoints — launch research without a chat session
-    # ------------------------------------------------------------------
-
-    class ResearchStartRequest(BaseModel):
-        query: str
-        # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
-        max_rounds: int = Field(default=0, ge=0, le=20)
-        search_provider: Optional[str] = None
-        endpoint_id: Optional[str] = None
-        model: Optional[str] = None
-        max_time: int = Field(default=300, ge=60, le=1800)
-        category: Optional[str] = None
-
-    @router.post("/api/research/start")
-    async def research_start(body: ResearchStartRequest, request: Request):
-        """Launch a research job from the dedicated panel."""
-        from src.auth_helpers import require_privilege
-        user = require_privilege(request, "can_use_research")
-        if user == "internal-tool":
-            tool_owner = (request.headers.get("X-Odysseus-Owner") or "").strip()
-            if tool_owner and tool_owner not in {"internal-tool", "api", "demo", "system"}:
-                auth_mgr = getattr(request.app.state, "auth_manager", None)
-                if auth_mgr is not None and getattr(auth_mgr, "is_configured", False):
-                    try:
-                        privs = auth_mgr.get_privileges(tool_owner) or {}
-                        if not privs.get("can_use_research", True):
-                            raise HTTPException(403, f"Your account is not allowed to can use research.")
-                    except HTTPException:
-                        raise
-                    except Exception:
-                        pass
-                user = tool_owner
-        session_id = f"rp-{uuid.uuid4().hex[:12]}"
-
-        if body.endpoint_id:
-            from src.database import SessionLocal
-            from src.database import ModelEndpoint
-            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-            db = SessionLocal()
-            try:
-                ep = db.query(ModelEndpoint).filter(
-                    ModelEndpoint.id == body.endpoint_id,
-                    ModelEndpoint.is_enabled == True,
-                ).first()
-                if not ep:
-                    raise HTTPException(404, "Endpoint not found or disabled")
-                base = normalize_base(ep.base_url)
-                ep_url = build_chat_url(base)
-                ep_headers = build_headers(ep.api_key, base)
-                ep_model = body.model or ""
-                if not ep_model:
-                    try:
-                        import json as _json
-                        models = _json.loads(ep.cached_models) if ep.cached_models else []
-                        if models:
-                            ep_model = _first_chat_model(models)
-                    except Exception:
-                        pass
-            finally:
-                db.close()
-        else:
-            ep_url, ep_model, ep_headers = resolve_endpoint("research")
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("utility")
-            # When neither research nor utility is configured, use the user's
-            # configured DEFAULT model (default_endpoint_id/default_model) rather
-            # than arbitrarily grabbing the first enabled endpoint's first model
-            # (which surfaced gpt-3.5). "Default" should mean the default model.
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("default")
-            if not ep_url:
-                ep_url, ep_model, ep_headers = resolve_endpoint("chat")
-            if not ep_url:
-                from src.database import SessionLocal
-                from src.database import ModelEndpoint
-                from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-                db = SessionLocal()
-                try:
-                    ep = db.query(ModelEndpoint).filter(
-                        ModelEndpoint.is_enabled == True,
-                    ).first()
-                    if ep:
-                        base = normalize_base(ep.base_url)
-                        ep_url = build_chat_url(base)
-                        ep_headers = build_headers(ep.api_key, base)
-                        ep_model = ""
-                        if ep.cached_models:
-                            try:
-                                import json as _json
-                                models = _json.loads(ep.cached_models)
-                                if models:
-                                    ep_model = _first_chat_model(models)
-                            except Exception:
-                                pass
-                finally:
-                    db.close()
-            if not ep_url:
-                raise HTTPException(400, "No endpoints configured. Add one in Settings first.")
-            if body.model:
-                ep_model = body.model
-
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
-        research_handler.start_research(
-            session_id=session_id,
-            query=body.query,
-            llm_endpoint=ep_url,
-            llm_model=ep_model,
-            max_time=body.max_time,
-            llm_headers=ep_headers,
-            max_rounds=effective_max_rounds,
-            search_provider=body.search_provider or None,
-            category=body.category or None,
-            owner=user,
-        )
-        return {"session_id": session_id, "status": "running", "query": body.query}
-
-    @router.get("/api/research/stream/{session_id}")
-    async def research_stream(session_id: str, request: Request):
-        """SSE stream of research progress events."""
-        user = _require_user(request)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        async def _generate():
-            last_progress = None
-            while True:
-                status = research_handler.get_status(session_id)
-                if status is None:
-                    yield f"data: {json.dumps({'status': 'not_found'})}\n\n"
-                    return
-                st = status.get("status", "")
-                progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
-                if st != "running":
-                    final = {'status': st, 'final': True}
-                    task = research_handler._active_tasks.get(session_id, {})
-                    if st == "error" and task.get("result"):
-                        final['error'] = str(task["result"])[:500]
-                    yield f"data: {json.dumps(final)}\n\n"
-                    return
-                await asyncio.sleep(1.5)
-
-        return StreamingResponse(
-            _generate(),
-            media_type="text/event-stream",
-            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
-        )
-
-    @router.post("/api/research/result-peek/{session_id}")
-    async def research_result_peek(session_id: str, request: Request):
-        """Get research result without clearing it (for panel use)."""
-        user = _require_user(request)
-        if not _owns_in_memory(session_id, user):
-            raise HTTPException(404, "No research found for this session")
-        result = research_handler.get_result(session_id)
-        if result is None:
-            p = Path("data/deep_research") / f"{session_id}.json"
-            if p.exists():
-                d = json.loads(p.read_text())
-                return {
-                    "result": d.get("result", ""),
-                    "sources": d.get("sources", []),
-                    "raw_findings": d.get("raw_findings", []),
-                    "category": d.get("category") or "",
-                }
-            raise HTTPException(404, "No research result available")
-        sources = research_handler.get_sources(session_id) or []
-        raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
-
-    @router.post("/api/research/spinoff/{session_id}")
-    async def research_spinoff(session_id: str, request: Request):
-        """Create a new chat session pre-seeded with this research as context.
-
-        Reads the persisted research result + sources for `session_id`, creates
-        a fresh session (inheriting endpoint/model/headers from the source
-        session if available, otherwise from the resolved chat endpoint), and
-        injects a single system message containing the report and sources so
-        the user can ask follow-up questions in a clean conversation.
-        """
-        _require_user(request)
-        if session_manager is None:
-            raise HTTPException(500, "session_manager not configured")
-
-        # Load research data — prefer in-memory result, fall back to disk
-        result = research_handler.get_result(session_id)
-        sources = research_handler.get_sources(session_id) or []
-        query = ""
-
-        path = Path("data/deep_research") / f"{session_id}.json"
-        if path.exists():
-            try:
-                disk = json.loads(path.read_text())
-                if not result:
-                    result = disk.get("result")
-                if not sources:
-                    sources = disk.get("sources", []) or []
-                query = disk.get("query", "") or ""
-            except Exception as e:
-                logger.warning(f"Could not read research JSON for spinoff: {e}")
-
-        if not result:
-            raise HTTPException(404, "No research result available for this session")
-
-        # Inherit endpoint/model/headers from the source session when possible.
-        # For panel-launched research (rp-* IDs), there is no chat session, so
-        # fall back through the same chain as /api/research/start: research →
-        # utility → first enabled endpoint in the DB.
-        ep_url, ep_model, ep_headers = "", "", {}
-        try:
-            src_sess = session_manager.get_session(session_id)
-            ep_url = src_sess.endpoint_url or ""
-            ep_model = src_sess.model or ""
-            ep_headers = dict(src_sess.headers or {})
-        except KeyError:
-            pass
-
-        def _merge(r_url, r_model, r_headers):
-            nonlocal ep_url, ep_model, ep_headers
-            if not ep_url and r_url:
-                ep_url = r_url
-            if not ep_model and r_model:
-                ep_model = r_model
-            if not ep_headers and r_headers:
-                ep_headers = dict(r_headers)
-
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("chat"))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("research"))
-        if not ep_url or not ep_model:
-            _merge(*resolve_endpoint("utility"))
-        if not ep_url or not ep_model:
-            # Last resort: any enabled endpoint
-            from src.database import SessionLocal
-            from src.database import ModelEndpoint
-            from src.endpoint_resolver import normalize_base, build_chat_url, build_headers
-            db = SessionLocal()
-            try:
-                ep = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).first()
-                if ep:
-                    base = normalize_base(ep.base_url)
-                    fallback_url = build_chat_url(base)
-                    fallback_headers = build_headers(ep.api_key, base)
-                    fallback_model = ""
-                    if ep.cached_models:
-                        try:
-                            models = json.loads(ep.cached_models)
-                            if models:
-                                fallback_model = models[0]
-                        except Exception:
-                            pass
-                    _merge(fallback_url, fallback_model, fallback_headers)
-            finally:
-                db.close()
-
-        if not ep_url or not ep_model:
-            raise HTTPException(400, "No endpoint configured — add one in Settings first")
-
-        # Create new session
-        new_sid = str(uuid.uuid4())
-        user = get_current_user(request)
-
-        title_query = (query or "research").strip()
-        if len(title_query) > 60:
-            title_query = title_query[:57] + "…"
-        new_name = f"Follow-up: {title_query}"
-
-        new_sess = session_manager.create_session(
-            session_id=new_sid,
-            name=new_name,
-            endpoint_url=ep_url,
-            model=ep_model,
-            rag=False,
-            owner=user,
-        )
-        if ep_headers:
-            new_sess.headers = ep_headers
-            session_manager.save_sessions()
-        try:
-            from src.event_bus import fire_event
-            fire_event("session_created", user)
-        except Exception:
-            logger.debug("session_created event dispatch failed", exc_info=True)
-
-        # Build the priming system message — report only, no sources injected.
-        # The user can open the visual report for source details; keeping sources
-        # out of the chat context saves tokens and avoids the AI fabricating
-        # citations.
-        date_str = datetime.utcnow().strftime("%Y-%m-%d")
-        primer = (
-            f"[Research context — {date_str}]\n\n"
-            f"The user previously ran a deep research investigation. Use the "
-            f"report below as your primary knowledge base when answering "
-            f"follow-up questions. If the user asks something not covered, "
-            f"say so plainly rather than guessing.\n\n"
-            f"=== ORIGINAL QUERY ===\n{query or '(not recorded)'}\n\n"
-            f"=== REPORT ===\n{result}"
-        )
-
-        from core.models import ChatMessage
-        new_sess.add_message(ChatMessage(
-            role="system",
-            content=primer,
-            metadata={"research_spinoff_from": session_id},
-        ))
-        session_manager.save_sessions()
-
-        return {
-            "session_id": new_sid,
-            "name": new_name,
-            "source_count": len(sources),
-        }
-
-    return router
+_sys.modules[__name__] = _canonical

routes/search/__init__.py

@@ -0,0 +1,5 @@
+"""Search route domain package (slice 2j, #4082/#4071).
+
+Contains search_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/search_routes.py re-exports from here.
+"""

routes/search/search_routes.py

@@ -0,0 +1,199 @@
+"""Search routes — /api/search/config GET, /api/search POST."""
+
+import html
+import json
+import logging
+from typing import Dict, Any
+
+from fastapi import APIRouter, Query, Request
+from fastapi.responses import HTMLResponse
+
+import time
+
+from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
+from services.search.core import _call_provider
+from services.search.providers import _get_provider_key, _get_search_instance
+
+logger = logging.getLogger(__name__)
+
+
+async def _request_values(request: Request) -> Dict[str, Any]:
+    """Accept JSON, form data, or query params for search endpoints.
+
+    The browser UI posts FormData, while the agent's generic app_api tool
+    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
+    runs, which made the model think SearXNG was broken.
+    """
+    values: Dict[str, Any] = dict(request.query_params)
+    content_type = (request.headers.get("content-type") or "").lower()
+    try:
+        if "application/json" in content_type:
+            body = await request.json()
+            if isinstance(body, dict):
+                values.update(body)
+        else:
+            form = await request.form()
+            values.update(dict(form))
+    except Exception:
+        pass
+    return values
+
+
+def setup_search_routes(config) -> APIRouter:
+    router = APIRouter(tags=["search"])
+
+    @router.get("/search/web", response_class=HTMLResponse)
+    async def web_search_page(q: str = Query("", min_length=0)) -> HTMLResponse:
+        """Browser-facing search results page for clickable agent web_search rows."""
+        safe_q = str(q or "").strip()
+        title = html.escape(safe_q or "Web search")
+        q_json = json.dumps(safe_q)
+        page = f"""<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>{title} - Odysseus Search</title>
+  <style>
+    :root {{ color-scheme: dark; --bg:#111; --fg:#eee; --muted:#999; --border:#333; --accent:#e05252; }}
+    body {{ margin:0; font:14px/1.45 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; background:var(--bg); color:var(--fg); }}
+    main {{ max-width:900px; margin:0 auto; padding:22px 18px 40px; }}
+    form {{ display:flex; gap:8px; margin:0 0 16px; }}
+    input {{ flex:1; min-width:0; height:34px; padding:0 10px; border:1px solid var(--border); border-radius:7px; background:#181818; color:var(--fg); font:inherit; }}
+    button {{ height:34px; padding:0 13px; border:1px solid color-mix(in srgb,var(--accent) 45%,var(--border)); border-radius:7px; background:color-mix(in srgb,var(--accent) 14%,transparent); color:var(--fg); font:inherit; cursor:pointer; }}
+    h1 {{ margin:0 0 14px; font-size:16px; font-weight:650; }}
+    .status {{ color:var(--muted); font-size:12px; margin:8px 0 14px; }}
+    .result {{ display:block; padding:11px 0; border-top:1px solid var(--border); text-decoration:none; color:inherit; }}
+    .result-title {{ color:var(--fg); font-weight:650; }}
+    .result-url {{ margin-top:3px; color:color-mix(in srgb,var(--accent) 78%,var(--fg)); font-size:12px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }}
+    .result-snippet {{ margin-top:5px; color:color-mix(in srgb,var(--fg) 72%,transparent); font-size:13px; }}
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Web Search</h1>
+    <form id="search-form">
+      <input id="query" value="{html.escape(safe_q, quote=True)}" autocomplete="off">
+      <button type="submit">Search</button>
+    </form>
+    <div class="status" id="status">Loading...</div>
+    <div id="results"></div>
+  </main>
+  <script>
+    const initialQuery = {q_json};
+    const input = document.getElementById('query');
+    const statusEl = document.getElementById('status');
+    const resultsEl = document.getElementById('results');
+    function esc(value) {{
+      return String(value || '').replace(/[&<>"']/g, ch => ({{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}}[ch]));
+    }}
+    async function runSearch(query) {{
+      query = String(query || '').trim();
+      if (!query) {{ statusEl.textContent = 'Enter a search query.'; resultsEl.innerHTML = ''; return; }}
+      statusEl.textContent = 'Searching...';
+      resultsEl.innerHTML = '';
+      const fd = new FormData();
+      fd.append('query', query);
+      const res = await fetch('/api/search', {{ method: 'POST', credentials: 'same-origin', body: fd }});
+      const data = await res.json().catch(() => ({{}}));
+      const sources = Array.isArray(data.sources) ? data.sources : [];
+      if (!res.ok || data.error) {{
+        statusEl.textContent = data.error || `Search failed (${{res.status}})`;
+        return;
+      }}
+      statusEl.textContent = sources.length ? `${{sources.length}} results` : 'No results';
+      resultsEl.innerHTML = sources.map(s => {{
+        const url = s.url || s.link || '';
+        const title = s.title || url || 'Untitled';
+        const snippet = s.snippet || s.content || '';
+        return `<a class="result" href="${{esc(url)}}" target="_blank" rel="noopener noreferrer">
+          <div class="result-title">${{esc(title)}}</div>
+          <div class="result-url">${{esc(url)}}</div>
+          <div class="result-snippet">${{esc(snippet)}}</div>
+        </a>`;
+      }}).join('');
+    }}
+    document.getElementById('search-form').addEventListener('submit', ev => {{
+      ev.preventDefault();
+      const q = input.value.trim();
+      const url = new URL(window.location.href);
+      url.searchParams.set('q', q);
+      history.replaceState(null, '', url);
+      runSearch(q);
+    }});
+    runSearch(initialQuery);
+  </script>
+</body>
+</html>"""
+        return HTMLResponse(page)
+
+    @router.get("/api/search/config")
+    async def get_search_settings() -> Dict[str, Any]:
+        return get_search_config()
+
+    @router.post("/api/search")
+    async def do_web_search(request: Request) -> Dict[str, Any]:
+        """Standalone web search — returns context string + source list.
+
+        Used by Compare mode to pre-search once and share results across panes.
+        """
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        if not query:
+            return {"context": "", "sources": [], "error": "query is required"}
+        time_filter = values.get("time_filter") or values.get("freshness")
+        if time_filter is not None:
+            time_filter = str(time_filter).strip() or None
+        try:
+            context, sources = comprehensive_web_search(
+                query, return_sources=True, time_filter=time_filter,
+            )
+            return {"context": context, "sources": sources}
+        except Exception as e:
+            logger.error(f"Standalone web search failed: {e}")
+            return {"context": "", "sources": [], "error": str(e)}
+
+    @router.get("/api/search/providers")
+    async def list_search_providers():
+        """Return available search providers with config status."""
+        providers = []
+        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
+            if pid == "disabled":
+                continue
+            available = True
+            if needs_key and not _get_provider_key(pid):
+                available = False
+            if needs_url and pid == "searxng" and not _get_search_instance():
+                available = False
+            providers.append({
+                "id": pid,
+                "label": label,
+                "available": available,
+            })
+        return providers
+
+    @router.post("/api/search/query")
+    async def search_with_provider(request: Request) -> Dict[str, Any]:
+        """Search using a specific provider. Used by compare search mode."""
+        values = await _request_values(request)
+        query = str(values.get("query") or values.get("q") or "").strip()
+        provider = str(values.get("provider") or "").strip()
+        try:
+            count = int(values.get("count") or values.get("limit") or 10)
+        except Exception:
+            count = 10
+        if not query:
+            return {"results": [], "provider": provider, "error": "query is required"}
+        if provider not in PROVIDER_INFO or provider == "disabled":
+            return {"results": [], "provider": provider, "error": "Unknown provider"}
+        t0 = time.time()
+        try:
+            results = _call_provider(provider, query, min(count, 20))
+            elapsed = round(time.time() - t0, 2)
+            return {"results": results, "provider": provider, "time": elapsed}
+        except Exception as e:
+            elapsed = round(time.time() - t0, 2)
+            logger.error(f"Search provider {provider} failed: {e}")
+            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
+
+    return router

routes/search_routes.py

@@ -1,111 +1,13 @@
-"""Search routes — /api/search/config GET, /api/search POST."""
+"""Backward-compat shim — canonical location is routes/search/search_routes.py.
 
-import logging
-from typing import Dict, Any
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.search_routes`` and ``from routes.search_routes import X``
+keep resolving to the canonical module. Keeps existing import paths working
+after slice 2j (#4082/#4071).
+"""
 
-from fastapi import APIRouter, Request
+import sys as _sys
 
-import time
+from routes.search import search_routes as _canonical  # noqa: F401
 
-from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO
-from services.search.core import _call_provider
-from services.search.providers import _get_provider_key, _get_search_instance
-
-logger = logging.getLogger(__name__)
-
-
-async def _request_values(request: Request) -> Dict[str, Any]:
-    """Accept JSON, form data, or query params for search endpoints.
-
-    The browser UI posts FormData, while the agent's generic app_api tool
-    posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler
-    runs, which made the model think SearXNG was broken.
-    """
-    values: Dict[str, Any] = dict(request.query_params)
-    content_type = (request.headers.get("content-type") or "").lower()
-    try:
-        if "application/json" in content_type:
-            body = await request.json()
-            if isinstance(body, dict):
-                values.update(body)
-        else:
-            form = await request.form()
-            values.update(dict(form))
-    except Exception:
-        pass
-    return values
-
-
-def setup_search_routes(config) -> APIRouter:
-    router = APIRouter(tags=["search"])
-
-    @router.get("/api/search/config")
-    async def get_search_settings() -> Dict[str, Any]:
-        return get_search_config()
-
-    @router.post("/api/search")
-    async def do_web_search(request: Request) -> Dict[str, Any]:
-        """Standalone web search — returns context string + source list.
-
-        Used by Compare mode to pre-search once and share results across panes.
-        """
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        if not query:
-            return {"context": "", "sources": [], "error": "query is required"}
-        time_filter = values.get("time_filter") or values.get("freshness")
-        if time_filter is not None:
-            time_filter = str(time_filter).strip() or None
-        try:
-            context, sources = comprehensive_web_search(
-                query, return_sources=True, time_filter=time_filter,
-            )
-            return {"context": context, "sources": sources}
-        except Exception as e:
-            logger.error(f"Standalone web search failed: {e}")
-            return {"context": "", "sources": [], "error": str(e)}
-
-    @router.get("/api/search/providers")
-    async def list_search_providers():
-        """Return available search providers with config status."""
-        providers = []
-        for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items():
-            if pid == "disabled":
-                continue
-            available = True
-            if needs_key and not _get_provider_key(pid):
-                available = False
-            if needs_url and pid == "searxng" and not _get_search_instance():
-                available = False
-            providers.append({
-                "id": pid,
-                "label": label,
-                "available": available,
-            })
-        return providers
-
-    @router.post("/api/search/query")
-    async def search_with_provider(request: Request) -> Dict[str, Any]:
-        """Search using a specific provider. Used by compare search mode."""
-        values = await _request_values(request)
-        query = str(values.get("query") or values.get("q") or "").strip()
-        provider = str(values.get("provider") or "").strip()
-        try:
-            count = int(values.get("count") or values.get("limit") or 10)
-        except Exception:
-            count = 10
-        if not query:
-            return {"results": [], "provider": provider, "error": "query is required"}
-        if provider not in PROVIDER_INFO or provider == "disabled":
-            return {"results": [], "provider": provider, "error": "Unknown provider"}
-        t0 = time.time()
-        try:
-            results = _call_provider(provider, query, min(count, 20))
-            elapsed = round(time.time() - t0, 2)
-            return {"results": results, "provider": provider, "time": elapsed}
-        except Exception as e:
-            elapsed = round(time.time() - t0, 2)
-            logger.error(f"Search provider {provider} failed: {e}")
-            return {"results": [], "provider": provider, "time": elapsed, "error": str(e)}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/signature_routes.py

@@ -21,10 +21,44 @@ from src.auth_helpers import get_current_user
 logger = logging.getLogger(__name__)
 
 
-_DATA_URL_RE = re.compile(
-    r'^data:image/(?P<fmt>png|jpeg|jpg);base64,(?P<data>.+)$',
-    re.IGNORECASE | re.DOTALL,
-)
+_DATA_URL_RE = re.compile(r"^data:image/png;base64,(?P<data>.+)$", re.IGNORECASE | re.DOTALL)
+_ANY_IMAGE_DATA_URL_RE = re.compile(r"^data:image/[^;]+;base64,", re.IGNORECASE)
+_PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
+_MAX_SIGNATURE_BYTES = 2 * 1024 * 1024
+_MAX_SIGNATURE_B64 = ((_MAX_SIGNATURE_BYTES + 2) // 3) * 4
+_MAX_SIGNATURE_DIMENSION = 4096
+
+
+def _normalize_signature_png(raw: str) -> str:
+    raw = (raw or "").strip()
+    m = _DATA_URL_RE.match(raw)
+    if m:
+        b64 = m.group("data")
+    elif _ANY_IMAGE_DATA_URL_RE.match(raw):
+        raise HTTPException(400, "Signature data must be a PNG image")
+    else:
+        b64 = raw
+    if len(b64) > _MAX_SIGNATURE_B64:
+        raise HTTPException(400, "Signature PNG is too large")
+    try:
+        payload = base64.b64decode(b64, validate=True)
+    except Exception:
+        raise HTTPException(400, "Signature data must be base64-encoded PNG bytes")
+    if not payload:
+        raise HTTPException(400, "Signature PNG is empty")
+    if len(payload) > _MAX_SIGNATURE_BYTES:
+        raise HTTPException(400, "Signature PNG is too large")
+    if not payload.startswith(_PNG_MAGIC):
+        raise HTTPException(400, "Signature data must be a PNG image")
+    return base64.b64encode(payload).decode("ascii")
+
+
+def _signature_dimension(value: Optional[int]) -> Optional[int]:
+    if value is None:
+        return None
+    if not isinstance(value, int) or value < 1 or value > _MAX_SIGNATURE_DIMENSION:
+        raise HTTPException(400, "Signature dimensions are invalid")
+    return value
 
 
 class SignatureCreate(BaseModel):
@@ -67,24 +101,18 @@ def setup_signature_routes() -> APIRouter:
     @router.post("/api/signatures")
     async def create_signature(request: Request, req: SignatureCreate) -> Dict[str, Any]:
         user = get_current_user(request)
-        raw = (req.data or "").strip()
-        m = _DATA_URL_RE.match(raw)
-        b64 = m.group("data") if m else raw
-        try:
-            payload = base64.b64decode(b64, validate=True)
-            if not payload:
-                raise ValueError("empty payload")
-        except Exception:
-            raise HTTPException(400, "Signature data must be base64-encoded PNG bytes")
+        b64 = _normalize_signature_png(req.data)
+        width = _signature_dimension(req.width)
+        height = _signature_dimension(req.height)
 
         sig = Signature(
             id=str(uuid.uuid4()),
             owner=user,
             name=(req.name or "Signature").strip() or "Signature",
             data_png=b64,
-            width=req.width,
-            height=req.height,
-            svg=req.svg,
+            width=width,
+            height=height,
+            svg=None,
         )
         db = SessionLocal()
         try:

routes/stt_routes.py

@@ -4,6 +4,8 @@
 from fastapi import APIRouter, HTTPException, UploadFile, File
 import logging
 
+from src.upload_limits import read_upload_limited, STT_MAX_AUDIO_BYTES
+
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +32,7 @@ def setup_stt_routes(stt_service):
                     detail={"message": "STT service not available or set to browser mode"}
                 )
 
-            audio_bytes = await file.read()
+            audio_bytes = await read_upload_limited(file, STT_MAX_AUDIO_BYTES, "Audio file")
             if not audio_bytes:
                 raise HTTPException(status_code=400, detail={"message": "Empty audio file"})
 

routes/task/__init__.py

@@ -0,0 +1,5 @@
+"""Task route domain package (slice 2p, #4082/#4071).
+
+Contains task_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/task_routes.py re-exports from here.
+"""

routes/task_routes.py

@@ -1,910 +1,18 @@
-"""CRUD routes for scheduled tasks."""
+"""Backward-compat shim — canonical location is routes/task/task_routes.py.
 
-import json
-import logging
-import secrets
-import uuid
-from datetime import datetime
-from typing import Optional, Dict, Any
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.task_routes``, ``from routes.task_routes import X``,
+``importlib.import_module("routes.task_routes")``, the
+``import ... as task_routes`` + ``monkeypatch.setattr(task_routes,
+"SessionLocal", ...)`` / ``"get_current_user"`` pattern used by multiple
+tests, and the ``task_routes.__file__`` reads in test_auth_regressions.py
+all operate on the *same* object the application actually uses. Keeps
+existing import paths working after slice 2p (#4082/#4071).
+Source-introspection tests read the canonical file by path.
+"""
 
-from fastapi import APIRouter, HTTPException, Request
-from pydantic import BaseModel
+import sys as _sys
 
-from core.database import SessionLocal, ScheduledTask, TaskRun
-from src.auth_helpers import get_current_user
-from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS
-from routes.prefs_routes import _load_for_user, _save_for_user
+from routes.task import task_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-
-class TaskCreate(BaseModel):
-    name: Optional[str] = None
-    prompt: Optional[str] = None
-    task_type: str = "llm"                        # "llm" | "action" | "research"
-    action: Optional[str] = None                  # builtin action name
-    schedule: Optional[str] = None                # "once" | "daily" | "weekly" | "monthly" | "cron"
-    scheduled_time: str = "09:00"                 # HH:MM
-    scheduled_day: Optional[int] = None           # day-of-week (0=Mon) or day-of-month
-    scheduled_date: Optional[str] = None          # ISO datetime for "once"
-    cron_expression: Optional[str] = None         # cron string e.g. "*/5 * * * *"
-    trigger_type: str = "schedule"                # "schedule" | "event" | "webhook"
-    trigger_event: Optional[str] = None           # e.g. "session_created"
-    trigger_count: Optional[int] = None           # fire every N events
-    output_target: str = "session"
-    model: Optional[str] = None
-    endpoint_url: Optional[str] = None
-    then_task_id: Optional[str] = None            # chain: run this task after success
-    notifications_enabled: Optional[bool] = None  # None lets action-specific defaults apply
-
-
-class TaskUpdate(BaseModel):
-    name: Optional[str] = None
-    prompt: Optional[str] = None
-    task_type: Optional[str] = None
-    action: Optional[str] = None
-    schedule: Optional[str] = None
-    scheduled_time: Optional[str] = None
-    scheduled_day: Optional[int] = None
-    scheduled_date: Optional[str] = None
-    cron_expression: Optional[str] = None
-    trigger_type: Optional[str] = None
-    trigger_event: Optional[str] = None
-    trigger_count: Optional[int] = None
-    output_target: Optional[str] = None
-    model: Optional[str] = None
-    endpoint_url: Optional[str] = None
-    then_task_id: Optional[str] = None
-    notifications_enabled: Optional[bool] = None
-
-
-def _display_task_name(t: ScheduledTask) -> str:
-    defs = HOUSEKEEPING_DEFAULTS.get(t.action) if t.action else None
-    if defs and (t.name or "") in set(defs.get("legacy_names") or []):
-        return defs["name"]
-    return t.name
-
-
-def _task_to_dict(t: ScheduledTask, include_last_run_result: bool = False) -> dict:
-    defs = HOUSEKEEPING_DEFAULTS.get(t.action) if t.action else None
-    d = {
-        "id": t.id,
-        "name": _display_task_name(t),
-        "prompt": t.prompt,
-        "task_type": t.task_type or "llm",
-        "action": t.action,
-        "schedule": t.schedule,
-        "scheduled_time": t.scheduled_time,
-        "scheduled_day": t.scheduled_day,
-        "scheduled_date": t.scheduled_date.isoformat() + "Z" if t.scheduled_date else None,
-        "cron_expression": t.cron_expression,
-        "trigger_type": t.trigger_type or "schedule",
-        "trigger_event": t.trigger_event,
-        "trigger_count": t.trigger_count,
-        "trigger_counter": t.trigger_counter or 0,
-        "next_run": t.next_run.isoformat() + "Z" if t.next_run else None,
-        "last_run": t.last_run.isoformat() + "Z" if t.last_run else None,
-        "status": t.status,
-        "output_target": t.output_target,
-        "session_id": t.session_id,
-        "crew_member_id": getattr(t, "crew_member_id", None),
-        "model": t.model,
-        "endpoint_url": t.endpoint_url,
-        "run_count": t.run_count or 0,
-        "then_task_id": t.then_task_id,
-        "notifications_enabled": bool(getattr(t, "notifications_enabled", True)),
-        "webhook_token": t.webhook_token if (t.trigger_type or "schedule") == "webhook" else None,
-        "created_at": t.created_at.isoformat() + "Z" if t.created_at else None,
-        "updated_at": t.updated_at.isoformat() + "Z" if t.updated_at else None,
-    }
-    # Built-in housekeeping tasks (identified by their action) are flagged so
-    # the UI can mark them and offer "revert to default" once altered.
-    d["is_builtin"] = defs is not None
-    if defs:
-        default_names = {defs["name"], *set(defs.get("legacy_names") or [])}
-        d["is_modified"] = (
-            (t.name or "") not in default_names
-            or (t.schedule or "") != (defs["schedule"] or "")
-            or (t.scheduled_time or "") != (defs["scheduled_time"] or "")
-            or (t.cron_expression or "") != (defs["cron_expression"] or "")
-        )
-    else:
-        d["is_modified"] = False
-    if include_last_run_result and t.runs:
-        last = t.runs[0]  # ordered desc by started_at
-        d["last_run_status"] = last.status
-        d["last_run_result"] = (last.result or last.error or "")[:500]
-    return d
-
-
-def _run_to_dict(r: TaskRun) -> dict:
-    return {
-        "id": r.id,
-        "task_id": r.task_id,
-        "started_at": r.started_at.isoformat() + "Z" if r.started_at else None,
-        "finished_at": r.finished_at.isoformat() + "Z" if r.finished_at else None,
-        "status": r.status,
-        "result": r.result,
-        "error": r.error,
-        "tokens_used": r.tokens_used,
-        "model": r.model,
-    }
-
-
-def _run_research_id(task: ScheduledTask) -> str:
-    if (task.task_type or "llm") == "research" and task.session_id:
-        return task.session_id
-    return ""
-
-
-def _resolve_run_endpoint(db, task: ScheduledTask, run: TaskRun) -> str:
-    """Best-effort endpoint URL for reopening a task run in chat."""
-    if getattr(task, "endpoint_url", None):
-        return task.endpoint_url or ""
-
-    try:
-        if getattr(task, "session_id", None):
-            from core.database import Session as DbSession
-            sess = db.query(DbSession).filter(DbSession.id == task.session_id).first()
-            if sess and sess.endpoint_url:
-                return sess.endpoint_url or ""
-    except Exception:
-        pass
-
-    model = (getattr(run, "model", None) or getattr(task, "model", None) or "").strip()
-    if not model:
-        return ""
-
-    try:
-        from core.database import ModelEndpoint
-        eps = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
-        for ep in eps:
-            cached = []
-            if ep.cached_models:
-                try:
-                    cached = json.loads(ep.cached_models) or []
-                except Exception:
-                    cached = []
-            if model in cached:
-                return ep.base_url or ""
-    except Exception:
-        pass
-    return ""
-
-
-def setup_task_routes(task_scheduler) -> APIRouter:
-    router = APIRouter(prefix="/api/tasks", tags=["tasks"])
-
-    def _owner(request: Request):
-        return get_current_user(request)
-
-    async def _generate_task_name(prompt: str) -> str:
-        """Use LLM to generate a short task name from the prompt."""
-        try:
-            from src.llm_core import llm_call_async
-            from core.database import Session as DbSession
-            db = SessionLocal()
-            try:
-                recent = db.query(DbSession).filter(
-                    DbSession.endpoint_url.isnot(None),
-                    DbSession.model.isnot(None),
-                ).order_by(DbSession.created_at.desc()).first()
-                if not recent:
-                    return prompt[:50].strip()
-                url, model = recent.endpoint_url, recent.model
-            finally:
-                db.close()
-
-            result = await llm_call_async(
-                url=url, model=model,
-                messages=[
-                    {"role": "system", "content": "Generate a short title (3-5 words, no quotes) for this scheduled task. Reply with ONLY the title, nothing else."},
-                    {"role": "user", "content": prompt[:500]},
-                ],
-                max_tokens=20,
-                timeout=15,
-            )
-            title = result.strip().strip('"\'').strip()
-            return title[:60] if title else prompt[:50].strip()
-        except Exception:
-            first = prompt.split('\n')[0].split('.')[0].strip()
-            return first[:50] if first else "Untitled Task"
-
-    @router.get("")
-    async def list_tasks(request: Request, status: Optional[str] = None,
-                         include_last_run: bool = False):
-        user = _owner(request)
-        if user:
-            await task_scheduler.ensure_defaults(user)
-        else:
-            db_seed = SessionLocal()
-            try:
-                owners = {
-                    row[0] for row in db_seed.query(ScheduledTask.owner)
-                    .filter(ScheduledTask.task_type == "action")
-                    .filter(ScheduledTask.action.in_(list(HOUSEKEEPING_DEFAULTS.keys())))
-                    .all()
-                    if row[0]
-                }
-            finally:
-                db_seed.close()
-            for owner in owners:
-                await task_scheduler.ensure_defaults(owner)
-        db = SessionLocal()
-        try:
-            q = db.query(ScheduledTask)
-            if user:
-                q = q.filter(ScheduledTask.owner == user)
-            if status:
-                q = q.filter(ScheduledTask.status == status)
-            tasks = q.order_by(ScheduledTask.created_at.desc()).all()
-            return {"tasks": [_task_to_dict(t, include_last_run_result=include_last_run) for t in tasks]}
-        finally:
-            db.close()
-
-    @router.get("/onboarding")
-    async def get_tasks_onboarding(request: Request):
-        user = _owner(request)
-        prefs = _load_for_user(user) or {}
-        return {
-            "opened": bool(prefs.get("tasks_opened")),
-            "enabled": bool(prefs.get("tasks_enabled")),
-        }
-
-    @router.post("/onboarding")
-    async def update_tasks_onboarding(request: Request, body: dict):
-        user = _owner(request)
-        prefs = _load_for_user(user) or {}
-        prefs["tasks_opened"] = True
-        enable = bool(body.get("enabled"))
-        if enable:
-            prefs["tasks_enabled"] = True
-        _save_for_user(user, prefs)
-        if user:
-            await task_scheduler.ensure_defaults(user)
-
-        resumed = 0
-        if enable:
-            db = SessionLocal()
-            try:
-                tasks = db.query(ScheduledTask).filter(
-                    ScheduledTask.owner == user,
-                    ScheduledTask.task_type == "action",
-                    ScheduledTask.action.in_(list(HOUSEKEEPING_DEFAULTS.keys())),
-                ).all()
-                for task in tasks:
-                    defs = HOUSEKEEPING_DEFAULTS.get(task.action or "")
-                    if defs and defs.get("ship_paused"):
-                        continue
-                    if task.status == "active":
-                        continue
-                    task.status = "active"
-                    if (task.trigger_type or "schedule") == "schedule":
-                        task.next_run = compute_next_run(
-                            task.schedule,
-                            task.scheduled_time,
-                            task.scheduled_day,
-                            task.scheduled_date,
-                            cron_expression=task.cron_expression,
-                        )
-                    resumed += 1
-                db.commit()
-            finally:
-                db.close()
-        return {"ok": True, "opened": True, "enabled": bool(prefs.get("tasks_enabled")), "resumed": resumed}
-
-    # Actions that execute shell/SSH commands — restricted to admins.
-    # Non-admin users cannot create tasks with these action types via the
-    # API. See review CRIT-C.
-    _ADMIN_ONLY_ACTIONS = {"run_local", "run_script", "ssh_command"}
-
-    def _is_admin(user: str | None) -> bool:
-        if not user:
-            return False
-        # In-process tool-loopback marker — AuthMiddleware validated
-        # the internal token + loopback client before stamping this,
-        # so treat as admin-equivalent.
-        if user == "internal-tool":
-            return True
-        try:
-            from core.auth import AuthManager
-            auth = AuthManager()
-            if not auth.is_configured:
-                # Unconfigured single-user deploy: trust the local owner.
-                return True
-            return bool(auth.is_admin(user))
-        except Exception:
-            return False
-
-    @router.post("")
-    async def create_task(request: Request, req: TaskCreate):
-        user = _owner(request)
-
-        # Validate
-        if req.task_type in ("llm", "research") and not req.prompt:
-            raise HTTPException(400, "Prompt is required for LLM/research tasks")
-        if req.task_type == "action" and not req.action:
-            raise HTTPException(400, "Action name is required for action tasks")
-        # Block shell-executing action types for non-admins. action_run_local
-        # uses subprocess.run(shell=True) and ssh_command / run_script run
-        # arbitrary commands.
-        if req.task_type == "action" and req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
-            raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
-        if req.trigger_type == "schedule" and not req.schedule:
-            raise HTTPException(400, "Schedule is required for schedule-triggered tasks")
-        if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression:
-            raise HTTPException(400, "Cron expression is required for cron schedule")
-        if req.trigger_type == "schedule" and req.schedule == "cron" and req.cron_expression:
-            try:
-                from croniter import croniter
-                croniter(req.cron_expression)
-            except Exception:
-                raise HTTPException(400, "Invalid cron expression")
-        if req.trigger_type == "event" and not req.trigger_event:
-            raise HTTPException(400, "Event name is required for event-triggered tasks")
-        if req.trigger_type == "event" and not req.trigger_count:
-            raise HTTPException(400, "Trigger count is required for event-triggered tasks")
-
-        # Auto-generate name
-        name = req.name
-        if not name:
-            if req.task_type == "action":
-                from src.builtin_actions import BUILTIN_ACTION_INFO
-                name = BUILTIN_ACTION_INFO.get(req.action, req.action or "Action Task")
-            elif req.prompt:
-                name = await _generate_task_name(req.prompt)
-            else:
-                name = "Untitled Task"
-
-        # Compute next_run for schedule-triggered tasks
-        next_run = None
-        sched_date = None
-        if req.trigger_type == "schedule":
-            if req.schedule == "once" and req.scheduled_date:
-                try:
-                    sched_date = datetime.fromisoformat(req.scheduled_date.replace("Z", "+00:00")).replace(tzinfo=None)
-                except ValueError:
-                    raise HTTPException(400, "Invalid scheduled_date format")
-            next_run = compute_next_run(
-                req.schedule, req.scheduled_time,
-                req.scheduled_day, sched_date,
-                cron_expression=req.cron_expression,
-            )
-
-        # Generate webhook token if needed
-        webhook_token = None
-        if req.trigger_type == "webhook":
-            webhook_token = secrets.token_urlsafe(32)
-
-        task_id = str(uuid.uuid4())
-        db = SessionLocal()
-        try:
-            notifications_enabled = (
-                False if req.task_type == "action" and req.notifications_enabled is None
-                else bool(req.notifications_enabled) if req.notifications_enabled is not None
-                else True
-            )
-            task = ScheduledTask(
-                id=task_id,
-                owner=user,
-                name=name,
-                prompt=req.prompt,
-                task_type=req.task_type,
-                action=req.action,
-                schedule=req.schedule,
-                scheduled_time=req.scheduled_time,
-                scheduled_day=req.scheduled_day,
-                scheduled_date=sched_date,
-                cron_expression=req.cron_expression,
-                trigger_type=req.trigger_type,
-                trigger_event=req.trigger_event,
-                trigger_count=req.trigger_count,
-                trigger_counter=0,
-                next_run=next_run,
-                status="active" if (req.trigger_type in ("event", "webhook") or next_run) else "completed",
-                output_target=req.output_target,
-                model=req.model or None,
-                endpoint_url=req.endpoint_url or None,
-                then_task_id=req.then_task_id or None,
-                webhook_token=webhook_token,
-                notifications_enabled=notifications_enabled,
-            )
-            db.add(task)
-            db.commit()
-            db.refresh(task)
-            return _task_to_dict(task)
-        finally:
-            db.close()
-
-    @router.get("/notifications")
-    async def get_notifications(request: Request):
-        """Return and clear pending task-run notifications for the
-        current user. Anonymous callers get nothing (prevents
-        cross-tenant drain — see review CRIT-B)."""
-        user = _owner(request)
-        if not user:
-            return {"notifications": []}
-        notes = task_scheduler.pop_notifications(owner=user)
-        return {"notifications": notes}
-
-    @router.get("/{task_id}")
-    async def get_task(request: Request, task_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            return _task_to_dict(task)
-        finally:
-            db.close()
-
-    @router.put("/{task_id}")
-    async def update_task(request: Request, task_id: str, req: TaskUpdate):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-
-            if req.name is not None:
-                task.name = req.name
-            if req.prompt is not None:
-                task.prompt = req.prompt
-            if req.task_type is not None:
-                task.task_type = req.task_type
-            if req.action is not None:
-                # Same admin-only gate as create — see CRIT-C.
-                if req.action in _ADMIN_ONLY_ACTIONS and not _is_admin(user):
-                    raise HTTPException(403, f"Action '{req.action}' requires admin privileges")
-                task.action = req.action
-            if req.output_target is not None:
-                task.output_target = req.output_target
-            if req.model is not None:
-                task.model = req.model or None
-            if req.endpoint_url is not None:
-                task.endpoint_url = req.endpoint_url or None
-            if req.trigger_type is not None:
-                # Generate webhook token when switching to webhook trigger
-                if req.trigger_type == "webhook" and not task.webhook_token:
-                    task.webhook_token = secrets.token_urlsafe(32)
-                task.trigger_type = req.trigger_type
-            if req.trigger_event is not None:
-                task.trigger_event = req.trigger_event
-            if req.trigger_count is not None:
-                task.trigger_count = req.trigger_count
-            if req.then_task_id is not None:
-                task.then_task_id = req.then_task_id or None
-            if req.notifications_enabled is not None:
-                task.notifications_enabled = bool(req.notifications_enabled)
-            if req.cron_expression is not None:
-                if req.cron_expression:
-                    try:
-                        from croniter import croniter
-                        croniter(req.cron_expression)
-                    except Exception:
-                        raise HTTPException(400, "Invalid cron expression")
-                task.cron_expression = req.cron_expression or None
-
-            # Recompute next_run if schedule changed
-            schedule_changed = False
-            if req.schedule is not None:
-                task.schedule = req.schedule
-                schedule_changed = True
-            if req.scheduled_time is not None:
-                task.scheduled_time = req.scheduled_time
-                schedule_changed = True
-            if req.scheduled_day is not None:
-                task.scheduled_day = req.scheduled_day
-                schedule_changed = True
-            if req.scheduled_date is not None:
-                try:
-                    task.scheduled_date = datetime.fromisoformat(
-                        req.scheduled_date.replace("Z", "+00:00")
-                    ).replace(tzinfo=None)
-                except ValueError:
-                    raise HTTPException(400, "Invalid scheduled_date format")
-                schedule_changed = True
-
-            if req.cron_expression is not None:
-                schedule_changed = True
-
-            if schedule_changed and task.status == "active" and (task.trigger_type or "schedule") == "schedule":
-                task.next_run = compute_next_run(
-                    task.schedule, task.scheduled_time,
-                    task.scheduled_day, task.scheduled_date,
-                    cron_expression=task.cron_expression,
-                )
-
-            db.commit()
-            db.refresh(task)
-            return _task_to_dict(task)
-        finally:
-            db.close()
-
-    @router.delete("/{task_id}")
-    async def delete_task(request: Request, task_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            db.delete(task)
-            db.commit()
-            return {"ok": True}
-        finally:
-            db.close()
-
-    @router.post("/{task_id}/pause")
-    async def pause_task(request: Request, task_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            task.status = "paused"
-            db.commit()
-            return {"ok": True, "status": "paused"}
-        finally:
-            db.close()
-
-    @router.post("/{task_id}/resume")
-    async def resume_task(request: Request, task_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            task.status = "active"
-            if (task.trigger_type or "schedule") == "schedule":
-                task.next_run = compute_next_run(
-                    task.schedule, task.scheduled_time,
-                    task.scheduled_day, task.scheduled_date,
-                    cron_expression=task.cron_expression,
-                )
-            db.commit()
-            return {"ok": True, "status": "active", "next_run": task.next_run.isoformat() + "Z" if task.next_run else None}
-        finally:
-            db.close()
-
-    @router.post("/{task_id}/revert")
-    async def revert_task(request: Request, task_id: str):
-        """Reset a built-in (housekeeping) task to its default config."""
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            defs = HOUSEKEEPING_DEFAULTS.get(task.action) if task.action else None
-            if not defs:
-                raise HTTPException(400, "Not a built-in task")
-            task.name = defs["name"]
-            task.schedule = defs["schedule"]
-            task.scheduled_time = defs["scheduled_time"]
-            task.scheduled_day = None
-            task.scheduled_date = None
-            task.cron_expression = defs["cron_expression"]
-            task.trigger_type = defs.get("trigger_type", "schedule")
-            task.trigger_event = defs.get("trigger_event")
-            task.trigger_count = defs.get("trigger_count")
-            task.trigger_counter = 0
-            task.prompt = None
-            task.model = None
-            task.endpoint_url = None
-            task.status = "paused" if defs.get("ship_paused") else "active"
-            task.next_run = None
-            if task.trigger_type == "schedule":
-                task.next_run = compute_next_run(
-                    defs["schedule"], defs["scheduled_time"], None, None,
-                    cron_expression=defs["cron_expression"],
-                )
-            db.commit()
-            db.refresh(task)
-            return {"ok": True, "task": _task_to_dict(task)}
-        finally:
-            db.close()
-
-    @router.post("/{task_id}/run")
-    async def run_task_now(request: Request, task_id: str, force: bool = False):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-        finally:
-            db.close()
-        started = await task_scheduler.run_task_now(task_id, force=force)
-        if not started:
-            raise HTTPException(409, "Task is already running")
-        return {"ok": True, "message": "Task triggered" + (" in parallel" if force else "")}
-
-    @router.get("/runs/recent")
-    async def list_recent_runs(request: Request, limit: int = 50):
-        """Recent task runs across ALL tasks for this owner. Drives the Activity view."""
-        user = _owner(request)
-        limit = max(1, min(limit, 200))
-        db = SessionLocal()
-        try:
-            q = db.query(TaskRun, ScheduledTask).join(
-                ScheduledTask, TaskRun.task_id == ScheduledTask.id
-            )
-            if user:
-                # Strict owner scope — was previously OR'ing in `owner IS NULL`
-                # rows for "legacy single-user" back-compat, but that leaks any
-                # legacy/migrated task's full result text to every authenticated
-                # user. _migrate_assign_legacy_owner runs on startup to claim
-                # legacy rows for the admin, so the OR-NULL path is no longer
-                # needed for any sane deploy.
-                q = q.filter(ScheduledTask.owner == user)
-            # Pull a little extra before de-duping. When auth is bypassed on a
-            # local browser session, legacy/default tasks from multiple owners
-            # can be visible together; the built-in urgent-email scanner then
-            # produces several identical "no email accounts configured" rows in
-            # the same minute. Keep the task records intact, but collapse those
-            # duplicate Activity rows for display.
-            rows = q.order_by(TaskRun.started_at.desc()).limit(limit * 3).all()
-            deduped = []
-            seen_urgency_rows = set()
-            for r, t in rows:
-                if (t.action or "") == "check_email_urgency":
-                    ts = r.started_at.replace(second=0, microsecond=0) if r.started_at else None
-                    text = (r.result or r.error or "").strip()
-                    key = (ts, r.status or "", text)
-                    if key in seen_urgency_rows:
-                        continue
-                    seen_urgency_rows.add(key)
-                deduped.append((r, t))
-                if len(deduped) >= limit:
-                    break
-            return {
-                "runs": [
-                    {
-                        **_run_to_dict(r),
-                        "task_name": _display_task_name(t),
-                        "task_type": t.task_type or "llm",
-                        "action": t.action,
-                        # Model + endpoint the task ran on, so the Activity
-                        # view's "Open in chat" can reuse the same model.
-                        "model": r.model or t.model or "",
-                        "endpoint_url": _resolve_run_endpoint(db, t, r),
-                        "session_id": t.session_id or "",
-                        "research_id": _run_research_id(t),
-                        # Where the task delivered its result — the Activity tab
-                        # uses this to filter notification rows in/out.
-                        "output_target": t.output_target or "session",
-                    }
-                    for r, t in deduped
-                ]
-            }
-        finally:
-            db.close()
-
-    @router.get("/{task_id}/runs")
-    async def list_runs(request: Request, task_id: str, limit: int = 20, offset: int = 0):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            runs = db.query(TaskRun).filter(TaskRun.task_id == task_id)\
-                .order_by(TaskRun.started_at.desc())\
-                .offset(offset).limit(limit).all()
-            total = db.query(TaskRun).filter(TaskRun.task_id == task_id).count()
-            return {"runs": [_run_to_dict(r) for r in runs], "total": total}
-        finally:
-            db.close()
-
-    @router.get("/meta/output-targets")
-    async def list_output_targets(request: Request):
-        """List available output targets — only delivery/send tools, not all MCP tools."""
-        _owner(request)
-        targets = [
-            {"value": "session", "label": "Session", "description": "Save result to a chat session"},
-            {"value": "notification", "label": "Notification", "description": "Push a browser notification with the result (also saved to the session for history)"},
-            {"value": "email", "label": "Email me", "description": "Send result through your configured SMTP account"},
-        ]
-        # Only include tools whose NAME clearly indicates an outbound delivery
-        # action — match by verb in the tool name, not by any mention of "email"
-        # in the description (which falsely picked up search_email, list_email,
-        # etc.). Also exclude read/search/list tools whose names happen to start
-        # with a delivery verb.
-        _DELIVERY_VERBS = ("send", "notify", "post", "publish", "draft", "dispatch", "deliver")
-        _NON_DELIVERY = (
-            "search", "list", "get", "find", "read", "fetch", "view",
-            "tag", "label", "move", "archive", "delete", "mark", "schedule",
-        )
-        try:
-            from src.agent_tools import get_mcp_manager
-            mcp = get_mcp_manager()
-            if mcp:
-                for tool in mcp.get_all_tools():
-                    name_lower = tool.get("name", "").lower()
-                    if any(x in name_lower for x in _NON_DELIVERY):
-                        continue
-                    if not any(v in name_lower for v in _DELIVERY_VERBS):
-                        continue
-                    targets.append({
-                        "value": tool["qualified_name"],
-                        "label": f"{tool['server_name']} → {tool['name']}",
-                        "description": tool.get("description", ""),
-                    })
-        except Exception:
-            pass
-        return {"targets": targets}
-
-    @router.get("/meta/actions")
-    async def list_actions(request: Request):
-        """List available built-in actions."""
-        user = _owner(request)
-        from src.builtin_actions import BUILTIN_ACTION_INFO
-        return {"actions": [
-            {"name": name, "description": desc}
-            for name, desc in BUILTIN_ACTION_INFO.items()
-            if name not in _ADMIN_ONLY_ACTIONS or _is_admin(user)
-        ]}
-
-    @router.get("/meta/events")
-    async def list_events(request: Request):
-        """List available event triggers."""
-        _owner(request)
-        return {"events": [
-            {"name": "session_created", "description": "Fires when a new chat session is created"},
-            {"name": "message_sent", "description": "Fires when a user sends a message"},
-            {"name": "document_created", "description": "Fires when a document is created"},
-            {"name": "memory_added", "description": "Fires when a memory is added"},
-            {"name": "research_completed", "description": "Fires when a research report completes"},
-            {"name": "email_received", "description": "Fires when new inbox mail is observed"},
-            {"name": "skill_added", "description": "Fires when a new skill is created"},
-        ]}
-
-    @router.post("/{task_id}/webhook/{token}")
-    async def webhook_trigger(task_id: str, token: str):
-        """Unauthenticated endpoint — the token IS the auth."""
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(
-                ScheduledTask.id == task_id,
-                ScheduledTask.webhook_token == token,
-                ScheduledTask.status == "active",
-            ).first()
-            if not task:
-                raise HTTPException(404, "Not found")
-        finally:
-            db.close()
-        started = await task_scheduler.run_task_now(task_id)
-        if not started:
-            raise HTTPException(409, "Task is already running")
-        return {"ok": True, "message": "Task triggered via webhook"}
-
-    @router.post("/{task_id}/webhook-regenerate")
-    async def regenerate_webhook(request: Request, task_id: str):
-        user = _owner(request)
-        db = SessionLocal()
-        try:
-            task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
-            if not task:
-                raise HTTPException(404, "Task not found")
-            if user and task.owner != user:
-                raise HTTPException(403, "Access denied")
-            task.webhook_token = secrets.token_urlsafe(32)
-            db.commit()
-            return {"ok": True, "webhook_token": task.webhook_token}
-        finally:
-            db.close()
-
-    # --- PARSE NATURAL LANGUAGE → TASK DRAFT (AI) ---
-    @router.post("/parse")
-    async def parse_task(request: Request) -> Dict[str, Any]:
-        """Turn a free-form description ("every weekday at 7am research the top
-        AI news and summarize it") into a structured task draft the frontend
-        can pre-fill the form with. Returns a draft only — the user reviews and
-        saves it, so a misread schedule never goes live unreviewed."""
-        from src.endpoint_resolver import resolve_endpoint
-        from src.llm_core import llm_call_async
-        from src.text_helpers import strip_think as _strip_think
-        import json as _json, re as _re
-        from datetime import datetime as _dt
-
-        body = await request.json()
-        desc = (body.get("description") or "").strip()
-        if not desc:
-            return {"success": False, "message": "Nothing to parse"}
-
-        now = _dt.now()
-        # Give the model the current date/time + weekday so relative phrasing
-        # ("tomorrow", "every Monday", "in an hour") resolves correctly.
-        ctx = now.strftime("%Y-%m-%d %H:%M (%A)")
-        sys = (
-            "You convert a user's description of a recurring or one-off task into "
-            "STRICT JSON for a task scheduler. The current local date/time is "
-            f"{ctx}. Output ONLY a JSON object, no prose, no markdown fences.\n\n"
-            "Schema (omit fields you can't infer):\n"
-            "{\n"
-            '  "task_type": "llm" | "research",  // "research" if it asks to research/investigate/find out; else "llm"\n'
-            '  "name": "short 3-6 word title",\n'
-            '  "prompt": "the instruction the AI should run on schedule (or the research question)",\n'
-            '  "schedule": "daily" | "weekly" | "monthly" | "once" | "cron",\n'
-            '  "scheduled_time": "HH:MM",        // 24h LOCAL time\n'
-            '  "scheduled_day": 0,               // weekly: 0=Mon..6=Sun; monthly: 1..31\n'
-            '  "scheduled_date": "YYYY-MM-DDTHH:MM",  // only for "once"\n'
-            '  "cron_expression": "m h dom mon dow",  // only if schedule is "cron"\n'
-            '  "output_target": "session" | "email" | "notification"  // use email when the user asks to email the result\n'
-            "}\n\n"
-            "Rules: default schedule to 'daily' if a time is given without a frequency. "
-            "Default scheduled_time to '09:00' if none is stated. For 'every weekday' "
-            "use cron '0 H * * 1-5'. Keep the prompt actionable and self-contained."
-        )
-        try:
-            url, model, headers = resolve_endpoint("utility")
-            if not url:
-                url, model, headers = resolve_endpoint("default")
-            if not (url and model):
-                return {"success": False, "message": "No model endpoint configured"}
-            raw = await llm_call_async(
-                url=url, model=model,
-                messages=[{"role": "system", "content": sys},
-                          {"role": "user", "content": desc[:1000]}],
-                temperature=0.2, max_tokens=400, headers=headers, timeout=45,
-            )
-            text = _strip_think(raw or "", prose=False, prompt_echo=False).strip()
-            if text.startswith("```"):
-                text = text.strip("`")
-                if text.lower().startswith("json"):
-                    text = text[4:].lstrip()
-            # Pull the first {...} block in case the model added stray text.
-            m = _re.search(r"\{.*\}", text, _re.S)
-            draft = _json.loads(m.group(0) if m else text)
-            if not isinstance(draft, dict):
-                raise ValueError("not an object")
-            # Whitelist + light validation so the frontend gets clean fields.
-            out: Dict[str, Any] = {}
-            if draft.get("task_type") in ("llm", "research"):
-                out["task_type"] = draft["task_type"]
-            else:
-                out["task_type"] = "llm"
-            for k in ("name", "prompt", "cron_expression", "scheduled_date"):
-                if isinstance(draft.get(k), str) and draft[k].strip():
-                    out[k] = draft[k].strip()
-            if draft.get("schedule") in ("daily", "weekly", "monthly", "once", "cron"):
-                out["schedule"] = draft["schedule"]
-            else:
-                out["schedule"] = "daily"
-            st = draft.get("scheduled_time")
-            if isinstance(st, str) and _re.match(r"^\d{1,2}:\d{2}$", st.strip()):
-                out["scheduled_time"] = st.strip()
-            if isinstance(draft.get("scheduled_day"), int):
-                out["scheduled_day"] = draft["scheduled_day"]
-            if draft.get("output_target") in ("session", "email", "notification"):
-                out["output_target"] = draft["output_target"]
-            out["trigger_type"] = "schedule"
-            if not out.get("prompt"):
-                return {"success": False, "message": "Could not extract a task instruction"}
-            return {"success": True, "draft": out}
-        except Exception as e:
-            logger.error(f"parse_task failed: {e}")
-            return {"success": False, "message": str(e)}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/upload_routes.py

@@ -3,36 +3,300 @@ import os
 import time
 import json
 import asyncio
-from fastapi import APIRouter, Request, File, UploadFile, HTTPException
-from typing import List
+import shutil
+import uuid
+from pathlib import Path
+from fastapi import APIRouter, Request, File, UploadFile, HTTPException, Form
+from typing import List, Optional
 import logging
 from core.middleware import require_admin
-from src.auth_helpers import get_current_user
+from core.database import (
+    SessionLocal,
+    ChatMessage as DbChatMessage,
+    CalendarCal,
+    CalendarEvent,
+    Document,
+    DocumentVersion,
+    GalleryImage,
+    Note,
+    Session as DbSession,
+)
+from src.auth_helpers import effective_user
+from src.attachment_refs import attachment_refs_from_metadata
+from src.constants import GENERATED_IMAGES_DIR
+from src.upload_handler import (
+    UploadCleanupSafetyError,
+    count_recent_uploads,
+    extract_upload_ids,
+)
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/upload", tags=["upload"])
+UPLOAD_RESPONSE_HEADERS = {"X-Content-Type-Options": "nosniff"}
+
+def _upload_ids_from_persisted_text(value: object) -> set[str]:
+    """Return canonical upload IDs embedded in persisted text.
+
+    This covers attachment reference lines/URIs and the PDF source markers
+    stored by the document editor. False positives are intentionally
+    conservative: retaining an extra upload is safer than deleting referenced
+    bytes.
+    """
+    return extract_upload_ids(value)
+
+
+def _upload_ids_from_message_metadata(raw_metadata: object) -> set[str]:
+    """Extract attachment IDs from a persisted chat metadata JSON value.
+
+    Malformed metadata raises instead of being treated as an empty reference
+    set. The admin cleanup route catches that failure and aborts cleanup.
+    """
+    if raw_metadata in (None, ""):
+        return set()
+    if isinstance(raw_metadata, str):
+        metadata = json.loads(raw_metadata)
+    else:
+        metadata = raw_metadata
+    if not isinstance(metadata, dict):
+        raise ValueError("chat message metadata must be a JSON object")
+
+    attachments = metadata.get("attachments")
+    if attachments is not None:
+        if not isinstance(attachments, list) or any(
+            not isinstance(item, dict) for item in attachments
+        ):
+            raise ValueError("chat message attachments metadata is malformed")
+
+    ids = {
+        str(ref["attachment_id"])
+        for ref in attachment_refs_from_metadata(metadata)
+        if ref.get("attachment_id")
+    }
+    # Preserve canonical IDs even in older metadata shapes not normalized by
+    # attachment_refs_from_metadata().
+    ids.update(_upload_ids_from_persisted_text(json.dumps(metadata)))
+    return ids
+
+
+def _collect_persisted_upload_references() -> tuple[set[str], set[str]]:
+    """Collect upload IDs/hashes still referenced by durable application data.
+
+    The caller must treat any exception as an incomplete scan and fail closed.
+    There is no distinct artifact table in the current schema; artifact-like
+    attachment references persisted in chat/document text are covered by the
+    canonical-ID scan.
+    """
+    referenced_ids: set[str] = set()
+    referenced_hashes: set[str] = set()
+    db = SessionLocal()
+    try:
+        for content, raw_metadata in db.query(
+            DbChatMessage.content,
+            DbChatMessage.meta_data,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+            referenced_ids.update(_upload_ids_from_message_metadata(raw_metadata))
+
+        for (content,) in db.query(Document.current_content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for (content,) in db.query(DocumentVersion.content).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(content))
+
+        for filename, file_hash in db.query(
+            GalleryImage.filename,
+            GalleryImage.file_hash,
+        ).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(filename))
+            if file_hash:
+                referenced_hashes.add(str(file_hash))
+
+        for image_url, color, content, items in db.query(
+            Note.image_url,
+            Note.color,
+            Note.content,
+            Note.items,
+        ).yield_per(500):
+            for value in (image_url, color, content, items):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        for (color,) in db.query(CalendarCal.color).yield_per(500):
+            referenced_ids.update(_upload_ids_from_persisted_text(color))
+
+        for color, description, location in db.query(
+            CalendarEvent.color,
+            CalendarEvent.description,
+            CalendarEvent.location,
+        ).yield_per(500):
+            for value in (color, description, location):
+                referenced_ids.update(_upload_ids_from_persisted_text(value))
+
+        return referenced_ids, referenced_hashes
+    finally:
+        db.close()
+
+
+def _run_reference_safe_cleanup(upload_handler) -> int:
+    referenced_ids, referenced_hashes = _collect_persisted_upload_references()
+    return upload_handler.cleanup_old_uploads(
+        referenced_upload_ids=referenced_ids,
+        referenced_upload_hashes=referenced_hashes,
+    )
 
 def setup_upload_routes(upload_handler):
     """Setup upload routes with the provided handler"""
+
+    def _upload_root() -> str:
+        from src.constants import UPLOAD_DIR
+        return os.path.realpath(getattr(upload_handler, "upload_dir", UPLOAD_DIR))
+
+    def _path_inside_upload_dir(path: str) -> bool:
+        try:
+            return os.path.commonpath([_upload_root(), os.path.realpath(path)]) == _upload_root()
+        except Exception:
+            return False
+
+    def _resolve_upload_path(file_id: str) -> str:
+        from src.constants import UPLOAD_DIR
+        upload_root = getattr(upload_handler, "upload_dir", UPLOAD_DIR)
+        direct = os.path.join(upload_root, file_id)
+        if os.path.lexists(direct):
+            if not _path_inside_upload_dir(direct):
+                raise HTTPException(403, "Access denied")
+            if os.path.isfile(direct):
+                return direct
+            raise HTTPException(404, "File not found")
+
+        for root, _dirs, files in os.walk(upload_root, followlinks=False):
+            if file_id not in files:
+                continue
+            path = os.path.join(root, file_id)
+            if not _path_inside_upload_dir(path):
+                raise HTTPException(403, "Access denied")
+            if os.path.isfile(path):
+                return path
+            raise HTTPException(404, "File not found")
+
+        raise HTTPException(404, "File not found")
+
+    def _valid_session_id_for_owner(db, session_id: str | None, owner: str | None) -> str | None:
+        if not session_id:
+            return None
+        sess = db.query(DbSession).filter(DbSession.id == session_id).first()
+        if not sess:
+            return None
+        if owner and sess.owner and sess.owner != owner:
+            return None
+        return session_id
+
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None,
+                                       gallery_id: str | None = None) -> str | None:
+        """Make chat-uploaded images visible in Gallery without changing chat storage."""
+        is_image_file = getattr(upload_handler, "is_image_file", None)
+        if not callable(is_image_file):
+            return None
+        if not is_image_file(meta.get("name", ""), meta.get("mime", "")):
+            return None
+
+        source_path = meta.get("path")
+        if not source_path or not os.path.isfile(source_path):
+            return None
+
+        db = SessionLocal()
+        try:
+            file_hash = meta.get("hash")
+            if gallery_id:
+                existing = db.query(GalleryImage).filter(
+                    GalleryImage.id == gallery_id,
+                    GalleryImage.is_active == True,  # noqa: E712
+                ).first()
+                if existing and (not owner or existing.owner == owner):
+                    image_dir = Path(GENERATED_IMAGES_DIR)
+                    image_dir.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(source_path, image_dir / existing.filename)
+                    existing.file_hash = file_hash
+                    existing.file_size = meta.get("size")
+                    existing.width = meta.get("width")
+                    existing.height = meta.get("height")
+                    db.commit()
+                    return existing.id
+            if file_hash:
+                q = db.query(GalleryImage).filter(
+                    GalleryImage.file_hash == file_hash,
+                    GalleryImage.is_active == True,  # noqa: E712
+                )
+                if owner:
+                    q = q.filter(GalleryImage.owner == owner)
+                existing = q.first()
+                if existing:
+                    return existing.id
+
+            image_dir = Path(GENERATED_IMAGES_DIR)
+            image_dir.mkdir(parents=True, exist_ok=True)
+            ext = Path(meta.get("name") or source_path).suffix.lower()
+            if ext not in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
+                mime_ext = {
+                    "image/png": ".png",
+                    "image/jpeg": ".jpg",
+                    "image/jpg": ".jpg",
+                    "image/webp": ".webp",
+                    "image/gif": ".gif",
+                }.get(meta.get("mime", ""))
+                ext = mime_ext or ".png"
+            filename = f"{uuid.uuid4().hex[:12]}{ext}"
+            dest_path = image_dir / filename
+            shutil.copy2(source_path, dest_path)
+
+            image_id = str(uuid.uuid4())
+            db.add(GalleryImage(
+                id=image_id,
+                filename=filename,
+                prompt=meta.get("name") or "Chat upload",
+                model="chat-upload",
+                owner=owner,
+                session_id=_valid_session_id_for_owner(db, session_id, owner),
+                file_hash=file_hash,
+                width=meta.get("width"),
+                height=meta.get("height"),
+                file_size=meta.get("size"),
+            ))
+            db.commit()
+            return image_id
+        except Exception as e:
+            db.rollback()
+            logger.warning("Failed to add chat image upload to gallery: %s", e)
+            return None
+        finally:
+            db.close()
     
     @router.post("")
-    async def api_upload(request: Request, files: List[UploadFile] = File(...)):
+    async def api_upload(
+        request: Request,
+        files: List[UploadFile] = File(...),
+        session_id: Optional[str] = Form(None),
+        gallery_id: Optional[str] = Form(None),
+    ):
         """Upload files with enhanced security and organization."""
+        if not isinstance(session_id, str):
+            session_id = None
         if not files:
             raise HTTPException(400, "No files uploaded")
             
         client_ip = request.client.host if request.client else "unknown"
         out = []
-        
-        # Limit concurrent uploads per IP
-        ip_upload_count = sum(
-            1 for f in files 
-            if client_ip in upload_handler.upload_rate_log and 
-            any(now > time.time() - 10 for now in upload_handler.upload_rate_log[client_ip][-len(files):])
+
+        # Limit concurrent uploads per IP. Count genuine recent upload events —
+        # NOT the number of files in this batch. The previous check summed over
+        # `files`, so a single multi-file request counted itself as N concurrent
+        # uploads and tripped the limit (issue #1346: "attach more than one file
+        # → the model doesn't even see them"). save_upload still enforces the
+        # per-minute sliding-window rate limit per file.
+        recent_uploads = count_recent_uploads(
+            upload_handler.upload_rate_log.get(client_ip, []), time.time()
         )
-        
-        if ip_upload_count >= upload_handler.max_concurrent_uploads:
+
+        if recent_uploads >= upload_handler.max_concurrent_uploads:
             raise HTTPException(
                 status_code=429,
                 detail=f"Maximum concurrent uploads ({upload_handler.max_concurrent_uploads}) exceeded"
@@ -40,18 +304,25 @@ def setup_upload_routes(upload_handler):
         
         for u in files:
             try:
-                meta = upload_handler.save_upload(u, client_ip, owner=get_current_user(request))
-                out.append({
+                owner = effective_user(request)
+                meta = upload_handler.save_upload(u, client_ip, owner=owner)
+                promoted_gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id, gallery_id)
+                item = {
                     "id": meta["id"],
                     "name": meta["name"],
                     "mime": meta["mime"],
                     "size": meta["size"],
                     "hash": meta["hash"],
+                    "checksum_sha256": meta.get("checksum_sha256") or meta["hash"],
                     "uploaded_at": meta["uploaded_at"],
+                    "created_at": meta.get("created_at") or meta["uploaded_at"],
                     "width": meta.get("width"),
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
-                })
+                }
+                if promoted_gallery_id:
+                    item["gallery_id"] = promoted_gallery_id
+                out.append(item)
             except HTTPException:
                 raise
             except Exception as e:
@@ -67,7 +338,23 @@ def setup_upload_routes(upload_handler):
     async def manual_cleanup(request: Request):
         """Manually trigger cleanup of old uploads."""
         require_admin(request)
-        cleaned_count = upload_handler.cleanup_old_uploads()
+        try:
+            cleaned_count = await asyncio.to_thread(
+                _run_reference_safe_cleanup,
+                upload_handler,
+            )
+        except UploadCleanupSafetyError:
+            logger.exception("Upload cleanup aborted because index safety checks failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup aborted because upload index integrity could not be verified",
+            )
+        except Exception:
+            logger.exception("Upload cleanup skipped because reference discovery failed")
+            raise HTTPException(
+                503,
+                "Upload cleanup skipped because persisted references could not be verified",
+            )
         return {"status": "success", "files_cleaned": cleaned_count}
 
     @router.get("/stats")
@@ -87,45 +374,33 @@ def setup_upload_routes(upload_handler):
         client isn't downloading the full-resolution photo just to show it tiny."""
         if not upload_handler.validate_upload_id(file_id):
             raise HTTPException(400, "Invalid file ID")
-        # Search upload directories for the file
-        from src.constants import UPLOAD_DIR
         import mimetypes as _mt
-        path = os.path.join(UPLOAD_DIR, file_id)
-        if not os.path.exists(path):
-            for root, dirs, files in os.walk(UPLOAD_DIR):
-                if file_id in files:
-                    path = os.path.join(root, file_id)
-                    break
-            else:
-                raise HTTPException(404, "File not found")
-        if not upload_handler.inside_base_dir(path):
-            raise HTTPException(403, "Access denied")
         # Look up original filename and owner from uploads.json
         original_name = file_id
-        info = None
-        uploads_db = os.path.join(UPLOAD_DIR, "uploads.json")
-        if os.path.exists(uploads_db):
-            with open(uploads_db) as f:
-                db = json.load(f)
-            info = next((fi for fi in db.values() if fi["id"] == file_id), None)
-            if info:
-                original_name = info.get("name", file_id)
+        # _load_upload_index() tolerates a missing/corrupt uploads.json (it falls
+        # back to the .bak sibling, then to {}), so a truncated DB degrades to
+        # "no metadata" instead of a 500 from an unhandled JSONDecodeError.
+        db = upload_handler._load_upload_index()
+        info = next((fi for fi in db.values() if fi.get("id") == file_id), None)
+        if info:
+            original_name = info.get("name", file_id)
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner") if info else None
         if auth_configured:
             if not current_user:
                 raise HTTPException(403, "Access denied")
             if file_owner != current_user and not auth_mgr.is_admin(current_user):
                 raise HTTPException(404, "File not found")
-        mime = _mt.guess_type(path)[0] or "application/octet-stream"
+        path = _resolve_upload_path(file_id)
+        mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "application/octet-stream"
         from fastapi.responses import FileResponse
         # Downscaled thumbnail for image previews — generated once and cached.
         if thumb and mime.startswith("image/"):
             try:
                 from PIL import Image, ImageOps
-                thumb_dir = os.path.join(UPLOAD_DIR, ".thumbs")
+                thumb_dir = os.path.join(_upload_root(), ".thumbs")
                 os.makedirs(thumb_dir, exist_ok=True)
                 thumb_path = os.path.join(thumb_dir, file_id + ".jpg")
                 if (not os.path.exists(thumb_path)
@@ -141,29 +416,55 @@ def setup_upload_routes(upload_handler):
                     if im.mode not in ("RGB", "L"):
                         im = im.convert("RGB")
                     im.save(thumb_path, "JPEG", quality=80)
-                return FileResponse(thumb_path, media_type="image/jpeg")
+                return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS)
             except Exception as e:
                 logger.warning(f"Thumbnail generation failed for {file_id}: {e}")
                 # Fall through to the full image.
-        return FileResponse(path, media_type=mime, filename=original_name)
+        return FileResponse(
+            path,
+            media_type=mime,
+            filename=original_name,
+            headers=UPLOAD_RESPONSE_HEADERS,
+        )
 
     def _load_upload_info(file_id: str):
         """Look up the uploads.json record for a file_id, with owner/auth checks."""
-        from src.constants import UPLOAD_DIR
-        info = None
-        uploads_db = os.path.join(UPLOAD_DIR, "uploads.json")
-        if os.path.exists(uploads_db):
-            with open(uploads_db) as f:
-                db = json.load(f)
-            info = next((fi for fi in db.values() if fi["id"] == file_id), None)
-        return info
+        # Corruption-tolerant load (see download_file): a bad uploads.json yields
+        # {} rather than raising JSONDecodeError out of the vision path.
+        db = upload_handler._load_upload_index()
+        return next((fi for fi in db.values() if fi.get("id") == file_id), None)
 
     def _vision_cache_path(file_id: str) -> str:
-        from src.constants import UPLOAD_DIR
-        cache_dir = os.path.join(UPLOAD_DIR, ".vision")
+        cache_dir = os.path.join(_upload_root(), ".vision")
         os.makedirs(cache_dir, exist_ok=True)
         return os.path.join(cache_dir, file_id + ".txt")
 
+    def _sync_gallery_caption_for_upload(info: dict | None, owner: str | None, text: str) -> None:
+        """Copy upload OCR/vision text onto the promoted gallery image row."""
+        if not info:
+            return
+        file_hash = info.get("hash")
+        if not file_hash:
+            return
+        db = SessionLocal()
+        try:
+            q = db.query(GalleryImage).filter(
+                GalleryImage.file_hash == file_hash,
+                GalleryImage.is_active == True,  # noqa: E712
+            )
+            if owner:
+                q = q.filter(GalleryImage.owner == owner)
+            img = q.first()
+            if not img:
+                return
+            img.caption = (text or "").strip()
+            db.commit()
+        except Exception as e:
+            db.rollback()
+            logger.warning("Failed to sync OCR caption to gallery image: %s", e)
+        finally:
+            db.close()
+
     @router.get("/{file_id}/vision")
     async def get_vision_text(request: Request, file_id: str, force: int = 0):
         """Return the vision-model OCR/description for an uploaded image.
@@ -171,49 +472,42 @@ def setup_upload_routes(upload_handler):
         subsequent loads are instant. Pass force=1 to recompute."""
         if not upload_handler.validate_upload_id(file_id):
             raise HTTPException(400, "Invalid file ID")
-        from src.constants import UPLOAD_DIR
-        path = os.path.join(UPLOAD_DIR, file_id)
-        if not os.path.exists(path):
-            for root, dirs, files in os.walk(UPLOAD_DIR):
-                if file_id in files:
-                    path = os.path.join(root, file_id)
-                    break
-            else:
-                raise HTTPException(404, "File not found")
-        if not upload_handler.inside_base_dir(path):
-            raise HTTPException(403, "Access denied")
         info = _load_upload_info(file_id)
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner") if info else None
         if auth_configured:
             if not current_user:
                 raise HTTPException(403, "Access denied")
             if file_owner != current_user and not auth_mgr.is_admin(current_user):
                 raise HTTPException(404, "File not found")
+        path = _resolve_upload_path(file_id)
         import mimetypes as _mt
-        mime = _mt.guess_type(path)[0] or ""
+        mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or ""
         if not mime.startswith("image/"):
             raise HTTPException(400, "Not an image")
         cache_path = _vision_cache_path(file_id)
         if not force and os.path.exists(cache_path):
             try:
-                with open(cache_path) as f:
-                    return {"text": f.read(), "cached": True}
+                with open(cache_path, encoding="utf-8") as f:
+                    cached_text = f.read()
+                _sync_gallery_caption_for_upload(info, file_owner or current_user, cached_text)
+                return {"text": cached_text, "cached": True}
             except Exception as e:
                 logger.warning(f"Vision cache read failed for {file_id}: {e}")
         from src.document_processor import analyze_image_with_vl
         try:
-            text = analyze_image_with_vl(path) or ""
+            text = analyze_image_with_vl(path, owner=current_user) or ""
         except Exception as e:
             logger.error(f"Vision analysis failed for {file_id}: {e}")
             raise HTTPException(500, f"Vision analysis failed: {e}")
         try:
-            with open(cache_path, "w") as f:
+            with open(cache_path, "w", encoding="utf-8") as f:
                 f.write(text)
         except Exception as e:
             logger.warning(f"Vision cache write failed for {file_id}: {e}")
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"text": text, "cached": False}
 
     @router.put("/{file_id}/vision")
@@ -227,19 +521,24 @@ def setup_upload_routes(upload_handler):
             raise HTTPException(404, "File not found")
         auth_mgr = getattr(request.app.state, "auth_manager", None)
         auth_configured = bool(auth_mgr and auth_mgr.is_configured)
-        current_user = get_current_user(request)
+        current_user = effective_user(request)
         file_owner = info.get("owner")
         if auth_configured:
             if not current_user:
                 raise HTTPException(403, "Access denied")
             if file_owner != current_user and not auth_mgr.is_admin(current_user):
                 raise HTTPException(404, "File not found")
-        body = await request.json()
+        _resolve_upload_path(file_id)
+        try:
+            body = await request.json()
+        except json.JSONDecodeError:
+            raise HTTPException(400, "Request body must be valid JSON")
         text = (body or {}).get("text", "")
         if not isinstance(text, str):
             raise HTTPException(400, "text must be a string")
-        with open(_vision_cache_path(file_id), "w") as f:
+        with open(_vision_cache_path(file_id), "w", encoding="utf-8") as f:
             f.write(text)
+        _sync_gallery_caption_for_upload(info, file_owner or current_user, text)
         return {"ok": True}
 
     async def periodic_rate_limit_cleanup():

routes/vault/__init__.py

@@ -0,0 +1,5 @@
+"""Vault route domain package (slice 2k, #4082/#4071).
+
+Contains vault_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/vault_routes.py re-exports from here.
+"""

routes/vault/vault_routes.py

@@ -0,0 +1,242 @@
+"""
+vault_routes.py
+
+Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
+Stores the BW_SESSION key in data/vault.json with restrictive permissions.
+"""
+
+import json
+import logging
+import os
+import shutil
+import asyncio
+from pathlib import Path
+from datetime import datetime
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+
+from core.middleware import require_admin
+from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool
+from src.constants import VAULT_FILE as _VAULT_FILE
+
+logger = logging.getLogger(__name__)
+
+VAULT_FILE = Path(_VAULT_FILE)
+
+
+def _find_bw() -> str:
+    """Locate the bw binary, checking PATH and common npm-global locations.
+
+    On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by
+    which_tool via PATHEXT.
+    """
+    p = which_tool("bw")
+    if p:
+        return p
+    if IS_WINDOWS:
+        appdata = os.environ.get("APPDATA", os.path.expanduser("~"))
+        for candidate in (
+            os.path.join(appdata, "npm", "bw.cmd"),
+            os.path.join(appdata, "npm", "bw.exe"),
+        ):
+            if os.path.isfile(candidate):
+                return candidate
+        return "bw"
+    home = os.path.expanduser("~")
+    for candidate in (
+        f"{home}/.npm-global/bin/bw",
+        f"{home}/.nvm/versions/node/*/bin/bw",
+        "/usr/local/bin/bw",
+        "/opt/homebrew/bin/bw",
+    ):
+        if "*" in candidate:
+            import glob
+            for m in glob.glob(candidate):
+                if os.path.isfile(m) and os.access(m, os.X_OK):
+                    return m
+        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+            return candidate
+    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
+
+
+def _load_config() -> dict:
+    if VAULT_FILE.exists():
+        try:
+            data = json.loads(VAULT_FILE.read_text(encoding="utf-8"))
+            return data if isinstance(data, dict) else {}
+        except Exception:
+            pass
+    return {}
+
+
+def _save_config(cfg: dict):
+    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
+    VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
+    # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir
+    # is ACL-restricted already).
+    safe_chmod(str(VAULT_FILE), 0o600)
+
+
+async def _run_bw(args: list, session: str = None, input_text: str = None,
+                  bw_password: str = None) -> tuple:
+    env = {}
+    env.update(os.environ)
+    if session:
+        env["BW_SESSION"] = session
+    # Secrets must never be passed as argv — process arguments are world-readable
+    # via `ps` / `/proc/<pid>/cmdline` to any local user. Keep --passwordenv
+    # support for bw commands that need it; unlock/login callers should prefer
+    # stdin so the master password is not left in the child environment either.
+    if bw_password is not None:
+        env["BW_PASSWORD"] = bw_password
+    bw_path = _find_bw()
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            bw_path, *args,
+            stdin=asyncio.subprocess.PIPE if input_text else None,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+    except FileNotFoundError:
+        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
+    except Exception as e:
+        return "", f"Failed to launch bw: {e}", 1
+    try:
+        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
+    except Exception as e:
+        return "", f"bw subprocess error: {e}", 1
+    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
+
+
+class VaultConfig(BaseModel):
+    server_url: str = ""
+    email: str = ""
+
+
+class VaultUnlockRequest(BaseModel):
+    master_password: str
+
+
+class VaultLoginRequest(BaseModel):
+    email: str
+    master_password: str
+
+
+def setup_vault_routes():
+    router = APIRouter(prefix="/api/vault", tags=["vault"])
+
+    @router.get("/config")
+    async def get_config(request: Request):
+        """Return vault config (no sensitive fields)."""
+        require_admin(request)
+        cfg = _load_config()
+        return {
+            "server_url": cfg.get("server_url", ""),
+            "email": cfg.get("email", ""),
+            "unlocked": bool(cfg.get("session")),
+            "unlocked_at": cfg.get("unlocked_at", ""),
+            "bw_installed": await _check_bw_installed(),
+        }
+
+    @router.post("/config")
+    async def save_config(req: VaultConfig, request: Request):
+        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg["server_url"] = req.server_url.strip().rstrip("/")
+        cfg["email"] = req.email.strip()
+
+        if cfg["server_url"]:
+            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
+            if rc != 0:
+                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
+
+        _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/login")
+    async def login(req: VaultLoginRequest, request: Request):
+        """Log in to Vaultwarden (required once per account)."""
+        require_admin(request)
+        cfg = _load_config()
+        # Update email
+        cfg["email"] = req.email
+        _save_config(cfg)
+
+        stdout, stderr, rc = await _run_bw(
+            ["login", req.email, "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            # Already logged in is OK
+            if "already logged in" in stderr.lower():
+                return {"ok": True, "already": True}
+            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
+        # bw login --raw prints session key on success (when 2FA disabled)
+        if stdout:
+            cfg["session"] = stdout
+            cfg["unlocked_at"] = datetime.utcnow().isoformat()
+            _save_config(cfg)
+        return {"ok": True}
+
+    @router.post("/unlock")
+    async def unlock(req: VaultUnlockRequest, request: Request):
+        """Unlock the vault and save the session key."""
+        require_admin(request)
+        # Pass the master password on stdin, not argv. argv is visible through
+        # `ps` / /proc/<pid>/cmdline; stdin also avoids leaving the secret in
+        # the child process environment.
+        stdout, stderr, rc = await _run_bw(
+            ["unlock", "--raw"],
+            input_text=req.master_password + "\n",
+        )
+        if rc != 0:
+            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
+        session = stdout.strip()
+        if not session:
+            return {"ok": False, "error": "bw returned empty session"}
+        cfg = _load_config()
+        cfg["session"] = session
+        cfg["unlocked_at"] = datetime.utcnow().isoformat()
+        _save_config(cfg)
+        return {"ok": True, "message": "Vault unlocked"}
+
+    @router.post("/lock")
+    async def lock(request: Request):
+        """Lock the vault (clear session from config)."""
+        require_admin(request)
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        # Also tell bw to lock
+        await _run_bw(["lock"])
+        return {"ok": True, "message": "Vault locked"}
+
+    @router.post("/logout")
+    async def logout(request: Request):
+        """Log out of the Bitwarden CLI completely."""
+        require_admin(request)
+        await _run_bw(["logout"])
+        cfg = _load_config()
+        cfg.pop("session", None)
+        cfg.pop("email", None)
+        cfg.pop("unlocked_at", None)
+        _save_config(cfg)
+        return {"ok": True}
+
+    return router
+
+
+async def _check_bw_installed() -> bool:
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            _find_bw(), "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.communicate()
+        return proc.returncode == 0
+    except Exception:
+        return False

routes/vault_routes.py

@@ -1,216 +1,14 @@
-"""
-vault_routes.py
+"""Backward-compat shim — canonical location is routes/vault/vault_routes.py.
 
-Vaultwarden / Bitwarden CLI integration — config and unlock endpoints.
-Stores the BW_SESSION key in data/vault.json with restrictive permissions.
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.vault_routes``, ``from routes.vault_routes import X``,
+and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used
+by test_vault_password_not_in_argv.py all operate on the *same* object.
+Keeps existing import paths working after slice 2k (#4082/#4071).
 """
 
-import json
-import logging
-import os
-import shutil
-import asyncio
-from pathlib import Path
-from datetime import datetime
-from fastapi import APIRouter, Request
-from pydantic import BaseModel
+import sys as _sys
 
-from core.middleware import require_admin
+from routes.vault import vault_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-VAULT_FILE = Path("data/vault.json")
-
-
-def _find_bw() -> str:
-    """Locate the bw binary, checking PATH and common npm-global locations."""
-    p = shutil.which("bw")
-    if p:
-        return p
-    home = os.path.expanduser("~")
-    for candidate in (
-        f"{home}/.npm-global/bin/bw",
-        f"{home}/.nvm/versions/node/*/bin/bw",
-        "/usr/local/bin/bw",
-        "/opt/homebrew/bin/bw",
-    ):
-        if "*" in candidate:
-            import glob
-            for m in glob.glob(candidate):
-                if os.path.isfile(m) and os.access(m, os.X_OK):
-                    return m
-        elif os.path.isfile(candidate) and os.access(candidate, os.X_OK):
-            return candidate
-    return "bw"  # fall back to PATH lookup (will FileNotFoundError, handled below)
-
-
-def _load_config() -> dict:
-    if VAULT_FILE.exists():
-        try:
-            return json.loads(VAULT_FILE.read_text())
-        except Exception:
-            pass
-    return {}
-
-
-def _save_config(cfg: dict):
-    VAULT_FILE.parent.mkdir(parents=True, exist_ok=True)
-    VAULT_FILE.write_text(json.dumps(cfg, indent=2))
-    try:
-        os.chmod(str(VAULT_FILE), 0o600)
-    except Exception:
-        pass
-
-
-async def _run_bw(args: list, session: str = None, input_text: str = None) -> tuple:
-    env = {}
-    env.update(os.environ)
-    if session:
-        env["BW_SESSION"] = session
-    bw_path = _find_bw()
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            bw_path, *args,
-            stdin=asyncio.subprocess.PIPE if input_text else None,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            env=env,
-        )
-    except FileNotFoundError:
-        return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127
-    except Exception as e:
-        return "", f"Failed to launch bw: {e}", 1
-    try:
-        stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None)
-    except Exception as e:
-        return "", f"bw subprocess error: {e}", 1
-    return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode
-
-
-class VaultConfig(BaseModel):
-    server_url: str = ""
-    email: str = ""
-
-
-class VaultUnlockRequest(BaseModel):
-    master_password: str
-
-
-class VaultLoginRequest(BaseModel):
-    email: str
-    master_password: str
-
-
-def setup_vault_routes():
-    router = APIRouter(prefix="/api/vault", tags=["vault"])
-
-    @router.get("/config")
-    async def get_config(request: Request):
-        """Return vault config (no sensitive fields)."""
-        require_admin(request)
-        cfg = _load_config()
-        return {
-            "server_url": cfg.get("server_url", ""),
-            "email": cfg.get("email", ""),
-            "unlocked": bool(cfg.get("session")),
-            "unlocked_at": cfg.get("unlocked_at", ""),
-            "bw_installed": await _check_bw_installed(),
-        }
-
-    @router.post("/config")
-    async def save_config(req: VaultConfig, request: Request):
-        """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg["server_url"] = req.server_url.strip().rstrip("/")
-        cfg["email"] = req.email.strip()
-
-        if cfg["server_url"]:
-            _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]])
-            if rc != 0:
-                return {"ok": False, "error": f"bw config failed: {stderr[:300]}"}
-
-        _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/login")
-    async def login(req: VaultLoginRequest, request: Request):
-        """Log in to Vaultwarden (required once per account)."""
-        require_admin(request)
-        cfg = _load_config()
-        # Update email
-        cfg["email"] = req.email
-        _save_config(cfg)
-
-        stdout, stderr, rc = await _run_bw(
-            ["login", req.email, "--raw"],
-            input_text=req.master_password + "\n",
-        )
-        if rc != 0:
-            # Already logged in is OK
-            if "already logged in" in stderr.lower():
-                return {"ok": True, "already": True}
-            return {"ok": False, "error": f"Login failed: {stderr[:300]}"}
-        # bw login --raw prints session key on success (when 2FA disabled)
-        if stdout:
-            cfg["session"] = stdout
-            cfg["unlocked_at"] = datetime.utcnow().isoformat()
-            _save_config(cfg)
-        return {"ok": True}
-
-    @router.post("/unlock")
-    async def unlock(req: VaultUnlockRequest, request: Request):
-        """Unlock the vault and save the session key."""
-        require_admin(request)
-        stdout, stderr, rc = await _run_bw(
-            ["unlock", req.master_password, "--raw"],
-        )
-        if rc != 0:
-            return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"}
-        session = stdout.strip()
-        if not session:
-            return {"ok": False, "error": "bw returned empty session"}
-        cfg = _load_config()
-        cfg["session"] = session
-        cfg["unlocked_at"] = datetime.utcnow().isoformat()
-        _save_config(cfg)
-        return {"ok": True, "message": "Vault unlocked"}
-
-    @router.post("/lock")
-    async def lock(request: Request):
-        """Lock the vault (clear session from config)."""
-        require_admin(request)
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        # Also tell bw to lock
-        await _run_bw(["lock"])
-        return {"ok": True, "message": "Vault locked"}
-
-    @router.post("/logout")
-    async def logout(request: Request):
-        """Log out of the Bitwarden CLI completely."""
-        require_admin(request)
-        await _run_bw(["logout"])
-        cfg = _load_config()
-        cfg.pop("session", None)
-        cfg.pop("email", None)
-        cfg.pop("unlocked_at", None)
-        _save_config(cfg)
-        return {"ok": True}
-
-    return router
-
-
-async def _check_bw_installed() -> bool:
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            _find_bw(), "--version",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        await proc.communicate()
-        return proc.returncode == 0
-    except Exception:
-        return False
+_sys.modules[__name__] = _canonical

routes/webhook/__init__.py

@@ -0,0 +1,5 @@
+"""Webhook route domain package (slice 2l, #4082/#4071).
+
+Contains webhook_routes.py, migrated from the flat routes/ directory.
+Backward-compat shim at routes/webhook_routes.py re-exports from here.
+"""

routes/webhook/webhook_routes.py

@@ -0,0 +1,395 @@
+"""Webhook, API Token, and sync chat routes."""
+
+import uuid
+import logging
+from typing import Optional
+
+import httpx
+from fastapi import APIRouter, HTTPException, Request, Form
+from pydantic import BaseModel, Field
+
+from core.database import SessionLocal, Webhook, ModelEndpoint
+from src.auth_helpers import owner_filter
+from src.url_security import validate_public_http_url
+from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api", tags=["webhooks"])
+
+# Input limits
+MAX_NAME_LEN = 100
+MAX_URL_LEN = 2048
+MAX_SECRET_LEN = 256
+MAX_MESSAGE_LEN = 32_000
+
+
+from core.middleware import require_admin as _require_admin
+
+
+def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]):
+    """First enabled ModelEndpoint visible to token_owner — their own rows plus
+    legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would
+    let a chat-scoped token fall back onto another user's private endpoint and
+    silently spend that owner's API key/quota. Prefer owner rows before shared
+    rows. Fails closed to null-owner rows only when token_owner is absent.
+    Does not validate base_url — admin-configured local/LAN endpoints remain allowed.
+    """
+    query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)  # noqa: E712
+    if token_owner:
+        query = owner_filter(query, ModelEndpoint, token_owner)
+        return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first()
+    return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first()  # noqa: E711
+
+
+def _caller_owns_session(sess_owner, caller) -> bool:
+    """Strict session-ownership gate for the token-authenticated sync-chat
+    endpoint (`POST /api/v1/chat`).
+
+    Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner
+    gates in notes/calendar/gallery: a caller may resume a session ONLY when
+    its owner matches them exactly. A null/empty session owner (legacy or
+    migrated rows) is deliberately NOT resumable by an arbitrary token — the
+    old ``sess_owner and sess_owner != caller`` form skipped the check whenever
+    ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile
+    device) could resume such a session, inject a message, and read back its
+    history and reuse the owner's endpoint credentials. Fail closed: an
+    unresolvable caller also returns False.
+    """
+    if not caller:
+        return False
+    return sess_owner == caller
+
+
+def setup_webhook_routes(
+    webhook_manager: WebhookManager,
+    auth_manager,
+    session_manager=None,
+    api_key_manager=None,
+) -> APIRouter:
+
+    @router.get("/webhooks")
+    def list_webhooks(request: Request):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            hooks = db.query(Webhook).all()
+            return [
+                {
+                    "id": w.id,
+                    "name": w.name,
+                    "url": w.url,
+                    "has_secret": bool(w.secret),
+                    "events": w.events.split(",") if w.events else [],
+                    "is_active": w.is_active,
+                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
+                    "last_status_code": w.last_status_code,
+                    "last_error": w.last_error,
+                    "created_at": w.created_at.isoformat() if w.created_at else None,
+                }
+                for w in hooks
+            ]
+        finally:
+            db.close()
+
+    @router.post("/webhooks")
+    def create_webhook(
+        request: Request,
+        name: str = Form(""),
+        url: str = Form(""),
+        secret: str = Form(""),
+        events: str = Form(""),
+    ):
+        _require_admin(request)
+        name = name.strip()[:MAX_NAME_LEN]
+        if not name:
+            raise HTTPException(400, "Webhook name is required")
+        try:
+            url = validate_webhook_url(url)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+        try:
+            events = validate_events(events)
+        except ValueError as e:
+            raise HTTPException(400, str(e))
+
+        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
+        # Encrypt the secret at rest using the same Fernet key as API keys
+        encrypted_secret = None
+        if secret_val and api_key_manager:
+            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
+        elif secret_val:
+            encrypted_secret = secret_val  # Fallback if no encryption available
+
+        webhook_id = str(uuid.uuid4())[:8]
+        db = SessionLocal()
+        try:
+            db.add(Webhook(
+                id=webhook_id,
+                name=name,
+                url=url,
+                secret=encrypted_secret,
+                events=events,
+                is_active=True,
+            ))
+            db.commit()
+        finally:
+            db.close()
+
+        return {"id": webhook_id, "name": name}
+
+    @router.post("/webhooks/{webhook_id}/test")
+    async def test_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            url, secret = wh.url, wh.secret
+        finally:
+            db.close()
+
+        await webhook_manager.deliver_test(webhook_id, url, secret)
+        return {"status": "sent"}
+
+    @router.patch("/webhooks/{webhook_id}")
+    def toggle_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
+            if not wh:
+                raise HTTPException(404, "Webhook not found")
+            wh.is_active = not wh.is_active
+            db.commit()
+            return {"id": webhook_id, "is_active": wh.is_active}
+        finally:
+            db.close()
+
+    @router.delete("/webhooks/{webhook_id}")
+    def delete_webhook(request: Request, webhook_id: str):
+        _require_admin(request)
+        db = SessionLocal()
+        try:
+            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
+            db.commit()
+            if not deleted:
+                raise HTTPException(404, "Webhook not found")
+        finally:
+            db.close()
+        return {"status": "deleted"}
+
+    # ================================================================
+    # Sync Chat Endpoint (for n8n / Make / Activepieces)
+    # ================================================================
+
+    # Known provider base URLs — auto-resolved from api_key prefix or model name
+    KNOWN_PROVIDERS = {
+        "deepseek": "https://api.deepseek.com/v1",
+        "openai": "https://api.openai.com/v1",
+        "mistral": "https://api.mistral.ai/v1",
+        "groq": "https://api.groq.com/openai/v1",
+        "together": "https://api.together.xyz/v1",
+        "openrouter": "https://openrouter.ai/api/v1",
+        "ollama": "https://ollama.com/api",
+        "opencode-zen": "https://opencode.ai/zen/v1",
+        "opencode-go": "https://opencode.ai/zen/go/v1",
+        "fireworks": "https://api.fireworks.ai/inference/v1",
+        "venice": "https://api.venice.ai/api/v1",
+        "kimi-code": "https://api.kimi.com/coding/v1",
+        "kimicode": "https://api.kimi.com/coding/v1",
+    }
+
+    # Model prefix → provider mapping for auto-detection
+    MODEL_PROVIDER_MAP = {
+        "deepseek": "deepseek",
+        "gpt-": "openai",
+        "o1": "openai",
+        "o3": "openai",
+        "o4": "openai",
+        "mistral": "mistral",
+        "llama": "groq",
+        "mixtral": "groq",
+        "kimi-for-coding": "kimi-code",
+        "kimi": "kimi-code",
+    }
+
+    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
+        """Try to auto-resolve a base URL from provider name or model prefix."""
+        if provider and provider.lower() in KNOWN_PROVIDERS:
+            return KNOWN_PROVIDERS[provider.lower()]
+        if model:
+            model_lower = model.lower()
+            for prefix, prov in MODEL_PROVIDER_MAP.items():
+                if model_lower.startswith(prefix):
+                    return KNOWN_PROVIDERS[prov]
+        return None
+
+    class SyncChatRequest(BaseModel):
+        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
+        model: Optional[str] = Field(None, max_length=200)
+        session: Optional[str] = Field(None, max_length=100)
+        api_key: Optional[str] = Field(None, max_length=256)
+        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
+        provider: Optional[str] = Field(None, max_length=50)
+
+    @router.post("/v1/chat")
+    async def sync_chat(request: Request, body: SyncChatRequest):
+        if not getattr(request.state, "api_token", False):
+            raise HTTPException(403, "This endpoint requires an API token")
+        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
+        if "chat" not in scopes:
+            raise HTTPException(403, "API token is not scoped for chat")
+        token_owner = getattr(request.state, "api_token_owner", None)
+
+        from core.models import ChatMessage
+        from src.llm_core import llm_call_async
+        from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
+
+        message = body.message.strip()
+        if not message:
+            raise HTTPException(400, "Message is required")
+
+        session_id = body.session
+        sess = None
+
+        # --- Case 1: Resume an existing session ---
+        if session_id and session_manager:
+            try:
+                sess = session_manager.get_session(session_id)
+            except (KeyError, Exception):
+                raise HTTPException(404, "Session not found")
+            # SECURITY: verify the API-token's user owns this session — without
+            # this any token holder could resume any user's chat by passing its
+            # ID. The token's user is on request.state.user (set by API-token
+            # middleware); fall back to require_user if not present.
+            try:
+                from src.auth_helpers import get_current_user as _gcu
+                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
+            except Exception:
+                _tok_user = None
+            # Strict ownership (see _caller_owns_session): fail closed so a
+            # null-owner / cross-owner session can't be resumed by an arbitrary
+            # chat-scoped token.
+            _sess_owner = getattr(sess, "owner", None)
+            if not _caller_owns_session(_sess_owner, _tok_user):
+                raise HTTPException(404, "Session not found")
+
+        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
+        if not sess and body.api_key:
+            api_key = body.api_key.strip()
+            model = body.model or "deepseek-chat"
+
+            # Validate only token-supplied direct base_url; auto-resolved known-provider
+            # URLs are not subject to extra local/LAN blocking beyond existing provider logic.
+            direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None
+            if direct_base_url:
+                try:
+                    base_url = validate_public_http_url(direct_base_url)
+                except ValueError as e:
+                    detail = str(e).replace("URL", "base_url", 1)
+                    raise HTTPException(400, detail)
+            else:
+                base_url = _resolve_base_url(model, body.provider)
+            if not base_url:
+                raise HTTPException(400,
+                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
+                    "or provider ('deepseek', 'openai', 'groq', etc.)")
+            base_url = normalize_base(base_url)
+            endpoint_url = build_chat_url(base_url)
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            sess.headers = build_headers(api_key, base_url)
+            session_manager.save_sessions()
+            session_id = sid
+
+        # --- Case 3: Fall back to first configured ModelEndpoint ---
+        if not sess:
+            db = SessionLocal()
+            try:
+                ep = _select_api_chat_fallback_endpoint(db, token_owner)
+            finally:
+                db.close()
+
+            if not ep:
+                raise HTTPException(400,
+                    "No session, api_key, or configured endpoints. "
+                    "Pass api_key + model, or configure an endpoint in Admin.")
+
+            base_url = normalize_base(ep.base_url)
+            endpoint_url = build_chat_url(base_url)
+            model = body.model or "auto"
+            api_key = ep.api_key
+            if getattr(ep, "provider_auth_id", None):
+                try:
+                    from src.endpoint_resolver import resolve_endpoint_runtime
+                    base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner)
+                    endpoint_url = build_chat_url(base_url)
+                except Exception:
+                    raise HTTPException(500, "Could not resolve endpoint credentials")
+
+            if model == "auto":
+                try:
+                    async with httpx.AsyncClient(timeout=5) as client:
+                        models_url = build_models_url(base_url)
+                        hdrs = build_headers(api_key, base_url)
+                        if models_url:
+                            resp = await client.get(models_url, headers=hdrs)
+                            resp.raise_for_status()
+                            data = resp.json()
+                            items = data if isinstance(data, list) else (data.get("data") or [])
+                            ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")]
+                            if not ids and isinstance(data, dict):
+                                ids = [
+                                    m.get("name") or m.get("model")
+                                    for m in (data.get("models") or [])
+                                    if m.get("name") or m.get("model")
+                                ]
+                        else:
+                            import json as _json
+                            ids = _json.loads(ep.cached_models or "[]")
+                        model = ids[0] if ids else "auto"
+                except Exception:
+                    raise HTTPException(500, "Could not discover models from endpoint")
+
+            if not session_manager:
+                raise HTTPException(500, "Session manager not available")
+
+            sid = str(uuid.uuid4())
+            sess = session_manager.create_session(
+                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
+                model=model, owner=token_owner,
+            )
+            if api_key:
+                sess.headers = build_headers(api_key, base_url)
+                session_manager.save_sessions()
+            session_id = sid
+
+        # --- Send message and get response ---
+        sess.add_message(ChatMessage("user", message))
+
+        messages = [{"role": m.role, "content": m.content} for m in sess.history]
+
+        reply = await llm_call_async(
+            sess.endpoint_url, sess.model, messages,
+            headers=sess.headers, timeout=120,
+        )
+        sess.add_message(ChatMessage("assistant", reply))
+        session_manager.save_sessions()
+
+        webhook_manager.fire_and_forget("chat.completed", {
+            "session_id": session_id, "model": sess.model,
+            "user_message": message[:2000], "response": reply[:2000],
+        })
+
+        return {"response": reply, "session_id": session_id, "model": sess.model}
+
+    return router

routes/webhook_routes.py

@@ -1,322 +1,16 @@
-"""Webhook, API Token, and sync chat routes."""
+"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py.
 
-import asyncio
-import uuid
-import logging
-from typing import Optional
+This module is replaced in ``sys.modules`` by the canonical module object so
+that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``,
+``importlib.import_module("routes.webhook_routes")``, and the
+``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod,
+...)`` pattern used by test_null_owner_gates.py all operate on the *same*
+object. Keeps existing import paths working after slice 2l (#4082/#4071).
+Source-introspection tests read the canonical file by path.
+"""
 
-import httpx
-from fastapi import APIRouter, HTTPException, Request, Form
-from pydantic import BaseModel, Field
+import sys as _sys
 
-from core.database import SessionLocal, Webhook
-from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events
+from routes.webhook import webhook_routes as _canonical  # noqa: F401
 
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/api", tags=["webhooks"])
-
-# Input limits
-MAX_NAME_LEN = 100
-MAX_URL_LEN = 2048
-MAX_SECRET_LEN = 256
-MAX_MESSAGE_LEN = 32_000
-
-
-from core.middleware import require_admin as _require_admin
-
-
-def setup_webhook_routes(
-    webhook_manager: WebhookManager,
-    auth_manager,
-    session_manager=None,
-    api_key_manager=None,
-) -> APIRouter:
-
-    @router.get("/webhooks")
-    def list_webhooks(request: Request):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            hooks = db.query(Webhook).all()
-            return [
-                {
-                    "id": w.id,
-                    "name": w.name,
-                    "url": w.url,
-                    "has_secret": bool(w.secret),
-                    "events": w.events.split(",") if w.events else [],
-                    "is_active": w.is_active,
-                    "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None,
-                    "last_status_code": w.last_status_code,
-                    "last_error": w.last_error,
-                    "created_at": w.created_at.isoformat() if w.created_at else None,
-                }
-                for w in hooks
-            ]
-        finally:
-            db.close()
-
-    @router.post("/webhooks")
-    def create_webhook(
-        request: Request,
-        name: str = Form(""),
-        url: str = Form(""),
-        secret: str = Form(""),
-        events: str = Form(""),
-    ):
-        _require_admin(request)
-        name = name.strip()[:MAX_NAME_LEN]
-        if not name:
-            raise HTTPException(400, "Webhook name is required")
-        try:
-            url = validate_webhook_url(url)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-        try:
-            events = validate_events(events)
-        except ValueError as e:
-            raise HTTPException(400, str(e))
-
-        secret_val = secret.strip()[:MAX_SECRET_LEN] or None
-        # Encrypt the secret at rest using the same Fernet key as API keys
-        encrypted_secret = None
-        if secret_val and api_key_manager:
-            encrypted_secret = api_key_manager.encrypt_api_key(secret_val)
-        elif secret_val:
-            encrypted_secret = secret_val  # Fallback if no encryption available
-
-        webhook_id = str(uuid.uuid4())[:8]
-        db = SessionLocal()
-        try:
-            db.add(Webhook(
-                id=webhook_id,
-                name=name,
-                url=url,
-                secret=encrypted_secret,
-                events=events,
-                is_active=True,
-            ))
-            db.commit()
-        finally:
-            db.close()
-
-        return {"id": webhook_id, "name": name}
-
-    @router.post("/webhooks/{webhook_id}/test")
-    async def test_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            url, secret = wh.url, wh.secret
-        finally:
-            db.close()
-
-        await webhook_manager.deliver_test(webhook_id, url, secret)
-        return {"status": "sent"}
-
-    @router.patch("/webhooks/{webhook_id}")
-    def toggle_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            wh = db.query(Webhook).filter(Webhook.id == webhook_id).first()
-            if not wh:
-                raise HTTPException(404, "Webhook not found")
-            wh.is_active = not wh.is_active
-            db.commit()
-            return {"id": webhook_id, "is_active": wh.is_active}
-        finally:
-            db.close()
-
-    @router.delete("/webhooks/{webhook_id}")
-    def delete_webhook(request: Request, webhook_id: str):
-        _require_admin(request)
-        db = SessionLocal()
-        try:
-            deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete()
-            db.commit()
-            if not deleted:
-                raise HTTPException(404, "Webhook not found")
-        finally:
-            db.close()
-        return {"status": "deleted"}
-
-    # ================================================================
-    # Sync Chat Endpoint (for n8n / Make / Activepieces)
-    # ================================================================
-
-    # Known provider base URLs — auto-resolved from api_key prefix or model name
-    KNOWN_PROVIDERS = {
-        "deepseek": "https://api.deepseek.com/v1",
-        "openai": "https://api.openai.com/v1",
-        "mistral": "https://api.mistral.ai/v1",
-        "groq": "https://api.groq.com/openai/v1",
-        "together": "https://api.together.xyz/v1",
-        "openrouter": "https://openrouter.ai/api/v1",
-        "fireworks": "https://api.fireworks.ai/inference/v1",
-    }
-
-    # Model prefix → provider mapping for auto-detection
-    MODEL_PROVIDER_MAP = {
-        "deepseek": "deepseek",
-        "gpt-": "openai",
-        "o1": "openai",
-        "o3": "openai",
-        "o4": "openai",
-        "mistral": "mistral",
-        "llama": "groq",
-        "mixtral": "groq",
-    }
-
-    def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]:
-        """Try to auto-resolve a base URL from provider name or model prefix."""
-        if provider and provider.lower() in KNOWN_PROVIDERS:
-            return KNOWN_PROVIDERS[provider.lower()]
-        if model:
-            model_lower = model.lower()
-            for prefix, prov in MODEL_PROVIDER_MAP.items():
-                if model_lower.startswith(prefix):
-                    return KNOWN_PROVIDERS[prov]
-        return None
-
-    class SyncChatRequest(BaseModel):
-        message: str = Field(..., max_length=MAX_MESSAGE_LEN)
-        model: Optional[str] = Field(None, max_length=200)
-        session: Optional[str] = Field(None, max_length=100)
-        api_key: Optional[str] = Field(None, max_length=256)
-        base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN)
-        provider: Optional[str] = Field(None, max_length=50)
-
-    @router.post("/v1/chat")
-    async def sync_chat(request: Request, body: SyncChatRequest):
-        if not getattr(request.state, "api_token", False):
-            raise HTTPException(403, "This endpoint requires an API token")
-        scopes = set(getattr(request.state, "api_token_scopes", []) or [])
-        if "chat" not in scopes:
-            raise HTTPException(403, "API token is not scoped for chat")
-        token_owner = getattr(request.state, "api_token_owner", None)
-
-        from core.models import ChatMessage
-        from src.llm_core import llm_call_async
-        from core.database import ModelEndpoint
-
-        message = body.message.strip()
-        if not message:
-            raise HTTPException(400, "Message is required")
-
-        session_id = body.session
-        sess = None
-
-        # --- Case 1: Resume an existing session ---
-        if session_id and session_manager:
-            try:
-                sess = session_manager.get_session(session_id)
-            except (KeyError, Exception):
-                raise HTTPException(404, "Session not found")
-            # SECURITY: verify the API-token's user owns this session — without
-            # this any token holder could resume any user's chat by passing its
-            # ID. The token's user is on request.state.user (set by API-token
-            # middleware); fall back to require_user if not present.
-            try:
-                from src.auth_helpers import get_current_user as _gcu
-                _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request)
-            except Exception:
-                _tok_user = None
-            _sess_owner = getattr(sess, "owner", None)
-            if _tok_user and _sess_owner and _sess_owner != _tok_user:
-                raise HTTPException(404, "Session not found")
-
-        # --- Case 2: Direct API key + model (no pre-configured endpoint needed) ---
-        if not sess and body.api_key:
-            api_key = body.api_key.strip()
-            model = body.model or "deepseek-chat"
-
-            # Resolve base_url: explicit > provider name > model prefix auto-detect
-            base_url = body.base_url.strip().rstrip("/") if body.base_url else None
-            if not base_url:
-                base_url = _resolve_base_url(model, body.provider)
-            if not base_url:
-                raise HTTPException(400,
-                    "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') "
-                    "or provider ('deepseek', 'openai', 'groq', etc.)")
-
-            endpoint_url = base_url + "/chat/completions"
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            sess.headers = {"Authorization": f"Bearer {api_key}"}
-            session_manager.save_sessions()
-            session_id = sid
-
-        # --- Case 3: Fall back to first configured ModelEndpoint ---
-        if not sess:
-            db = SessionLocal()
-            try:
-                ep = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).first()
-            finally:
-                db.close()
-
-            if not ep:
-                raise HTTPException(400,
-                    "No session, api_key, or configured endpoints. "
-                    "Pass api_key + model, or configure an endpoint in Admin.")
-
-            endpoint_url = ep.base_url.rstrip("/") + "/chat/completions"
-            model = body.model or "auto"
-            api_key = ep.api_key
-
-            if model == "auto":
-                try:
-                    async with httpx.AsyncClient(timeout=5) as client:
-                        models_url = ep.base_url.rstrip("/") + "/models"
-                        hdrs = {"Authorization": f"Bearer {api_key}"} if api_key else {}
-                        resp = await client.get(models_url, headers=hdrs)
-                        resp.raise_for_status()
-                        ids = [m.get("id") for m in (resp.json().get("data") or []) if m.get("id")]
-                        model = ids[0] if ids else "auto"
-                except Exception:
-                    raise HTTPException(500, "Could not discover models from endpoint")
-
-            if not session_manager:
-                raise HTTPException(500, "Session manager not available")
-
-            sid = str(uuid.uuid4())
-            sess = session_manager.create_session(
-                session_id=sid, name="API Chat", endpoint_url=endpoint_url,
-                model=model, owner=token_owner,
-            )
-            if api_key:
-                sess.headers = {"Authorization": f"Bearer {api_key}"}
-                session_manager.save_sessions()
-            session_id = sid
-
-        # --- Send message and get response ---
-        sess.add_message(ChatMessage("user", message))
-
-        messages = [{"role": m.role, "content": m.content} for m in sess.history]
-
-        reply = await llm_call_async(
-            sess.endpoint_url, sess.model, messages,
-            headers=sess.headers, timeout=120,
-        )
-        sess.add_message(ChatMessage("assistant", reply))
-        session_manager.save_sessions()
-
-        asyncio.create_task(webhook_manager.fire("chat.completed", {
-            "session_id": session_id, "model": sess.model,
-            "user_message": message[:2000], "response": reply[:2000],
-        }))
-
-        return {"response": reply, "session_id": session_id, "model": sess.model}
-
-    return router
+_sys.modules[__name__] = _canonical

routes/workspace_routes.py

@@ -0,0 +1,107 @@
+"""Workspace API - browse server directories to pick a tool workspace folder."""
+import os
+from fastapi import APIRouter, Request, HTTPException, Query
+
+from src.auth_helpers import get_current_user
+from src.tool_security import owner_is_admin_or_single_user
+
+# Cap entries returned per directory (mirrors filesystem_tools._CODENAV_MAX_HITS).
+# A huge directory shouldn't dump thousands of rows into the picker; the user can
+# type/paste a path to jump straight in instead.
+_MAX_BROWSE_DIRS = 500
+
+
+def setup_workspace_routes():
+    router = APIRouter(prefix="/api/workspace", tags=["workspace"])
+
+    @router.get("/browse")
+    def browse(request: Request, path: str = Query(default="")):
+        """List subdirectories of `path` (default: home) so the UI can navigate
+        the server filesystem and pick a workspace folder. Directories only.
+
+        ADMIN-ONLY: this enumerates the server filesystem, so it is gated the
+        same way the file/shell tools are (read_file/write_file/bash are in
+        NON_ADMIN_BLOCKED_TOOLS). A non-admin who can't use those tools must not
+        be able to map the host's directory tree either.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace browsing is admin-only")
+
+        # Resolve symlinks so the reported path is canonical and the UI navigates
+        # real directories (defends against symlink games in displayed paths).
+        target = os.path.realpath(os.path.expanduser(path.strip() or "~"))
+        if not os.path.isdir(target):
+            target = os.path.realpath(os.path.expanduser("~"))
+
+        dirs = []
+        try:
+            with os.scandir(target) as it:
+                for entry in it:
+                    try:
+                        # Don't follow symlinks when classifying - a symlinked
+                        # dir is skipped rather than letting the browser wander
+                        # off via a link. Hidden entries are omitted.
+                        if entry.is_dir(follow_symlinks=False) and not entry.name.startswith("."):
+                            # Build the child path server-side with os.path.join
+                            # so it's correct on Windows (backslashes) and Linux.
+                            dirs.append({"name": entry.name, "path": os.path.join(target, entry.name)})
+                    except OSError:
+                        continue
+        except (PermissionError, OSError):
+            dirs = []
+
+        dirs_sorted = sorted(dirs, key=lambda d: d["name"].lower())
+        truncated = len(dirs_sorted) > _MAX_BROWSE_DIRS
+        parent = os.path.dirname(target)
+        from src.tool_execution import vet_workspace
+        return {
+            "path": target,
+            "parent": parent if parent and parent != target else None,
+            "dirs": dirs_sorted[:_MAX_BROWSE_DIRS],
+            "truncated": truncated,
+            # Whether this directory may be bound as a workspace (filesystem
+            # roots and sensitive dirs may be browsed through but not chosen).
+            "selectable": vet_workspace(target) is not None,
+        }
+
+    @router.get("/vet")
+    def vet(request: Request, path: str = Query(default="")):
+        """Validate a workspace path without binding it.
+
+        The UI calls this before persisting a manually typed path (/workspace
+        set) so a typo, file path, deleted folder, sensitive dir, or filesystem
+        root is rejected up front with the canonical path returned on success,
+        instead of being stored client-side and silently dropped at chat time.
+        Admin-gated like /browse: it confirms path existence on the host.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace selection is admin-only")
+        from src.tool_execution import vet_workspace
+        resolved = vet_workspace(path)
+        return {"ok": resolved is not None, "path": resolved}
+
+    @router.get("/default")
+    def default_workspace(request: Request):
+        """Return the explicitly configured backend workspace, if usable.
+
+        WebUI has no local launch directory: it runs against this backend's
+        filesystem. An explicit default gives it the same zero-setup behavior
+        as TUI while keeping workspace access opt-in and server-vetted.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace default is admin-only")
+
+        configured = os.environ.get("ODYSSEUS_WORKSPACE_DEFAULT", "").strip()
+        if not configured:
+            return {"ok": False, "path": None}
+
+        from src.tool_execution import vet_workspace
+        from src.workspace_paths import backend_workspace_path
+
+        resolved = vet_workspace(backend_workspace_path(configured))
+        return {"ok": resolved is not None, "path": resolved}
+
+    return router