feat(photo-editor): NL editor planner, image ops, email/gallery server

Server-side photo editor: natural-language editor agent route
(/api/editor/agent), photo_editor tool family + style guide, image-op
provider proxy with progress, email server hardening, gallery/hwfit
routes, and graceful shutdown.

Source: vendor/photocraft (ad1cff44) editor-server files, merged onto
upstream/dev (a8c147b2). Conflict resolution documented in
pr_work/B_conflicts.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

routes/email/email_helpers.py

@@ -17,8 +17,6 @@ import base64
 import time
 import imaplib
 import smtplib
-import socket
-import ssl
 import email as email_mod
 import email.header
 import email.utils
@@ -38,7 +36,6 @@ from typing import Optional, List
 
 from src.auth_helpers import _auth_disabled, get_current_user
 from src.secret_storage import decrypt as _decrypt
-from src.url_safety import OutboundAddressBlocked, connect_outbound_tcp
 
 logger = logging.getLogger(__name__)
 
@@ -163,94 +160,6 @@ def _smtp_security_mode(cfg: dict) -> str:
     return "ssl"
 
 
-# Raised when a mail host resolves into a denied range (see _mail_private_blocked).
-MailServerAddressBlocked = OutboundAddressBlocked
-
-
-def _mail_private_blocked(owner: str | None) -> bool:
-    """Whether private/loopback/shared mail destinations are denied for *owner*.
-
-    Link-local (cloud metadata), multicast, reserved and unspecified addresses
-    are always denied (src/url_safety.py). Private ranges are where a mail
-    tester or saved account becomes an internal-network probe, so they are
-    only allowed for an admin or in single-user mode (AUTH_ENABLED=false), the
-    same principals trusted with host-level tools. Operators of multi-user
-    deployments with a LAN mail server opt in with EMAIL_ALLOW_PRIVATE_IPS=true;
-    EMAIL_BLOCK_PRIVATE_IPS=true denies private ranges for everyone.
-    """
-    if os.getenv("EMAIL_BLOCK_PRIVATE_IPS", "").strip().lower() == "true":
-        return True
-    if os.getenv("EMAIL_ALLOW_PRIVATE_IPS", "").strip().lower() == "true":
-        return False
-    from src.tool_security import owner_is_admin_or_single_user
-
-    return not owner_is_admin_or_single_user((owner or "").strip() or None)
-
-
-def _mail_socket_timeout(timeout):
-    return socket._GLOBAL_DEFAULT_TIMEOUT if timeout is None else timeout
-
-
-class _PolicyIMAP4(imaplib.IMAP4):
-    """IMAP4 whose socket comes from connect_outbound_tcp.
-
-    The mail host is resolved once and connected to only at the addresses the
-    address policy approved, so DNS rebinding between check and connect is
-    impossible. STARTTLS still verifies against ``self.host``.
-    """
-
-    def __init__(self, host, port, *, block_private: bool, timeout=None):
-        self._block_private = block_private
-        super().__init__(host, port, timeout=timeout)
-
-    def _create_socket(self, timeout):
-        if timeout is not None and not timeout:
-            raise ValueError("Non-blocking socket (timeout=0) is not supported")
-        return connect_outbound_tcp(
-            self.host, self.port, timeout=_mail_socket_timeout(timeout), block_private=self._block_private,
-        )
-
-
-class _PolicyIMAP4_SSL(imaplib.IMAP4_SSL):
-    """IMAP4_SSL over a policy-checked socket; TLS SNI/verification use the hostname."""
-
-    def __init__(self, host, port, *, block_private: bool, timeout=None, ssl_context=None):
-        self._block_private = block_private
-        super().__init__(host, port, ssl_context=ssl_context, timeout=timeout)
-
-    def _create_socket(self, timeout):
-        sock = _PolicyIMAP4._create_socket(self, timeout)
-        return self.ssl_context.wrap_socket(sock, server_hostname=self.host)
-
-
-class _PolicySMTP(smtplib.SMTP):
-    """SMTP whose socket comes from connect_outbound_tcp (see _PolicyIMAP4)."""
-
-    def __init__(self, host="", port=0, *, block_private: bool, **kwargs):
-        self._block_private = block_private
-        super().__init__(host, port, **kwargs)
-
-    def _get_socket(self, host, port, timeout):
-        if timeout is not None and not timeout:
-            raise ValueError("Non-blocking socket (timeout=0) is not supported")
-        return connect_outbound_tcp(
-            host, port, timeout=timeout, block_private=self._block_private,
-            source_address=self.source_address,
-        )
-
-
-class _PolicySMTP_SSL(smtplib.SMTP_SSL):
-    """SMTP_SSL over a policy-checked socket; TLS SNI/verification use the hostname."""
-
-    def __init__(self, host="", port=0, *, block_private: bool, **kwargs):
-        self._block_private = block_private
-        super().__init__(host, port, **kwargs)
-
-    def _get_socket(self, host, port, timeout):
-        sock = _PolicySMTP._get_socket(self, host, port, timeout)
-        return self.context.wrap_socket(sock, server_hostname=self._host)
-
-
 def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message: str | bytes, timeout: int = 30) -> None:
     """Send through SMTP using the configured transport security mode."""
     host = cfg["smtp_host"]
@@ -269,15 +178,14 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message
             smtp.login(user, password)
 
     security = _smtp_security_mode(cfg)
-    block_private = _mail_private_blocked(cfg.get("owner"))
 
     if security == "ssl":
-        with _PolicySMTP_SSL(host, port, timeout=timeout, block_private=block_private) as smtp:
+        with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp:
             _auth_smtp(smtp)
             smtp.sendmail(from_addr, recipients, message)
         return
 
-    with _PolicySMTP(host, port, timeout=timeout, block_private=block_private) as smtp:
+    with smtplib.SMTP(host, port, timeout=timeout) as smtp:
         if security == "starttls":
             smtp.starttls()
         _auth_smtp(smtp)
@@ -316,39 +224,6 @@ def _friendly_email_auth_error(protocol: str, host: str, error: object) -> str:
     return raw[:200]
 
 
-# Bound at import: callers (and tests) may swap out imaplib.IMAP4 itself.
-_IMAP_ABORT = imaplib.IMAP4.abort
-
-
-def _mail_connection_test_error(protocol: str, host: str, error: BaseException) -> str:
-    """User-facing result for a failed IMAP/SMTP connection test.
-
-    Transport failures are reported by category, never with the peer's own
-    bytes: a non-mail service answering on the chosen host/port would
-    otherwise have its banner echoed back (an internal-service fingerprinting
-    oracle). Errors from a server that does speak the protocol (login/auth
-    rejections) keep their text via _friendly_email_auth_error, since that is
-    what the user needs to fix their settings.
-    """
-    if isinstance(error, MailServerAddressBlocked):
-        return f"{protocol} server address is not allowed by this server's network policy"
-    if isinstance(error, (_IMAP_ABORT, smtplib.SMTPConnectError, smtplib.SMTPServerDisconnected)):
-        return f"{protocol} server did not respond like an {protocol} server"
-    if isinstance(error, ssl.SSLCertVerificationError):
-        return f"{protocol} TLS certificate verification failed"
-    if isinstance(error, ssl.SSLError):
-        return f"{protocol} TLS handshake failed; check the port and security setting"
-    if isinstance(error, (socket.timeout, TimeoutError)):
-        return f"{protocol} connection timed out"
-    if isinstance(error, ConnectionRefusedError):
-        return f"{protocol} connection refused"
-    if isinstance(error, socket.gaierror):
-        return f"{protocol} server name could not be resolved"
-    if isinstance(error, OSError) and not isinstance(error, smtplib.SMTPException):
-        return f"{protocol} connection failed"
-    return _friendly_email_auth_error(protocol, host, error)
-
-
 def _strip_think(text: str) -> str:
     """Email-flavored think strip — thin wrapper over the central helper.
 
@@ -810,21 +685,18 @@ def _ensure_sender_signatures_table(conn):
         _lg.getLogger(__name__).warning(f"sender_signatures owner-migration skipped: {_mig_e}")
 
 
-def attachment_extract_dir(folder: str, uid: str, *, owner: str, account_id: str | None) -> Path:
-    """Containment-safe extraction directory for one message's attachments.
+def attachment_extract_dir(folder: str, uid: str) -> Path:
+    """Containment-safe extraction directory for an attachment.
 
-    IMAP UIDs are small per-mailbox counters, so `folder_uid` alone put every
-    user's and every account's "INBOX 42" in one directory, where extractions
-    overwrote each other by filename (a path handed to one user's agent could
-    then hold another user's attachment). See src.mail_attachment_paths: the
-    directory is keyed by (owner, account, folder, uid) and contained in
-    ATTACHMENTS_DIR whatever `folder`/`uid` hold."""
-    from src.mail_attachment_paths import attachment_scope_dir
-
-    try:
-        return attachment_scope_dir(ATTACHMENTS_DIR, folder, uid, owner=owner, account_id=account_id)
-    except ValueError:
+    `folder` and `uid` are user-controlled (query/path params). Flatten them to
+    a single safe path segment so a value like folder='../../tmp' can't escape
+    ATTACHMENTS_DIR, then assert containment as belt-and-suspenders."""
+    key = re.sub(r"[^A-Za-z0-9._-]", "_", f"{folder}_{uid}") or "_"
+    target = (ATTACHMENTS_DIR / key).resolve()
+    base = ATTACHMENTS_DIR.resolve()
+    if target != base and base not in target.parents:
         raise HTTPException(400, "Invalid attachment location")
+    return target
 
 
 def _init_scheduled_db():
@@ -1205,8 +1077,6 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
                 cfg = {
                     "account_id": row.id,
                     "account_name": row.name,
-                    # Principal for the outbound mail address policy.
-                    "owner": row.owner or owner or "",
                     "smtp_host": row.smtp_host or "",
                     "smtp_port": int(row.smtp_port or 465),
                     "smtp_security": _smtp_security_mode({"smtp_security": getattr(row, "smtp_security", ""), "smtp_port": row.smtp_port}),
@@ -1240,7 +1110,6 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
     cfg = {
         "account_id": resolved_id,
         "account_name": "legacy",
-        "owner": owner or "",
         "smtp_host": settings.get("smtp_host", os.environ.get("SMTP_HOST", "")),
         "smtp_port": int(settings.get("smtp_port", os.environ.get("SMTP_PORT", "465")) or 465),
         "smtp_security": _smtp_security_mode({
@@ -1304,16 +1173,11 @@ def _open_imap_connection(
     starttls: bool,
     timeout: int = _IMAP_TIMEOUT_SECONDS,
     ssl_context=None,
-    owner: str | None = None,
 ):
-    """Open an IMAP connection using the configured security mode.
-
-    `owner` is the principal the destination is judged for
-    (see _mail_private_blocked)."""
+    """Open an IMAP connection using the configured security mode."""
     port = int(port or 993)
-    block_private = _mail_private_blocked(owner)
     if starttls:
-        conn = _PolicyIMAP4(host, port, timeout=timeout, block_private=block_private)
+        conn = imaplib.IMAP4(host, port, timeout=timeout)
         try:
             if ssl_context:
                 conn.starttls(ssl_context=ssl_context)
@@ -1329,9 +1193,9 @@ def _open_imap_connection(
             raise
     elif port == 993:
         kwargs = {"ssl_context": ssl_context} if ssl_context else {}
-        conn = _PolicyIMAP4_SSL(host, port, timeout=timeout, block_private=block_private, **kwargs)
+        conn = imaplib.IMAP4_SSL(host, port, timeout=timeout, **kwargs)
     else:
-        conn = _PolicyIMAP4(host, port, timeout=timeout, block_private=block_private)
+        conn = imaplib.IMAP4(host, port, timeout=timeout)
     try:
         conn.sock.settimeout(timeout)
     except Exception:
@@ -1371,7 +1235,6 @@ def _imap_connect(account_id: str | None = None, owner: str = "",
         cfg["imap_port"],
         starttls=bool(cfg.get("imap_starttls")),
         timeout=timeout,
-        owner=cfg.get("owner"),
     )
     try:
         if cfg.get("oauth_provider") == "google":

routes/email/email_routes.py

@@ -36,10 +36,10 @@ from pathlib import Path
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 
-from starlette.concurrency import run_in_threadpool
 from fastapi import APIRouter, Query, UploadFile, File, BackgroundTasks, HTTPException, Depends, Request
 from fastapi.responses import FileResponse, StreamingResponse
 from src.constants import DATA_DIR
+from src import shutdown as _shutdown
 from src.path_confinement import confine
 
 from src.llm_core import llm_call_async
@@ -61,7 +61,6 @@ from .email_helpers import (
     _fetch_sender_thread_context, _pre_retrieve_context,
     _EMAIL_REPLY_SYS_PROMPT_BASE, _POOL_HOOKS,
     _friendly_email_auth_error, _email_summary_failure_log_detail,
-    _mail_connection_test_error, _mail_private_blocked, _PolicySMTP, _PolicySMTP_SSL,
     _generate_email_summary, EMAIL_SUMMARY_ERROR_CODE, EMAIL_SUMMARY_ERROR_MESSAGE,
     SendEmailRequest, ExtractStyleRequest,
     ATTACHMENTS_DIR, COMPOSE_UPLOADS_DIR, SCHEDULED_DB,
@@ -1543,44 +1542,6 @@ def _envelope_recipients(*fields: str) -> list:
     return out
 
 
-_MD_LINK_URL_RE = re.compile(r"\((https?://)[^)\s]*")
-
-
-def _md_links_to_html(s: str) -> str:
-    """Linear ``re.sub(r"\\[([^\\]]+)\\]\\((https?://[^)\\s]+)\\)", '<a href="\\2">\\1</a>', s)``.
-
-    Every `[` before the next `]` shares that `]`, so only the first can match;
-    and when a URL runs into whitespace/end without `)`, every later `[` whose
-    `]` also lies before that stop fails the same way. Skipping those spans
-    keeps a `[[[` or `[a](http://x[a](http://x` flood O(n) instead of the
-    regex's O(n^2).
-    """
-    out: list[str] = []
-    emitted = scan = 0
-    while True:
-        open_at = s.find("[", scan)
-        if open_at < 0:
-            break
-        close_at = s.find("]", open_at + 1)
-        if close_at < 0:
-            break
-        scan = close_at + 1
-        if close_at == open_at + 1:
-            continue
-        url = _MD_LINK_URL_RE.match(s, close_at + 1)
-        if url is None:
-            continue
-        end = url.end()
-        if end < len(s) and s[end] == ")" and end > url.end(1):
-            out.append(s[emitted:open_at])
-            out.append(f'<a href="{s[url.start(1):end]}">{s[open_at + 1:close_at]}</a>')
-            emitted = scan = end + 1
-        elif end > url.end(1):
-            scan = max(scan, s.rfind("]", scan, end) + 1)
-    out.append(s[emitted:])
-    return "".join(out)
-
-
 def _md_to_email_html(text: str) -> str:
     """Render the compose markdown body to a SAFE HTML fragment for the email's
     text/html part. Everything is HTML-escaped FIRST (so a pasted <script> /
@@ -1596,7 +1557,7 @@ def _md_to_email_html(text: str) -> str:
         s = re.sub(r"~~([^~]+)~~", r"<del>\1</del>", s)
         s = re.sub(r"`([^`]+)`", r"<code>\1</code>", s)
         # links: text + http(s) url only (escape() already neutralised quotes)
-        s = _md_links_to_html(s)
+        s = re.sub(r"\[([^\]]+)\]\((https?://[^)\s]+)\)", r'<a href="\2">\1</a>', s)
         return s
 
     parts: list[str] = []
@@ -1815,13 +1776,60 @@ def setup_email_routes():
             return None
         return v[1]
 
+    # Stale-while-revalidate snapshot store: a last-good list response per key
+    # with NO TTL, so a list view can render instantly from a snapshot (or the
+    # durable index) and refresh in the background instead of blocking ~2-5 s on
+    # IMAP. Kept small and separate from the TTL cache so invalidation of the
+    # fresh cache never removes the fallback snapshot.
+    _LIST_STALE = {}
+
+    def _list_stale_get(key):
+        return _LIST_STALE.get(key)
+
+    def _list_stale_put(key, value):
+        _LIST_STALE[key] = value
+        if len(_LIST_STALE) > 96:
+            for k in list(_LIST_STALE.keys())[:-48]:
+                _LIST_STALE.pop(k, None)
+
+    _LIST_REFRESHING = set()
+
     def _list_cache_put(key, value):
         _LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
+        # Also snapshot the last-good response for stale-while-revalidate.
+        _list_stale_put(key, value)
         # Cap size
         if len(_LIST_CACHE) > 64:
             for k in list(_LIST_CACHE.keys())[:-32]:
                 _LIST_CACHE.pop(k, None)
 
+    def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner):
+        """Refresh a list snapshot off the event loop, at most once per key.
+
+        Runs the blocking IMAP fetch in a daemon thread, then repopulates both
+        the TTL cache and the stale snapshot so the next list view is warm.
+        """
+        if cache_key in _LIST_REFRESHING:
+            return
+        _LIST_REFRESHING.add(cache_key)
+
+        def _run():
+            try:
+                refreshed = _list_emails_sync(*sync_args)
+                if refreshed and not refreshed.get("error"):
+                    _list_cache_put(cache_key, refreshed)
+                    try:
+                        _schedule_recent_email_warm(
+                            refreshed.get("emails") or [], folder, account_id, owner)
+                    except Exception:
+                        pass
+            except Exception as exc:  # pragma: no cover - background best effort
+                logger.debug("email list background refresh failed: %s", exc)
+            finally:
+                _LIST_REFRESHING.discard(cache_key)
+
+        _threading.Thread(target=_run, daemon=True, name="email-list-refresh").start()
+
     def _folder_cache_get(account_id, owner):
         key = (account_id or "", owner or "")
         v = _FOLDER_CACHE.get(key)
@@ -2848,6 +2856,46 @@ def setup_email_routes():
                         owner, account_id or "", folder, filter, limit, offset, elapsed_ms,
                     )
                 return cached
+        # Stale-while-revalidate: never block a plain list view on IMAP while
+        # we already hold a recent snapshot or a durable index. Serve it now
+        # with a `stale` marker and refresh in the background so the next view
+        # is warm. Blocking IMAP only remains for a genuine cold start with no
+        # snapshot and no indexed rows (e.g. a brand-new account/folder).
+        if not from_addr and not has_attachments and not (date_from or date_to):
+            snapshot = _list_stale_get(ck)
+            if snapshot is None:
+                try:
+                    indexed_emails, indexed_total, indexed_at = _email_index_list(
+                        owner, account_id, folder, filter, limit, offset, False,
+                    )
+                except Exception:
+                    indexed_emails, indexed_total, indexed_at = [], 0, None
+                if indexed_total:
+                    snapshot = {
+                        "emails": indexed_emails,
+                        "total": indexed_total,
+                        "folder": folder,
+                        "offset": offset,
+                        "sync": {
+                            "source": "index",
+                            "indexed": indexed_total,
+                            "updated_at": indexed_at,
+                        },
+                    }
+            if snapshot is not None:
+                payload = dict(snapshot)
+                payload["emails"] = list(payload.get("emails") or [])
+                _hide_unlinked_calendar_tags(payload["emails"])
+                sync_meta = dict(payload.get("sync") or {})
+                sync_meta["source"] = "cache_stale" if _list_stale_get(ck) is not None else "index"
+                sync_meta["stale"] = True
+                payload["sync"] = sync_meta
+                _schedule_list_refresh(
+                    (folder, limit, offset, filter, account_id, from_addr,
+                     bool(has_attachments), owner, True, date_from or "", date_to or ""),
+                    ck, folder, account_id, owner,
+                )
+                return payload
         result = await _asyncio.to_thread(
             _list_emails_sync, folder, limit, offset, filter, account_id, from_addr,
             bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "",
@@ -3964,7 +4012,7 @@ def setup_email_routes():
             _row, att = fixture_att
             filename = str(att.get("filename") or f"attachment-{index}.txt")
             safe_name = re.sub(r"[^\w\s\-.]", "_", filename).strip() or f"attachment-{index}.txt"
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             target_dir.mkdir(parents=True, exist_ok=True)
             filepath = target_dir / safe_name
             filepath.write_bytes(str(att.get("content") or "").encode("utf-8"))
@@ -3983,7 +4031,7 @@ def setup_email_routes():
             msg = email_mod.message_from_bytes(raw)
 
             # Extract to a per-email folder
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             filepath = _extract_attachment_to_disk(msg, index, target_dir)
             if not filepath:
                 return {"error": f"Attachment index {index} not found"}
@@ -4015,7 +4063,7 @@ def setup_email_routes():
             if not attachments:
                 raise HTTPException(status_code=404, detail="No downloadable attachments")
 
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             zip_buf = io.BytesIO()
             used_names: dict[str, int] = {}
             with zipfile.ZipFile(zip_buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
@@ -4155,7 +4203,7 @@ def setup_email_routes():
                                     att for att in _list_attachments_from_msg(msg)
                                     if not _is_likely_signature_image_attachment(att)
                                 ]
-                                target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=acct)
+                                target_dir = attachment_extract_dir(folder, uid)
                                 for att in attachments:
                                     idx = att.get("index")
                                     if idx is None:
@@ -4220,7 +4268,7 @@ def setup_email_routes():
                     continue
                 if not ct.lower().startswith("image/"):
                     raise HTTPException(status_code=415, detail="Content-ID is not an image")
-                target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+                target_dir = attachment_extract_dir(folder, uid)
                 filepath = _extract_attachment_to_disk(msg, idx, target_dir)
                 if not filepath:
                     raise HTTPException(status_code=404, detail="Inline image not found")
@@ -4258,7 +4306,7 @@ def setup_email_routes():
             raw = msg_data[0][1]
             msg = email_mod.message_from_bytes(raw)
 
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             filepath = _extract_attachment_to_disk(msg, index, target_dir)
             if not filepath:
                 return {"error": f"Attachment index {index} not found"}
@@ -4586,9 +4634,7 @@ def setup_email_routes():
                 try:
                     content = filepath.read_text(encoding="utf-8", errors="replace")
                 except Exception as e:
-                    # OSError text includes the server-side extraction path.
-                    logger.warning("Failed to read text attachment %s: %s", base, e)
-                    return {"error": "Failed to read text file", "filename": base}
+                    return {"error": f"Failed to read text file: {e}", "filename": base}
                 doc_id = _create_markdown_doc(content, "Imported from email attachment")
                 return {"doc_id": doc_id, "filename": filepath.name}
 
@@ -4605,7 +4651,7 @@ def setup_email_routes():
             _row, att = fixture_att
             filename = str(att.get("filename") or f"attachment-{index}.txt")
             safe_name = re.sub(r"[^\w\s\-.]", "_", filename).strip() or f"attachment-{index}.txt"
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             target_dir.mkdir(parents=True, exist_ok=True)
             filepath = target_dir / safe_name
             filepath.write_bytes(str(att.get("content") or "").encode("utf-8"))
@@ -4619,7 +4665,7 @@ def setup_email_routes():
             raw = msg_data[0][1]
             msg = email_mod.message_from_bytes(raw)
 
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             filepath = _extract_attachment_to_disk(msg, index, target_dir)
             if not filepath:
                 return {"error": f"Attachment index {index} not found"}
@@ -5273,7 +5319,7 @@ def setup_email_routes():
                 return {"success": False, "error": "Email not found"}
             raw = msg_data[0][1]
             msg = email_mod.message_from_bytes(raw)
-            target_dir = attachment_extract_dir(folder, uid, owner=owner, account_id=account_id)
+            target_dir = attachment_extract_dir(folder, uid)
             filepath = _extract_attachment_to_disk(msg, index, target_dir)
             if not filepath:
                 return {"success": False, "error": f"Attachment index {index} not found"}
@@ -6290,7 +6336,7 @@ def setup_email_routes():
                     await queue.put({'type': 'result', **result})
                 task = asyncio.create_task(run())
                 try:
-                    while True:
+                    while not _shutdown.is_shutting_down():
                         item = await queue.get()
                         yield 'data: ' + json.dumps(item) + '\n\n'
                         if item.get('type') == 'result':
@@ -6980,158 +7026,6 @@ def setup_email_routes():
         finally:
             db.close()
 
-    def _test_mail_connections(body: dict, owner: str) -> dict:
-        """Blocking IMAP/SMTP connection test for /accounts/test.
-
-        Runs in the threadpool: each probe can block for its full socket
-        timeout, which on the event loop stalled every other request. The
-        connections go through the outbound mail address policy for `owner`
-        (the user who chose the hosts)."""
-        imap_result = {"ok": False}
-        smtp_result = None
-
-        imap_host = (body.get("imap_host") or "").strip()
-        imap_port, imap_port_err = _coerce_port(body.get("imap_port"), 993)
-        imap_user = (body.get("imap_user") or "").strip()
-        imap_pass = body.get("imap_password") or ""
-        imap_starttls = bool(body.get("imap_starttls"))
-        oauth_provider = body.get("oauth_provider") or ""
-
-        google_token = None
-        google_token_loaded = False
-        google_ssl_context = (
-            ssl.create_default_context()
-            if oauth_provider == "google"
-            else None
-        )
-
-        def _google_token():
-            nonlocal google_token, google_token_loaded
-            if not google_token_loaded:
-                google_token = _get_valid_google_token(body.get("account_id"), body)
-                google_token_loaded = True
-            if not google_token:
-                raise RuntimeError("Google OAuth token unavailable — reconnect the account")
-            return google_token
-
-        if imap_port_err:
-            imap_result = {"ok": False, "error": imap_port_err}
-        elif not (imap_host and imap_user and (imap_pass or oauth_provider == "google")):
-            imap_result = {"ok": False, "error": "Need IMAP host, username, and password"}
-        elif oauth_provider == "google" and _normalized_mail_host(imap_host) != _GOOGLE_OAUTH_IMAP_HOST:
-            imap_result = {"ok": False, "error": "Google OAuth IMAP requires imap.gmail.com"}
-        elif oauth_provider == "google" and not _google_oauth_imap_transport_allowed(imap_port, imap_starttls):
-            imap_result = {"ok": False, "error": "Google OAuth IMAP requires TLS on port 993 or STARTTLS on port 143"}
-        else:
-            # Connection mode resolution:
-            #   STARTTLS on  → plain IMAP4 + .starttls() (upgrade)
-            #   STARTTLS off + port 993 → IMAP4_SSL (implicit SSL, "IMAPS")
-            #   STARTTLS off + any other port → plain IMAP4 (no encryption)
-            # Without the last branch, local servers exposed on a non-993
-            # port (Dovecot on 31143, etc.) would always fail the SSL
-            # handshake because they're not actually wrapped in TLS.
-            try:
-                imap_kwargs = {
-                    "starttls": imap_starttls,
-                    "timeout": _IMAP_TIMEOUT_SECONDS,
-                    "owner": owner,
-                }
-                if google_ssl_context:
-                    imap_kwargs["ssl_context"] = google_ssl_context
-                conn = _open_imap_connection(
-                    imap_host,
-                    imap_port,
-                    **imap_kwargs,
-                )
-                try:
-                    if oauth_provider == "google":
-                        token = _google_token()
-                        conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(imap_user, token))
-                    else:
-                        conn.login(imap_user, imap_pass)
-                    imap_result = {"ok": True}
-                finally:
-                    try: conn.logout()
-                    except Exception: pass
-            except Exception as e:
-                imap_result = {"ok": False, "error": _mail_connection_test_error("IMAP", imap_host, e)}
-
-        smtp_host = (body.get("smtp_host") or "").strip()
-        smtp_port, smtp_port_err = _coerce_port(body.get("smtp_port"), 465)
-        if smtp_host and smtp_port_err:
-            smtp_result = {"ok": False, "error": smtp_port_err}
-        elif oauth_provider == "google" and smtp_host and _normalized_mail_host(smtp_host) != _GOOGLE_OAUTH_SMTP_HOST:
-            smtp_result = {"ok": False, "error": "Google OAuth SMTP requires smtp.gmail.com"}
-        elif (
-            oauth_provider == "google"
-            and smtp_host
-            and not _google_oauth_smtp_transport_allowed(
-                smtp_port,
-                _smtp_security_mode({"smtp_security": body.get("smtp_security"), "smtp_port": smtp_port}),
-            )
-        ):
-            smtp_result = {"ok": False, "error": "Google OAuth SMTP requires TLS on port 465 or STARTTLS on port 587"}
-        elif smtp_host:
-            smtp_security = _smtp_security_mode({"smtp_security": body.get("smtp_security"), "smtp_port": smtp_port})
-            smtp_user = (body.get("smtp_user") or imap_user).strip()
-            smtp_pass = body.get("smtp_password") or imap_pass
-            smtp = None
-            try:
-                if smtp_security == "ssl":
-                    smtp_kwargs = (
-                        {"context": google_ssl_context}
-                        if google_ssl_context
-                        else {}
-                    )
-                    smtp = _PolicySMTP_SSL(
-                        smtp_host,
-                        smtp_port,
-                        timeout=10,
-                        block_private=_mail_private_blocked(owner),
-                        **smtp_kwargs,
-                    )
-                else:
-                    smtp = _PolicySMTP(smtp_host, smtp_port, timeout=10, block_private=_mail_private_blocked(owner))
-                    if smtp_security == "starttls":
-                        try:
-                            if google_ssl_context:
-                                smtp.starttls(context=google_ssl_context)
-                            else:
-                                smtp.starttls()
-                        except Exception:
-                            # STARTTLS failed before the auth cleanup block.
-                            # Close the still-open plaintext socket explicitly.
-                            try:
-                                smtp.close()
-                            except Exception:
-                                pass
-                            smtp = None
-                            raise
-                if oauth_provider == "google":
-                    token = _google_token()
-                    smtp.ehlo()
-                    smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(smtp_user, token), initial_response_ok=True)
-                else:
-                    smtp.login(smtp_user, smtp_pass)
-                smtp_result = {"ok": True}
-            except Exception as e:
-                smtp_result = {"ok": False, "error": _mail_connection_test_error("SMTP", smtp_host, e)}
-            finally:
-                if smtp is not None:
-                    try:
-                        smtp.quit()
-                    except Exception:
-                        try:
-                            smtp.close()
-                        except Exception:
-                            pass
-
-        return {
-            "ok": imap_result["ok"] and (smtp_result is None or smtp_result["ok"]),
-            "imap": imap_result,
-            "smtp": smtp_result,
-        }
-
     @router.post("/accounts/test")
     async def test_account_config(req: Request, owner: str = Depends(require_user)):
         """Try to actually connect to the provided IMAP (and optionally SMTP)
@@ -7195,7 +7089,148 @@ def setup_email_routes():
             # inline test payloads select the OAuth branch or supply token data.
             body = {key: value for key, value in body.items() if key not in _SERVER_OWNED_OAUTH_FIELDS}
 
-        return await run_in_threadpool(_test_mail_connections, body, owner)
+        imap_result = {"ok": False}
+        smtp_result = None
+
+        imap_host = (body.get("imap_host") or "").strip()
+        imap_port, imap_port_err = _coerce_port(body.get("imap_port"), 993)
+        imap_user = (body.get("imap_user") or "").strip()
+        imap_pass = body.get("imap_password") or ""
+        imap_starttls = bool(body.get("imap_starttls"))
+        oauth_provider = body.get("oauth_provider") or ""
+
+        google_token = None
+        google_token_loaded = False
+        google_ssl_context = (
+            ssl.create_default_context()
+            if oauth_provider == "google"
+            else None
+        )
+
+        def _google_token():
+            nonlocal google_token, google_token_loaded
+            if not google_token_loaded:
+                google_token = _get_valid_google_token(body.get("account_id"), body)
+                google_token_loaded = True
+            if not google_token:
+                raise RuntimeError("Google OAuth token unavailable — reconnect the account")
+            return google_token
+
+        if imap_port_err:
+            imap_result = {"ok": False, "error": imap_port_err}
+        elif not (imap_host and imap_user and (imap_pass or oauth_provider == "google")):
+            imap_result = {"ok": False, "error": "Need IMAP host, username, and password"}
+        elif oauth_provider == "google" and _normalized_mail_host(imap_host) != _GOOGLE_OAUTH_IMAP_HOST:
+            imap_result = {"ok": False, "error": "Google OAuth IMAP requires imap.gmail.com"}
+        elif oauth_provider == "google" and not _google_oauth_imap_transport_allowed(imap_port, imap_starttls):
+            imap_result = {"ok": False, "error": "Google OAuth IMAP requires TLS on port 993 or STARTTLS on port 143"}
+        else:
+            # Connection mode resolution:
+            #   STARTTLS on  → plain IMAP4 + .starttls() (upgrade)
+            #   STARTTLS off + port 993 → IMAP4_SSL (implicit SSL, "IMAPS")
+            #   STARTTLS off + any other port → plain IMAP4 (no encryption)
+            # Without the last branch, local servers exposed on a non-993
+            # port (Dovecot on 31143, etc.) would always fail the SSL
+            # handshake because they're not actually wrapped in TLS.
+            try:
+                imap_kwargs = {
+                    "starttls": imap_starttls,
+                    "timeout": _IMAP_TIMEOUT_SECONDS,
+                }
+                if google_ssl_context:
+                    imap_kwargs["ssl_context"] = google_ssl_context
+                conn = _open_imap_connection(
+                    imap_host,
+                    imap_port,
+                    **imap_kwargs,
+                )
+                try:
+                    if oauth_provider == "google":
+                        token = _google_token()
+                        conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(imap_user, token))
+                    else:
+                        conn.login(imap_user, imap_pass)
+                    imap_result = {"ok": True}
+                finally:
+                    try: conn.logout()
+                    except Exception: pass
+            except Exception as e:
+                imap_result = {"ok": False, "error": _friendly_email_auth_error("IMAP", imap_host, e)}
+
+        smtp_host = (body.get("smtp_host") or "").strip()
+        smtp_port, smtp_port_err = _coerce_port(body.get("smtp_port"), 465)
+        if smtp_host and smtp_port_err:
+            smtp_result = {"ok": False, "error": smtp_port_err}
+        elif oauth_provider == "google" and smtp_host and _normalized_mail_host(smtp_host) != _GOOGLE_OAUTH_SMTP_HOST:
+            smtp_result = {"ok": False, "error": "Google OAuth SMTP requires smtp.gmail.com"}
+        elif (
+            oauth_provider == "google"
+            and smtp_host
+            and not _google_oauth_smtp_transport_allowed(
+                smtp_port,
+                _smtp_security_mode({"smtp_security": body.get("smtp_security"), "smtp_port": smtp_port}),
+            )
+        ):
+            smtp_result = {"ok": False, "error": "Google OAuth SMTP requires TLS on port 465 or STARTTLS on port 587"}
+        elif smtp_host:
+            smtp_security = _smtp_security_mode({"smtp_security": body.get("smtp_security"), "smtp_port": smtp_port})
+            smtp_user = (body.get("smtp_user") or imap_user).strip()
+            smtp_pass = body.get("smtp_password") or imap_pass
+            smtp = None
+            try:
+                if smtp_security == "ssl":
+                    smtp_kwargs = (
+                        {"context": google_ssl_context}
+                        if google_ssl_context
+                        else {}
+                    )
+                    smtp = smtplib.SMTP_SSL(
+                        smtp_host,
+                        smtp_port,
+                        timeout=10,
+                        **smtp_kwargs,
+                    )
+                else:
+                    smtp = smtplib.SMTP(smtp_host, smtp_port, timeout=10)
+                    if smtp_security == "starttls":
+                        try:
+                            if google_ssl_context:
+                                smtp.starttls(context=google_ssl_context)
+                            else:
+                                smtp.starttls()
+                        except Exception:
+                            # STARTTLS failed before the auth cleanup block.
+                            # Close the still-open plaintext socket explicitly.
+                            try:
+                                smtp.close()
+                            except Exception:
+                                pass
+                            smtp = None
+                            raise
+                if oauth_provider == "google":
+                    token = _google_token()
+                    smtp.ehlo()
+                    smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(smtp_user, token), initial_response_ok=True)
+                else:
+                    smtp.login(smtp_user, smtp_pass)
+                smtp_result = {"ok": True}
+            except Exception as e:
+                smtp_result = {"ok": False, "error": _friendly_email_auth_error("SMTP", smtp_host, e)}
+            finally:
+                if smtp is not None:
+                    try:
+                        smtp.quit()
+                    except Exception:
+                        try:
+                            smtp.close()
+                        except Exception:
+                            pass
+
+        return {
+            "ok": imap_result["ok"] and (smtp_result is None or smtp_result["ok"]),
+            "imap": imap_result,
+            "smtp": smtp_result,
+        }
 
     @router.post("/accounts/{account_id}/set-default")
     async def set_default_account(account_id: str, owner: str = Depends(require_user)):

routes/hwfit_routes.py

@@ -5,9 +5,8 @@ import shlex
 import subprocess
 from copy import deepcopy
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, HTTPException
 
-from core.middleware import require_admin
 from core.platform_compat import run_ssh_command
 from routes._validators import validate_remote_host, validate_ssh_port
 
@@ -199,11 +198,7 @@ def _inspect_model_path(model_path: str, host: str = "", ssh_port: str = "") ->
 
 
 def setup_hwfit_routes():
-    # Admin-gated like the Cookbook routes this backs (/api/cookbook/gpus does
-    # the same SSH host probing): `host`/`ssh_port` make the server SSH out with
-    # its own key to any syntactically valid host, and `model_path` probes the
-    # host filesystem. Single-user mode and internal tool calls still pass.
-    router = APIRouter(prefix="/api/hwfit", tags=["hwfit"], dependencies=[Depends(require_admin)])
+    router = APIRouter(prefix="/api/hwfit", tags=["hwfit"])
 
     @router.get("/system")
     def get_system(host: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False):
@@ -223,7 +218,7 @@ def setup_hwfit_routes():
         fresh=true bypasses the hardware-detection cache."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.fit import rank_models
-        from services.hwfit.models import get_models, refresh_dynamic_catalogs
+        from services.hwfit.models import get_models, model_catalog_path, refresh_dynamic_catalogs
         host, ssh_port = _validate_detection_target(host, ssh_port)
         system = deepcopy(detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh))
         if system.get("error"):
@@ -238,8 +233,7 @@ def setup_hwfit_routes():
             return {
                 "system": system,
                 "models": [],
-                "error": "Model catalog is empty. Use Rescan in Cookbook while online to populate runtime model data; offline recommendations require a previously populated cache.",
-                **({"catalog_refresh": catalog_refresh} if catalog_refresh is not None else {}),
+                "error": f"Model catalog missing or empty: {model_catalog_path()}",
             }
 
         if ignore_detected_gpu:

routes/model_routes.py

@@ -472,7 +472,7 @@ def _truthy(value: str | None) -> bool:
 
 _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
 _REFRESH_MODES = {"auto", "manual", "disabled"}
-_MODEL_TOOL_MODES = {"none", "compact", "full"}
+_MODEL_TOOL_MODES = {"none", "compact", "full", "spec"}
 _MODEL_TOOL_MODE_ALIASES = {
     "regular": "full",
     "odysseus_compact": "compact",
@@ -552,10 +552,10 @@ def _endpoint_refresh_timeout(ep: Any, category: str) -> float:
         val = 0
     if val > 0:
         return float(max(1, min(60, val)))
-    # llama.cpp and other local OpenAI-compatible servers can block briefly
-    # while warming/loading. A 2s local timeout makes working endpoints flicker
-    # offline before /v1/models is ready.
-    return 10.0 if category == "local" else 2.0
+    # Keep the background refresh short so a single dead endpoint cannot stall
+    # the model picker. A timed-out refresh never clears a non-empty model
+    # cache, so a briefly-warming local server is simply retried next round.
+    return 2.0
 
 
 def _manual_refresh_timeout(ep: Any, category: str, requested: Any = None) -> float:
@@ -1510,6 +1510,9 @@ def setup_model_routes(model_discovery):
     _refresh_inflight = {"v": False, "done": None}  # coarse single-flight guard
     _REFRESH_FAILURE_BASE = 300.0
     _REFRESH_FAILURE_MAX = 3600.0
+    # Even a user-triggered refresh must not re-probe an endpoint that just
+    # failed/timed out on every click; keep a short cooldown.
+    _REFRESH_FAILURE_MIN = 60.0
 
     def _refresh_key(base: str, api_key: Optional[str], provider_auth_id: Optional[str] = None) -> str:
         # Session-backed endpoints carry no static key; include their auth id
@@ -1557,14 +1560,20 @@ def setup_model_routes(model_discovery):
         if mode in ("manual", "disabled") and not force:
             return False, info
         fails = int(state.get("fail_count") or 0)
-        if fails and not force:
+        if fails:
             last_failure = float(state.get("last_failure") or 0.0)
             empty_local = (
                 not cached
                 and category == "local"
                 and str(getattr(ep, "id", "") or "").startswith("local-")
             )
-            if now - last_failure < _failure_delay(fails, empty_local=empty_local):
+            delay = _failure_delay(fails, empty_local=empty_local)
+            if force:
+                # A user-triggered refresh gets a short cooldown instead of the
+                # long background backoff, but still skips a just-failed
+                # endpoint so repeated clicks cannot stall the picker.
+                delay = min(delay, _REFRESH_FAILURE_MIN)
+            if now - last_failure < delay:
                 return False, info
         if cached and not force:
             interval = _endpoint_refresh_interval(ep, category)
@@ -3055,4 +3064,10 @@ def setup_model_routes(model_discovery):
 
     router._should_refresh_endpoint = _should_refresh_endpoint
     router._search_endpoint_catalog = search_endpoint_catalog
+    # Exposed for tests: the in-memory backoff state, its key function, and the
+    # background refresh driver (probes endpoints in parallel with the short
+    # per-endpoint timeout).
+    router._refresh_state = _refresh_state
+    router._refresh_key = _refresh_key
+    router._refresh_caches_bg = _refresh_caches_bg
     return router