feat(security): apply audited PR #6738 remediations and test hardening

- Enforce vision probe SSRF safe local host validation, SafeProbeBackend TOCTOU elimination, and session owner authorization
- Authorize session titling endpoints against registered owner configuration
- Sanitize sensitive metadata logging from agent loop debug logs
- Add async non-blocking execution for spec router tools and thinking decisions
- Harden email cache invalidation fencing and thread-safe mutation
- Add regression test suites for legacy browser authority, spec contracts, spec router request scoping, and task MCP delivery
This commit is contained in:
Alexandre Teixeira
2026-10-27 17:25:25 +00:00
parent 3246ae5d9c
commit 0133de6f0e
43 changed files with 2187 additions and 673 deletions
+79 -21
View File
@@ -583,15 +583,25 @@ async def auto_name_session(session_manager, sess):
except Exception:
_is_spec_model = False
if _is_spec_model and endpoint_url:
_headers = getattr(sess, "headers", None)
if isinstance(_headers, str):
try:
_headers = json.loads(_headers)
except Exception:
_headers = None
try:
from src.database import SessionLocal
from src.endpoint_resolver import (build_chat_url, build_headers,
resolve_endpoint_runtime, resolve_owner_registered_endpoint)
from src.owner_identity import auth_disabled
owner = getattr(sess, "owner", None)
if not owner and not auth_disabled():
raise ValueError("Session has no owner")
db = SessionLocal()
try:
endpoint = resolve_owner_registered_endpoint(db, endpoint_url, owner)
base, key = resolve_endpoint_runtime(endpoint, owner=owner)
title_url = build_chat_url(base)
title_headers = build_headers(key, base)
finally:
db.close()
title = await llm_call_async(
endpoint_url,
title_url,
model_name,
[
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation \u2014 just the title."},
@@ -599,7 +609,7 @@ async def auto_name_session(session_manager, sess):
],
temperature=0.3,
max_tokens=32,
headers=_headers or None,
headers=title_headers,
timeout=20,
thinking_mode="off",
)
@@ -625,6 +635,14 @@ async def auto_name_session(session_manager, sess):
return
owner = getattr(sess, "owner", None)
from src.owner_identity import auth_disabled
if not owner and not auth_disabled():
title = fallback_session_title(first_msg)
session_manager.update_session_name(sess.id, title)
logger.info(f"Auto-named unowned session {sess.id} deterministically: {title}")
return
from src.task_endpoint import resolve_task_endpoint
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=owner
)
@@ -632,24 +650,64 @@ async def auto_name_session(session_manager, sess):
logger.debug("[auto-name] No model provided, skipping")
return
sess_url = getattr(sess, "endpoint_url", None)
if t_url and sess_url and t_url == sess_url:
from src.database import SessionLocal
from src.endpoint_resolver import (
build_chat_url,
build_headers,
resolve_endpoint,
resolve_endpoint_runtime,
resolve_owner_registered_endpoint,
)
db = SessionLocal()
try:
endpoint = resolve_owner_registered_endpoint(db, t_url, owner)
base, key = resolve_endpoint_runtime(endpoint, owner=owner)
t_url = build_chat_url(base)
t_headers = build_headers(key, base)
except Exception as e:
logger.debug("[auto-name] session fallback endpoint %s unauthorized for %s: %s", t_url, owner, e)
t_url, t_model, t_headers = None, None, None
for prefix in ("utility", "default"):
try:
u, m, h = resolve_endpoint(prefix, owner=owner)
if u and m:
t_url, t_model, t_headers = u, m, h
break
except Exception:
pass
finally:
db.close()
if not t_url or not t_model:
title = fallback_session_title(first_msg)
session_manager.update_session_name(sess.id, title)
logger.info(f"Auto-named session {sess.id} deterministically (no authorized model endpoint): {title}")
return
# max_tokens big enough that reasoning models (Minimax M2,
# DeepSeek R1, QwQ, etc.) have headroom for <think>…</think>
# plus the actual title — 200 used to clip them mid-reasoning
# so strip_think left an empty string and no rename happened.
# Timeout matches: 60s gives slow local reasoners room to finish.
title = await llm_call_async(
t_url,
t_model,
[
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."},
{"role": "user", "content": first_msg},
],
temperature=0.3,
max_tokens=64,
headers=t_headers,
timeout=15,
reasoning_effort=effort_for_call(t_url, t_model, owner, sess),
)
try:
title = await llm_call_async(
t_url,
t_model,
[
{"role": "system", "content": "Generate a short title (3-6 words, no quotes) for a conversation that starts with this message. Reply with ONLY the title, nothing else. Do NOT include any thinking, reasoning, or explanation — just the title."},
{"role": "user", "content": first_msg},
],
temperature=0.3,
max_tokens=64,
headers=t_headers,
timeout=15,
reasoning_effort=effort_for_call(t_url, t_model, owner, sess),
)
except Exception as e:
logger.warning("[auto-name] non-spec title call failed for %s: %s", sess.id, e)
title = ""
title = title.strip().strip('"\'').strip()
# Strip <think>/<thinking> blocks (closed, dangling, or stray tags)
+6 -1
View File
@@ -331,6 +331,7 @@ def _is_external_discovery_request(text: str) -> bool:
_URL_SCHEME_RE = re.compile(r"https?://", re.I)
_PDF_URL_TAIL_RE = re.compile(r"\.pdf\b|/pdf/", re.I)
_EDITOR_SESSION_RE = re.compile(r"[A-Za-z0-9@._:-]{1,128}")
def _mentions_pdf_url(value: str) -> bool:
@@ -2693,11 +2694,15 @@ def setup_chat_routes(
# Scoped photo_editor agent context: the Quick Edit popup (and any
# image-edit chat turn) may carry the open image-editor session id and a
# canvas snapshot so the editor tool family can drive it in-process.
editor_session = (
# The id is quoted into a system message, so accept only an
# identifier; any other value is treated as no open editor.
editor_session = str(
form_data.get("editor_session")
or form_data.get("editor_session_id")
or ""
).strip()
if not _EDITOR_SESSION_RE.fullmatch(editor_session):
editor_session = ""
photo_editor_requested = (
str(form_data.get("photo_editor") or "").strip().lower()
in {"1", "true", "yes", "on"}
+73 -67
View File
@@ -1836,6 +1836,9 @@ def setup_email_routes():
return v[1]
_LIST_STALE = {}
_LIST_CACHE_LOCK = _threading.RLock()
# Invalidation fences reads already in flight, including background refresh.
_LIST_CACHE_GENERATION = [0]
def _list_stale_get(key):
return _LIST_STALE.get(key)
@@ -1849,13 +1852,12 @@ def setup_email_routes():
_LIST_REFRESHING = set()
def _list_cache_put(key, value):
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
# Also snapshot the last-good response for stale-while-revalidate.
_list_stale_put(key, value)
# Cap size
if len(_LIST_CACHE) > 64:
for k in list(_LIST_CACHE.keys())[:-32]:
_LIST_CACHE.pop(k, None)
with _LIST_CACHE_LOCK:
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
_list_stale_put(key, value)
if len(_LIST_CACHE) > 64:
for k in list(_LIST_CACHE.keys())[:-32]:
_LIST_CACHE.pop(k, None)
def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner):
"""Refresh a list snapshot off the event loop, at most once per key.
@@ -1863,15 +1865,20 @@ def setup_email_routes():
Runs the blocking IMAP fetch in a daemon thread, then repopulates both
the TTL cache and the stale snapshot so the next list view is warm.
"""
if cache_key in _LIST_REFRESHING:
return
_LIST_REFRESHING.add(cache_key)
with _LIST_CACHE_LOCK:
if cache_key in _LIST_REFRESHING:
return
_LIST_REFRESHING.add(cache_key)
generation = _LIST_CACHE_GENERATION[0]
def _run():
try:
refreshed = _list_emails_sync(*sync_args)
if refreshed and not refreshed.get("error"):
_list_cache_put(cache_key, refreshed)
with _LIST_CACHE_LOCK:
if generation != _LIST_CACHE_GENERATION[0]:
return
_list_cache_put(cache_key, refreshed)
try:
_schedule_recent_email_warm(
refreshed.get("emails") or [], folder, account_id, owner)
@@ -1880,7 +1887,8 @@ def setup_email_routes():
except Exception as exc: # pragma: no cover - background best effort
logger.debug("email list background refresh failed: %s", exc)
finally:
_LIST_REFRESHING.discard(cache_key)
with _LIST_CACHE_LOCK:
_LIST_REFRESHING.discard(cache_key)
_threading.Thread(target=_run, daemon=True, name="email-list-refresh").start()
@@ -1917,37 +1925,25 @@ def setup_email_routes():
hook(account_id, folder)
except Exception:
pass
if account_id is None and folder is None:
_LIST_CACHE.clear()
return
for k in list(_LIST_CACHE.keys()):
k_acct = k[0] if len(k) > 0 else ""
k_folder = k[1] if len(k) > 1 else ""
if (account_id is None or k_acct == (account_id or "")) and \
(folder is None or k_folder == folder):
_LIST_CACHE.pop(k, None)
with _LIST_CACHE_LOCK:
_LIST_CACHE_GENERATION[0] += 1
for cache in (_LIST_CACHE, _LIST_STALE):
for k in list(cache):
k_acct = k[0] if len(k) > 0 else ""
k_folder = k[1] if len(k) > 1 else ""
if (account_id is None or k_acct == (account_id or "")) and \
(folder is None or k_folder == folder):
cache.pop(k, None)
def _update_list_cache_seen(account_id, folder, uid, seen: bool):
uid_s = str(uid)
for key, (expires_at, value) in list(_LIST_CACHE.items()):
if key[0] != (account_id or "") or key[1] != folder:
continue
emails = list((value or {}).get("emails") or [])
changed = False
if seen and key[2] == "unread":
kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s]
if len(kept) != len(emails):
value = dict(value)
value["emails"] = kept
value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept)))
changed = True
else:
for e in emails:
if str((e or {}).get("uid") or "") == uid_s:
e["is_read"] = bool(seen)
changed = True
if changed:
_LIST_CACHE[key] = (expires_at, value)
def _index_update_flags(owner, account_id, folder, uid, flag, add):
with _LIST_CACHE_LOCK:
_email_index_update_flags(owner, account_id, folder, uid, flag, add)
_invalidate_list_cache(account_id, folder)
def _index_delete(owner, account_id, folder, uid):
with _LIST_CACHE_LOCK:
_email_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id, folder)
def _read_cache_get(key):
v = _READ_CACHE.get(key)
@@ -2327,6 +2323,8 @@ def setup_email_routes():
SECURITY: `owner` is propagated so when `account_id` is missing,
the fallback config lookup is scoped to this user's accounts only.
"""
with _LIST_CACHE_LOCK:
index_generation = _LIST_CACHE_GENERATION[0]
conn = None
conn_ok = False
try:
@@ -2621,7 +2619,9 @@ def setup_email_routes():
if parsed:
fetched_emails.append(parsed)
cached_rows[parsed["uid"]] = parsed
_email_index_upsert(owner, account_id, folder, fetched_emails)
with _LIST_CACHE_LOCK:
if index_generation == _LIST_CACHE_GENERATION[0]:
_email_index_upsert(owner, account_id, folder, fetched_emails)
for uid_num in uid_order:
row = cached_rows.get(uid_num)
if not row:
@@ -2915,7 +2915,7 @@ def setup_email_routes():
# with a `stale` marker and refresh in the background so the next view
# is warm. Blocking IMAP only remains for a genuine cold start with no
# snapshot and no indexed rows (e.g. a brand-new account/folder).
if not from_addr and not has_attachments and not (date_from or date_to):
if not manual_refresh and not from_addr and not has_attachments and not (date_from or date_to):
snapshot = _list_stale_get(ck)
if snapshot is None:
try:
@@ -2950,6 +2950,8 @@ def setup_email_routes():
ck, folder, account_id, owner,
)
return payload
with _LIST_CACHE_LOCK:
cache_generation = _LIST_CACHE_GENERATION[0]
result = await _asyncio.to_thread(
_list_emails_sync, folder, limit, offset, filter, account_id, from_addr,
bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "",
@@ -2958,7 +2960,9 @@ def setup_email_routes():
if offset == 0 and not from_addr and not has_attachments and filter in ("all", "unread", "unanswered", "undone"):
_record_email_received_events(owner, account_id, folder, result.get("emails") or [])
_schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner)
_list_cache_put(ck, result)
with _LIST_CACHE_LOCK:
if cache_generation == _LIST_CACHE_GENERATION[0]:
_list_cache_put(ck, result)
elapsed_ms = int((_time.monotonic() - started_at) * 1000)
if elapsed_ms > 1500:
logger.warning(
@@ -3306,7 +3310,7 @@ def setup_email_routes():
conn.select(_q(folder))
moved = _move_email_message(conn, uid, "Junk", role="junk")
if moved:
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id)
except Exception:
logger.debug("unsubscribe move-to-spam skipped", exc_info=True)
@@ -3323,7 +3327,7 @@ def setup_email_routes():
if deleted:
conn.expunge()
if deleted:
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id)
except Exception as exc:
delete_error = "Email was unsubscribed but could not be moved to Trash"
@@ -3405,7 +3409,7 @@ def setup_email_routes():
if moved:
changed += 1
cleaned_uids.append(uid)
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
else:
failed += 1
except Exception:
@@ -3494,6 +3498,8 @@ def setup_email_routes():
folder (Gmail does), we transparently swap to All Mail so the
search surfaces archived / labelled emails too. Plain IMAP
accounts fall back to whatever folder the caller specified."""
with _LIST_CACHE_LOCK:
index_generation = _LIST_CACHE_GENERATION[0]
if not q or len(q) < 2:
return {"emails": [], "total": 0, "query": q}
# CRLF in q would terminate the IMAP command early — reject defensively.
@@ -3637,7 +3643,9 @@ def setup_email_routes():
except Exception as e:
logger.warning(f"Error parsing search result {uid}: {e}")
continue
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
with _LIST_CACHE_LOCK:
if index_generation == _LIST_CACHE_GENERATION[0]:
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
return {
"emails": emails,
@@ -3774,8 +3782,7 @@ def setup_email_routes():
# accepted it, so the index and list cache cannot drift ahead of
# the mailbox.
if mark_seen and not mark_seen_failed:
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
_update_list_cache_seen(account_id, folder, uid, True)
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
related_attachments = []
if full and not _has_visible_attachments(msg):
@@ -3914,8 +3921,7 @@ def setup_email_routes():
status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)")
if status != "OK":
return False
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
_update_list_cache_seen(account_id, folder, uid, True)
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
return True
except Exception as e:
logger.warning(f"mark-seen after cached read failed uid={uid}: {e}")
@@ -5147,7 +5153,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _store_email_flag(conn, uid, "\\Seen", add=False):
return {"success": False, "error": "Email not found"}
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
_invalidate_list_cache(account_id, folder)
return {"success": True}
except Exception as e:
@@ -5167,7 +5173,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)):
return {"success": False, "error": "Email not found"}
_email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
_invalidate_list_cache(account_id, folder)
return {"success": True, "flagged": bool(on)}
except Exception as e:
@@ -5185,7 +5191,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _store_email_flag(conn, uid, "\\Seen", add=True):
return {"success": False, "error": "Email not found"}
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
_invalidate_list_cache(account_id, folder)
return {"success": True}
except Exception as e:
@@ -5205,7 +5211,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _move_email_message(conn, uid, "Archive", role="archive"):
return {"success": False, "error": "Email not found"}
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id)
return {"success": True}
except Exception as e:
@@ -5236,7 +5242,7 @@ def setup_email_routes():
# click or by another mailbox client; it is already gone
# from the requested folder, so do not trap the UI on a
# permanent “Email not found” error.
_email_index_delete(owner, account_id, folder, str(uid))
_index_delete(owner, account_id, folder, str(uid))
_invalidate_list_cache(account_id, folder)
return {"success": True, "already_deleted": True}
trash_folder = _resolve_mail_folder(conn, "Trash", "trash")
@@ -5254,9 +5260,9 @@ def setup_email_routes():
if not _move_email_message(conn, resolved_uid, trash_folder, role="trash"):
logger.warning("Email delete Trash move failed uid=%s resolved_uid=%s folder=%s trash=%s", uid, resolved_uid, folder, trash_folder)
return {"success": False, "error": "Could not move email to Trash"}
_email_index_delete(owner, account_id, folder, resolved_uid)
_index_delete(owner, account_id, folder, resolved_uid)
if resolved_uid != str(uid):
_email_index_delete(owner, account_id, folder, str(uid))
_index_delete(owner, account_id, folder, str(uid))
_invalidate_list_cache(account_id)
return {"success": True}
except Exception as e:
@@ -5272,7 +5278,7 @@ def setup_email_routes():
if not _store_email_flag(conn, uid, "\\Deleted", add=True):
return {"success": False, "error": "Email not found"}
conn.expunge()
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id, folder)
return {"success": True}
except Exception as e:
@@ -5340,7 +5346,7 @@ def setup_email_routes():
pending_expunge.append(uid)
continue
deleted_uids.append(uid)
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
except Exception:
failed_uids.append(uid)
logger.debug("Bulk email delete failed for uid=%s", uid, exc_info=True)
@@ -5362,7 +5368,7 @@ def setup_email_routes():
if expunge_ok:
for uid in pending_expunge:
deleted_uids.append(uid)
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
else:
# Prioritize preservation over cleanup: unflag \Deleted from our
# copied messages so future EXPUNGE operations do not destroy them.
@@ -5481,7 +5487,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _move_email_message(conn, uid, dest):
return {"success": False, "error": f"Failed to move to {dest}"}
_email_index_delete(owner, account_id, folder, uid)
_index_delete(owner, account_id, folder, uid)
_invalidate_list_cache(account_id)
return {"success": True}
except Exception as e:
@@ -5567,7 +5573,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _store_email_flag(conn, uid, "\\Answered", add=True):
return {"success": False, "error": "Email not found"}
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
_clear_done_response_tags(owner, account_id, folder, uid)
_invalidate_list_cache(account_id, folder)
return {"success": True}
@@ -5583,7 +5589,7 @@ def setup_email_routes():
conn.select(_q(folder))
if not _store_email_flag(conn, uid, "\\Answered", add=False):
return {"success": False, "error": "Email not found"}
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
_invalidate_list_cache(account_id, folder)
return {"success": True}
except Exception as e:
@@ -6250,7 +6256,7 @@ def setup_email_routes():
try:
st, _sel = imap.select(_q(_source_folder), readonly=False)
if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True):
_email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
_clear_done_response_tags(owner, _account_id, _source_folder, _source_uid)
_invalidate_list_cache(_account_id, _source_folder)
logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}")
+2 -7
View File
@@ -290,14 +290,9 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
# (most sessions have none), not an error. Return HTTP 200 with an
# explicit empty status so loading a session does not log a spurious
# 404; every client treats a non-"running" status as idle anyway.
# A session whose research exists but belongs to ANOTHER user stays a
# 404, so we do not leak the existence of someone else's research.
# Research owned by ANOTHER user answers identically, so the response
# does not reveal whether that research exists.
if not _owns_in_memory(session_id, user):
exists = session_id in getattr(research_handler, "_active_tasks", {})
if not exists:
exists = _find_research_path(session_id) is not None
if exists:
raise HTTPException(404, "No research found for this session")
return {"status": "none"}
status = research_handler.get_status(session_id)
if status is None: