mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-06 13:12:27 +01:00
feat(security): apply audited PR #6738 remediations and test hardening
- Enforce vision probe SSRF safe local host validation, SafeProbeBackend TOCTOU elimination, and session owner authorization - Authorize session titling endpoints against registered owner configuration - Sanitize sensitive metadata logging from agent loop debug logs - Add async non-blocking execution for spec router tools and thinking decisions - Harden email cache invalidation fencing and thread-safe mutation - Add regression test suites for legacy browser authority, spec contracts, spec router request scoping, and task MCP delivery
This commit is contained in:
Alexandre Teixeira
@@ -1836,6 +1836,9 @@ def setup_email_routes():
|
||||
return v[1]
|
||||
|
||||
_LIST_STALE = {}
|
||||
_LIST_CACHE_LOCK = _threading.RLock()
|
||||
# Invalidation fences reads already in flight, including background refresh.
|
||||
_LIST_CACHE_GENERATION = [0]
|
||||
|
||||
def _list_stale_get(key):
|
||||
return _LIST_STALE.get(key)
|
||||
@@ -1849,13 +1852,12 @@ def setup_email_routes():
|
||||
_LIST_REFRESHING = set()
|
||||
|
||||
def _list_cache_put(key, value):
|
||||
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
|
||||
# Also snapshot the last-good response for stale-while-revalidate.
|
||||
_list_stale_put(key, value)
|
||||
# Cap size
|
||||
if len(_LIST_CACHE) > 64:
|
||||
for k in list(_LIST_CACHE.keys())[:-32]:
|
||||
_LIST_CACHE.pop(k, None)
|
||||
with _LIST_CACHE_LOCK:
|
||||
_LIST_CACHE[key] = (_time.monotonic() + _LIST_TTL, value)
|
||||
_list_stale_put(key, value)
|
||||
if len(_LIST_CACHE) > 64:
|
||||
for k in list(_LIST_CACHE.keys())[:-32]:
|
||||
_LIST_CACHE.pop(k, None)
|
||||
|
||||
def _schedule_list_refresh(sync_args, cache_key, folder, account_id, owner):
|
||||
"""Refresh a list snapshot off the event loop, at most once per key.
|
||||
@@ -1863,15 +1865,20 @@ def setup_email_routes():
|
||||
Runs the blocking IMAP fetch in a daemon thread, then repopulates both
|
||||
the TTL cache and the stale snapshot so the next list view is warm.
|
||||
"""
|
||||
if cache_key in _LIST_REFRESHING:
|
||||
return
|
||||
_LIST_REFRESHING.add(cache_key)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if cache_key in _LIST_REFRESHING:
|
||||
return
|
||||
_LIST_REFRESHING.add(cache_key)
|
||||
generation = _LIST_CACHE_GENERATION[0]
|
||||
|
||||
def _run():
|
||||
try:
|
||||
refreshed = _list_emails_sync(*sync_args)
|
||||
if refreshed and not refreshed.get("error"):
|
||||
_list_cache_put(cache_key, refreshed)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if generation != _LIST_CACHE_GENERATION[0]:
|
||||
return
|
||||
_list_cache_put(cache_key, refreshed)
|
||||
try:
|
||||
_schedule_recent_email_warm(
|
||||
refreshed.get("emails") or [], folder, account_id, owner)
|
||||
@@ -1880,7 +1887,8 @@ def setup_email_routes():
|
||||
except Exception as exc: # pragma: no cover - background best effort
|
||||
logger.debug("email list background refresh failed: %s", exc)
|
||||
finally:
|
||||
_LIST_REFRESHING.discard(cache_key)
|
||||
with _LIST_CACHE_LOCK:
|
||||
_LIST_REFRESHING.discard(cache_key)
|
||||
|
||||
_threading.Thread(target=_run, daemon=True, name="email-list-refresh").start()
|
||||
|
||||
@@ -1917,37 +1925,25 @@ def setup_email_routes():
|
||||
hook(account_id, folder)
|
||||
except Exception:
|
||||
pass
|
||||
if account_id is None and folder is None:
|
||||
_LIST_CACHE.clear()
|
||||
return
|
||||
for k in list(_LIST_CACHE.keys()):
|
||||
k_acct = k[0] if len(k) > 0 else ""
|
||||
k_folder = k[1] if len(k) > 1 else ""
|
||||
if (account_id is None or k_acct == (account_id or "")) and \
|
||||
(folder is None or k_folder == folder):
|
||||
_LIST_CACHE.pop(k, None)
|
||||
with _LIST_CACHE_LOCK:
|
||||
_LIST_CACHE_GENERATION[0] += 1
|
||||
for cache in (_LIST_CACHE, _LIST_STALE):
|
||||
for k in list(cache):
|
||||
k_acct = k[0] if len(k) > 0 else ""
|
||||
k_folder = k[1] if len(k) > 1 else ""
|
||||
if (account_id is None or k_acct == (account_id or "")) and \
|
||||
(folder is None or k_folder == folder):
|
||||
cache.pop(k, None)
|
||||
|
||||
def _update_list_cache_seen(account_id, folder, uid, seen: bool):
|
||||
uid_s = str(uid)
|
||||
for key, (expires_at, value) in list(_LIST_CACHE.items()):
|
||||
if key[0] != (account_id or "") or key[1] != folder:
|
||||
continue
|
||||
emails = list((value or {}).get("emails") or [])
|
||||
changed = False
|
||||
if seen and key[2] == "unread":
|
||||
kept = [e for e in emails if str((e or {}).get("uid") or "") != uid_s]
|
||||
if len(kept) != len(emails):
|
||||
value = dict(value)
|
||||
value["emails"] = kept
|
||||
value["total"] = max(0, int(value.get("total") or 0) - (len(emails) - len(kept)))
|
||||
changed = True
|
||||
else:
|
||||
for e in emails:
|
||||
if str((e or {}).get("uid") or "") == uid_s:
|
||||
e["is_read"] = bool(seen)
|
||||
changed = True
|
||||
if changed:
|
||||
_LIST_CACHE[key] = (expires_at, value)
|
||||
def _index_update_flags(owner, account_id, folder, uid, flag, add):
|
||||
with _LIST_CACHE_LOCK:
|
||||
_email_index_update_flags(owner, account_id, folder, uid, flag, add)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
|
||||
def _index_delete(owner, account_id, folder, uid):
|
||||
with _LIST_CACHE_LOCK:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
|
||||
def _read_cache_get(key):
|
||||
v = _READ_CACHE.get(key)
|
||||
@@ -2327,6 +2323,8 @@ def setup_email_routes():
|
||||
SECURITY: `owner` is propagated so when `account_id` is missing,
|
||||
the fallback config lookup is scoped to this user's accounts only.
|
||||
"""
|
||||
with _LIST_CACHE_LOCK:
|
||||
index_generation = _LIST_CACHE_GENERATION[0]
|
||||
conn = None
|
||||
conn_ok = False
|
||||
try:
|
||||
@@ -2621,7 +2619,9 @@ def setup_email_routes():
|
||||
if parsed:
|
||||
fetched_emails.append(parsed)
|
||||
cached_rows[parsed["uid"]] = parsed
|
||||
_email_index_upsert(owner, account_id, folder, fetched_emails)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if index_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_email_index_upsert(owner, account_id, folder, fetched_emails)
|
||||
for uid_num in uid_order:
|
||||
row = cached_rows.get(uid_num)
|
||||
if not row:
|
||||
@@ -2915,7 +2915,7 @@ def setup_email_routes():
|
||||
# with a `stale` marker and refresh in the background so the next view
|
||||
# is warm. Blocking IMAP only remains for a genuine cold start with no
|
||||
# snapshot and no indexed rows (e.g. a brand-new account/folder).
|
||||
if not from_addr and not has_attachments and not (date_from or date_to):
|
||||
if not manual_refresh and not from_addr and not has_attachments and not (date_from or date_to):
|
||||
snapshot = _list_stale_get(ck)
|
||||
if snapshot is None:
|
||||
try:
|
||||
@@ -2950,6 +2950,8 @@ def setup_email_routes():
|
||||
ck, folder, account_id, owner,
|
||||
)
|
||||
return payload
|
||||
with _LIST_CACHE_LOCK:
|
||||
cache_generation = _LIST_CACHE_GENERATION[0]
|
||||
result = await _asyncio.to_thread(
|
||||
_list_emails_sync, folder, limit, offset, filter, account_id, from_addr,
|
||||
bool(has_attachments), owner, manual_refresh, date_from or "", date_to or "",
|
||||
@@ -2958,7 +2960,9 @@ def setup_email_routes():
|
||||
if offset == 0 and not from_addr and not has_attachments and filter in ("all", "unread", "unanswered", "undone"):
|
||||
_record_email_received_events(owner, account_id, folder, result.get("emails") or [])
|
||||
_schedule_recent_email_warm(result.get("emails") or [], folder, account_id, owner)
|
||||
_list_cache_put(ck, result)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if cache_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_list_cache_put(ck, result)
|
||||
elapsed_ms = int((_time.monotonic() - started_at) * 1000)
|
||||
if elapsed_ms > 1500:
|
||||
logger.warning(
|
||||
@@ -3306,7 +3310,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
moved = _move_email_message(conn, uid, "Junk", role="junk")
|
||||
if moved:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
except Exception:
|
||||
logger.debug("unsubscribe move-to-spam skipped", exc_info=True)
|
||||
@@ -3323,7 +3327,7 @@ def setup_email_routes():
|
||||
if deleted:
|
||||
conn.expunge()
|
||||
if deleted:
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
except Exception as exc:
|
||||
delete_error = "Email was unsubscribed but could not be moved to Trash"
|
||||
@@ -3405,7 +3409,7 @@ def setup_email_routes():
|
||||
if moved:
|
||||
changed += 1
|
||||
cleaned_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
else:
|
||||
failed += 1
|
||||
except Exception:
|
||||
@@ -3494,6 +3498,8 @@ def setup_email_routes():
|
||||
folder (Gmail does), we transparently swap to All Mail so the
|
||||
search surfaces archived / labelled emails too. Plain IMAP
|
||||
accounts fall back to whatever folder the caller specified."""
|
||||
with _LIST_CACHE_LOCK:
|
||||
index_generation = _LIST_CACHE_GENERATION[0]
|
||||
if not q or len(q) < 2:
|
||||
return {"emails": [], "total": 0, "query": q}
|
||||
# CRLF in q would terminate the IMAP command early — reject defensively.
|
||||
@@ -3637,7 +3643,9 @@ def setup_email_routes():
|
||||
except Exception as e:
|
||||
logger.warning(f"Error parsing search result {uid}: {e}")
|
||||
continue
|
||||
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
|
||||
with _LIST_CACHE_LOCK:
|
||||
if index_generation == _LIST_CACHE_GENERATION[0]:
|
||||
_email_index_upsert(owner, account_id, effective_folder, fetched_emails)
|
||||
|
||||
return {
|
||||
"emails": emails,
|
||||
@@ -3774,8 +3782,7 @@ def setup_email_routes():
|
||||
# accepted it, so the index and list cache cannot drift ahead of
|
||||
# the mailbox.
|
||||
if mark_seen and not mark_seen_failed:
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_update_list_cache_seen(account_id, folder, uid, True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
|
||||
related_attachments = []
|
||||
if full and not _has_visible_attachments(msg):
|
||||
@@ -3914,8 +3921,7 @@ def setup_email_routes():
|
||||
status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)")
|
||||
if status != "OK":
|
||||
return False
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_update_list_cache_seen(account_id, folder, uid, True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"mark-seen after cached read failed uid={uid}: {e}")
|
||||
@@ -5147,7 +5153,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Seen", add=False):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", False)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5167,7 +5173,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Flagged", add=bool(on)):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Flagged", bool(on))
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True, "flagged": bool(on)}
|
||||
except Exception as e:
|
||||
@@ -5185,7 +5191,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Seen", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Seen", True)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5205,7 +5211,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _move_email_message(conn, uid, "Archive", role="archive"):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5236,7 +5242,7 @@ def setup_email_routes():
|
||||
# click or by another mailbox client; it is already gone
|
||||
# from the requested folder, so do not trap the UI on a
|
||||
# permanent “Email not found” error.
|
||||
_email_index_delete(owner, account_id, folder, str(uid))
|
||||
_index_delete(owner, account_id, folder, str(uid))
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True, "already_deleted": True}
|
||||
trash_folder = _resolve_mail_folder(conn, "Trash", "trash")
|
||||
@@ -5254,9 +5260,9 @@ def setup_email_routes():
|
||||
if not _move_email_message(conn, resolved_uid, trash_folder, role="trash"):
|
||||
logger.warning("Email delete Trash move failed uid=%s resolved_uid=%s folder=%s trash=%s", uid, resolved_uid, folder, trash_folder)
|
||||
return {"success": False, "error": "Could not move email to Trash"}
|
||||
_email_index_delete(owner, account_id, folder, resolved_uid)
|
||||
_index_delete(owner, account_id, folder, resolved_uid)
|
||||
if resolved_uid != str(uid):
|
||||
_email_index_delete(owner, account_id, folder, str(uid))
|
||||
_index_delete(owner, account_id, folder, str(uid))
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5272,7 +5278,7 @@ def setup_email_routes():
|
||||
if not _store_email_flag(conn, uid, "\\Deleted", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
conn.expunge()
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5340,7 +5346,7 @@ def setup_email_routes():
|
||||
pending_expunge.append(uid)
|
||||
continue
|
||||
deleted_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
except Exception:
|
||||
failed_uids.append(uid)
|
||||
logger.debug("Bulk email delete failed for uid=%s", uid, exc_info=True)
|
||||
@@ -5362,7 +5368,7 @@ def setup_email_routes():
|
||||
if expunge_ok:
|
||||
for uid in pending_expunge:
|
||||
deleted_uids.append(uid)
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
else:
|
||||
# Prioritize preservation over cleanup: unflag \Deleted from our
|
||||
# copied messages so future EXPUNGE operations do not destroy them.
|
||||
@@ -5481,7 +5487,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _move_email_message(conn, uid, dest):
|
||||
return {"success": False, "error": f"Failed to move to {dest}"}
|
||||
_email_index_delete(owner, account_id, folder, uid)
|
||||
_index_delete(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -5567,7 +5573,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Answered", add=True):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Answered", True)
|
||||
_clear_done_response_tags(owner, account_id, folder, uid)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
@@ -5583,7 +5589,7 @@ def setup_email_routes():
|
||||
conn.select(_q(folder))
|
||||
if not _store_email_flag(conn, uid, "\\Answered", add=False):
|
||||
return {"success": False, "error": "Email not found"}
|
||||
_email_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
|
||||
_index_update_flags(owner, account_id, folder, uid, "\\Answered", False)
|
||||
_invalidate_list_cache(account_id, folder)
|
||||
return {"success": True}
|
||||
except Exception as e:
|
||||
@@ -6250,7 +6256,7 @@ def setup_email_routes():
|
||||
try:
|
||||
st, _sel = imap.select(_q(_source_folder), readonly=False)
|
||||
if st == "OK" and _store_email_flag(imap, _source_uid, "\\Answered", add=True):
|
||||
_email_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
|
||||
_index_update_flags(owner, _account_id, _source_folder, _source_uid, "\\Answered", True)
|
||||
_clear_done_response_tags(owner, _account_id, _source_folder, _source_uid)
|
||||
_invalidate_list_cache(_account_id, _source_folder)
|
||||
logger.info(f"Marked source UID {_source_uid} as \\Answered in {_source_folder}")
|
||||
|
||||
Reference in New Issue
Block a user