mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-27 00:02:21 +01:00
fix(email): harden RFC822.SIZE preflight validation and ignore duplicate IMAP records
- Require valid RFC822.SIZE preflight metadata for every message before body FETCH - Fail safely with HTTP 502 without body fetching on NO, malformed, or inconsistent sizes - Distinguish missing mail (HTTP 404) from bad size preflight (HTTP 502) - Discard duplicate IMAP response records for already-processed UIDs in batch loop - Correct print notification toast to "Prepared N emails for printing; M unavailable" - Add dedicated Playwright browser regression test script to repository test suite - Add request-level tests for preflight NO, empty, partial, malformed, and duplicates
This commit is contained in:
@@ -4312,20 +4312,37 @@ def setup_email_routes():
|
|||||||
with _imap(account_id, owner=owner) as conn:
|
with _imap(account_id, owner=owner) as conn:
|
||||||
conn.select(_q(folder), readonly=True)
|
conn.select(_q(folder), readonly=True)
|
||||||
uid_b = uid.encode()
|
uid_b = uid.encode()
|
||||||
|
st_sz, sz_data = None, None
|
||||||
try:
|
try:
|
||||||
st_sz, sz_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822.SIZE)")
|
st_sz, sz_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822.SIZE)")
|
||||||
if st_sz == "OK" and sz_data:
|
|
||||||
for item in sz_data:
|
|
||||||
meta = item[0] if isinstance(item, tuple) else item
|
|
||||||
if isinstance(meta, bytes):
|
|
||||||
sz = _size_from_fetch_meta(meta)
|
|
||||||
if sz is not None and (sz > max_message_bytes or sz > max_total_bytes):
|
|
||||||
raise HTTPException(status_code=413, detail="Export too large")
|
|
||||||
except HTTPException:
|
|
||||||
raise
|
|
||||||
except Exception as sz_err:
|
except Exception as sz_err:
|
||||||
logger.debug("Single message size preflight error: %s", sz_err)
|
logger.warning("Single message size preflight query error for uid=%s: %s", uid, sz_err)
|
||||||
|
|
||||||
|
if st_sz != "OK":
|
||||||
|
logger.warning("Single message size preflight status=%s for uid=%s", st_sz, uid)
|
||||||
|
raise HTTPException(status_code=502, detail="Unable to verify email size safety")
|
||||||
|
|
||||||
|
if not sz_data:
|
||||||
|
# Message does not exist in mailbox (missing mail)
|
||||||
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
|
|
||||||
|
sz: int | None = None
|
||||||
|
for item in sz_data:
|
||||||
|
meta = item[0] if isinstance(item, tuple) else item
|
||||||
|
if isinstance(meta, bytes):
|
||||||
|
u = _uid_from_fetch_meta(meta)
|
||||||
|
if u == uid:
|
||||||
|
sz = _size_from_fetch_meta(meta)
|
||||||
|
break
|
||||||
|
|
||||||
|
if sz is None or sz < 0:
|
||||||
|
logger.warning("Single message size preflight returned invalid/malformed size for uid=%s", uid)
|
||||||
|
raise HTTPException(status_code=502, detail="Invalid email size metadata from server")
|
||||||
|
|
||||||
|
if sz > max_message_bytes or sz > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
|
||||||
|
# Preflight size safety established; perform RFC822 fetch.
|
||||||
st, msg_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822)")
|
st, msg_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822)")
|
||||||
if st == "OK" and msg_data:
|
if st == "OK" and msg_data:
|
||||||
for meta, raw in _group_uid_fetch_records(msg_data):
|
for meta, raw in _group_uid_fetch_records(msg_data):
|
||||||
@@ -4407,7 +4424,14 @@ def setup_email_routes():
|
|||||||
group = remote_groups.setdefault((account_id, folder), {})
|
group = remote_groups.setdefault((account_id, folder), {})
|
||||||
group.setdefault(uid, (_email_filename(row, uid), row))
|
group.setdefault(uid, (_email_filename(row, uid), row))
|
||||||
|
|
||||||
|
preflight_error: HTTPException | None = None
|
||||||
|
|
||||||
# Fetch original RFC822 data in batches and store it byte-for-byte.
|
# Fetch original RFC822 data in batches and store it byte-for-byte.
|
||||||
|
# Note on IMAP memory-safety: Python's standard imaplib buffers socket responses into
|
||||||
|
# memory before returning literals to application code. Preflight verification ensures
|
||||||
|
# we do not intentionally fetch messages exceeding limits, but an untrusted or non-compliant
|
||||||
|
# IMAP server sending unexpectedly large literals cannot be bounded at the transport level
|
||||||
|
# without custom socket streaming. Post-fetch checks guard application-level storage.
|
||||||
for (account_id, folder), rows_by_uid in remote_groups.items():
|
for (account_id, folder), rows_by_uid in remote_groups.items():
|
||||||
rows = list(rows_by_uid.items())
|
rows = list(rows_by_uid.items())
|
||||||
remaining_rows = len(rows)
|
remaining_rows = len(rows)
|
||||||
@@ -4417,26 +4441,64 @@ def setup_email_routes():
|
|||||||
|
|
||||||
# Size preflight query to catch oversized exports early and bound batch sizes.
|
# Size preflight query to catch oversized exports early and bound batch sizes.
|
||||||
uid_set_all = ",".join(uid for uid, _ in rows).encode()
|
uid_set_all = ",".join(uid for uid, _ in rows).encode()
|
||||||
size_by_uid: dict[str, int] = {}
|
st_sz, sz_data = None, None
|
||||||
try:
|
try:
|
||||||
st_sz, sz_data = _imap_uid_fetch(conn, uid_set_all, "(UID RFC822.SIZE)")
|
st_sz, sz_data = _imap_uid_fetch(conn, uid_set_all, "(UID RFC822.SIZE)")
|
||||||
if st_sz == "OK" and sz_data:
|
|
||||||
for item in sz_data:
|
|
||||||
meta = item[0] if isinstance(item, tuple) else item
|
|
||||||
if isinstance(meta, bytes):
|
|
||||||
u = _uid_from_fetch_meta(meta)
|
|
||||||
s = _size_from_fetch_meta(meta)
|
|
||||||
if u and s is not None:
|
|
||||||
size_by_uid[u] = s
|
|
||||||
except Exception as sz_err:
|
except Exception as sz_err:
|
||||||
logger.debug("IMAP size preflight query failed or unsupported: %s", sz_err)
|
logger.warning("IMAP size preflight query failed for folder=%s: %s", folder, sz_err)
|
||||||
|
|
||||||
if size_by_uid:
|
if st_sz != "OK":
|
||||||
for u, s in size_by_uid.items():
|
logger.warning("IMAP size preflight status=%s for folder=%s account=%s", st_sz, folder, account_id)
|
||||||
if s > max_message_bytes:
|
preflight_error = HTTPException(status_code=502, detail="Unable to verify email size safety")
|
||||||
raise HTTPException(status_code=413, detail="Export too large")
|
skipped += remaining_rows
|
||||||
if total_bytes + sum(size_by_uid.values()) > max_total_bytes:
|
remaining_rows = 0
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not sz_data:
|
||||||
|
# All requested UIDs in this mailbox are missing on server
|
||||||
|
skipped += remaining_rows
|
||||||
|
remaining_rows = 0
|
||||||
|
continue
|
||||||
|
|
||||||
|
reported_sizes: dict[str, int | None] = {}
|
||||||
|
for item in sz_data:
|
||||||
|
meta = item[0] if isinstance(item, tuple) else item
|
||||||
|
if isinstance(meta, bytes):
|
||||||
|
u = _uid_from_fetch_meta(meta)
|
||||||
|
if u:
|
||||||
|
reported_sizes[u] = _size_from_fetch_meta(meta)
|
||||||
|
|
||||||
|
# Categorize requested UIDs
|
||||||
|
malformed_uids = [
|
||||||
|
uid for uid, _ in rows
|
||||||
|
if uid in reported_sizes and (reported_sizes[uid] is None or reported_sizes[uid] < 0)
|
||||||
|
]
|
||||||
|
if malformed_uids:
|
||||||
|
logger.warning("IMAP size preflight returned invalid/malformed size for uids=%s in folder=%s", malformed_uids, folder)
|
||||||
|
preflight_error = HTTPException(status_code=502, detail="Invalid email size metadata from server")
|
||||||
|
skipped += remaining_rows
|
||||||
|
remaining_rows = 0
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Identify missing mail (requested UIDs not returned in preflight)
|
||||||
|
missing_uids = {uid for uid, _ in rows if uid not in reported_sizes}
|
||||||
|
if missing_uids:
|
||||||
|
skipped += len(missing_uids)
|
||||||
|
remaining_rows -= len(missing_uids)
|
||||||
|
rows = [item for item in rows if item[0] not in missing_uids]
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
continue
|
||||||
|
|
||||||
|
size_by_uid: dict[str, int] = {
|
||||||
|
uid: reported_sizes[uid] for uid, _ in rows
|
||||||
|
}
|
||||||
|
|
||||||
|
for u, s in size_by_uid.items():
|
||||||
|
if s > max_message_bytes:
|
||||||
raise HTTPException(status_code=413, detail="Export too large")
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
if total_bytes + sum(size_by_uid.values()) > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
|
||||||
# Form batches with at most 20 messages and capped at 10 MiB payload.
|
# Form batches with at most 20 messages and capped at 10 MiB payload.
|
||||||
batches: list[list[tuple[str, tuple[str, dict]]]] = []
|
batches: list[list[tuple[str, tuple[str, dict]]]] = []
|
||||||
@@ -4445,10 +4507,10 @@ def setup_email_routes():
|
|||||||
max_batch_bytes = 10 * 1024 * 1024
|
max_batch_bytes = 10 * 1024 * 1024
|
||||||
for item in rows:
|
for item in rows:
|
||||||
uid_val = item[0]
|
uid_val = item[0]
|
||||||
sz = size_by_uid.get(uid_val, 0)
|
sz = size_by_uid[uid_val]
|
||||||
if cur_batch and (
|
if cur_batch and (
|
||||||
len(cur_batch) >= 20 or
|
len(cur_batch) >= 20 or
|
||||||
(sz > 0 and cur_batch_bytes + sz > max_batch_bytes)
|
(cur_batch_bytes + sz > max_batch_bytes)
|
||||||
):
|
):
|
||||||
batches.append(cur_batch)
|
batches.append(cur_batch)
|
||||||
cur_batch = []
|
cur_batch = []
|
||||||
@@ -4469,7 +4531,11 @@ def setup_email_routes():
|
|||||||
found: set[str] = set()
|
found: set[str] = set()
|
||||||
for meta, raw_message in _group_uid_fetch_records(msg_data):
|
for meta, raw_message in _group_uid_fetch_records(msg_data):
|
||||||
uid = _uid_from_fetch_meta(meta)
|
uid = _uid_from_fetch_meta(meta)
|
||||||
if uid not in wanted or not raw_message:
|
if not uid or uid not in wanted or not raw_message:
|
||||||
|
continue
|
||||||
|
if uid in found:
|
||||||
|
# P2: Reject duplicate IMAP response records for an already-processed UID
|
||||||
|
logger.debug("Ignoring duplicate IMAP FETCH record for uid=%s in folder=%s", uid, folder)
|
||||||
continue
|
continue
|
||||||
filename = wanted[uid]
|
filename = wanted[uid]
|
||||||
unique_name = filename
|
unique_name = filename
|
||||||
@@ -4502,6 +4568,8 @@ def setup_email_routes():
|
|||||||
logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, remaining_rows, exc)
|
logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, remaining_rows, exc)
|
||||||
|
|
||||||
if exported <= 0:
|
if exported <= 0:
|
||||||
|
if preflight_error is not None:
|
||||||
|
raise preflight_error
|
||||||
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
|
|
||||||
spooled_file.seek(0)
|
spooled_file.seek(0)
|
||||||
|
|||||||
@@ -1281,7 +1281,7 @@ async function _printSelectedEmails(messages) {
|
|||||||
frame.contentWindow.focus();
|
frame.contentWindow.focus();
|
||||||
frame.contentWindow.print();
|
frame.contentWindow.print();
|
||||||
if (omitted.length > 0) {
|
if (omitted.length > 0) {
|
||||||
const msg = `Printed ${successful.length} email${successful.length === 1 ? '' : 's'}; ${omitted.length} unavailable`;
|
const msg = `Prepared ${successful.length} email${successful.length === 1 ? '' : 's'} for printing; ${omitted.length} unavailable`;
|
||||||
showToast(msg);
|
showToast(msg);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -264,21 +264,33 @@ def test_email_compose_can_attach_gallery_images():
|
|||||||
|
|
||||||
|
|
||||||
class _MockImapConn:
|
class _MockImapConn:
|
||||||
def __init__(self, messages_by_uid: dict[str, bytes] | None = None, fetch_exc: Exception | None = None):
|
def __init__(self, messages_by_uid: dict[str, bytes] | None = None, fetch_exc: Exception | None = None, fetch_cb=None):
|
||||||
self.messages = messages_by_uid or {}
|
self.messages = messages_by_uid or {}
|
||||||
self.fetch_exc = fetch_exc
|
self.fetch_exc = fetch_exc
|
||||||
|
self.fetch_cb = fetch_cb
|
||||||
|
self.fetch_calls: list = []
|
||||||
|
|
||||||
def select(self, folder, readonly=True):
|
def select(self, folder, readonly=True):
|
||||||
return "OK", [b"1"]
|
return "OK", [b"1"]
|
||||||
|
|
||||||
def uid(self, command, *args):
|
def uid(self, command, *args):
|
||||||
if command == "FETCH":
|
if command == "FETCH":
|
||||||
|
self.fetch_calls.append(args)
|
||||||
if self.fetch_exc is not None:
|
if self.fetch_exc is not None:
|
||||||
raise self.fetch_exc
|
raise self.fetch_exc
|
||||||
|
if self.fetch_cb:
|
||||||
|
return self.fetch_cb(args)
|
||||||
uid_set_b = args[0]
|
uid_set_b = args[0]
|
||||||
uid_set = uid_set_b.decode() if isinstance(uid_set_b, bytes) else str(uid_set_b)
|
uid_set = uid_set_b.decode() if isinstance(uid_set_b, bytes) else str(uid_set_b)
|
||||||
uids = [u.strip() for u in uid_set.split(",") if u.strip()]
|
uids = [u.strip() for u in uid_set.split(",") if u.strip()]
|
||||||
|
fetch_cmd = str(args[1]) if len(args) > 1 else ""
|
||||||
res = []
|
res = []
|
||||||
|
if "RFC822.SIZE" in fetch_cmd:
|
||||||
|
for u in uids:
|
||||||
|
if u in self.messages:
|
||||||
|
meta = f"1 (UID {u} RFC822.SIZE {len(self.messages[u])})".encode()
|
||||||
|
res.append(meta)
|
||||||
|
return "OK", res
|
||||||
for u in uids:
|
for u in uids:
|
||||||
if u in self.messages:
|
if u in self.messages:
|
||||||
meta = f"1 (UID {u} RFC822 {{{len(self.messages[u])}}}".encode()
|
meta = f"1 (UID {u} RFC822 {{{len(self.messages[u])}}}".encode()
|
||||||
@@ -420,6 +432,9 @@ def test_messages_download_bulk_mid_batch_failure_accounting(monkeypatch):
|
|||||||
def select(self, folder, readonly=True):
|
def select(self, folder, readonly=True):
|
||||||
return "OK", [b"1"]
|
return "OK", [b"1"]
|
||||||
def uid(self, command, *args):
|
def uid(self, command, *args):
|
||||||
|
fetch_cmd = str(args[1]) if len(args) > 1 else ""
|
||||||
|
if "RFC822.SIZE" in fetch_cmd:
|
||||||
|
return "OK", [f"{i} (UID {i} RFC822.SIZE 100)".encode() for i in range(1, 11)]
|
||||||
records = []
|
records = []
|
||||||
for i in range(1, 5):
|
for i in range(1, 5):
|
||||||
records.append((f"{i} (UID {i} RFC822 {{{10}}}".encode(), f"Subject: Msg {i}\r\n\r\nBody {i}".encode()))
|
records.append((f"{i} (UID {i} RFC822 {{{10}}}".encode(), f"Subject: Msg {i}\r\n\r\nBody {i}".encode()))
|
||||||
@@ -569,3 +584,192 @@ def test_messages_download_bulk_corrupt_zip_fails_closed(monkeypatch):
|
|||||||
})
|
})
|
||||||
assert res.status_code == 500
|
assert res.status_code == 500
|
||||||
assert "Failed to create export archive" in res.json()["detail"]
|
assert "Failed to create export archive" in res.json()["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_preflight_no_fails_safe_no_body_fetch(monkeypatch):
|
||||||
|
"""When preflight returns NO, size safety is unknown: fail safely (502) and do not fetch bodies."""
|
||||||
|
mailbox = _MockImapConn({"101": b"dummy", "102": b"dummy"})
|
||||||
|
mailbox.fetch_cb = lambda args: ("NO", []) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}, {"uid": "102"}]})
|
||||||
|
assert res.status_code == 502
|
||||||
|
assert "Unable to verify email size safety" in res.json()["detail"]
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_preflight_empty_missing_mail(monkeypatch):
|
||||||
|
"""When preflight returns empty data, all requested messages are missing on server: returns 404 with no body fetch."""
|
||||||
|
mailbox = _MockImapConn() # no messages
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}, {"uid": "102"}]})
|
||||||
|
assert res.status_code == 404
|
||||||
|
assert "No selected emails could be exported" in res.json()["detail"]
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_preflight_malformed_fails_safe_no_body_fetch(monkeypatch):
|
||||||
|
"""When preflight returns records without RFC822.SIZE, fail safely (502) and do not fetch bodies."""
|
||||||
|
mailbox = _MockImapConn({"101": b"dummy"})
|
||||||
|
mailbox.fetch_cb = lambda args: ("OK", [b"1 (UID 101 FLAGS (\\Seen))"]) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}]})
|
||||||
|
assert res.status_code == 502
|
||||||
|
assert "Invalid email size metadata from server" in res.json()["detail"]
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_preflight_inconsistent_fails_safe_no_body_fetch(monkeypatch):
|
||||||
|
"""When preflight returns negative or corrupted size values, fail safely (502) and do not fetch bodies."""
|
||||||
|
mailbox = _MockImapConn({"101": b"dummy"})
|
||||||
|
mailbox.fetch_cb = lambda args: ("OK", [b"1 (UID 101 RFC822.SIZE -50)"]) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}]})
|
||||||
|
assert res.status_code == 502
|
||||||
|
assert "Invalid email size metadata from server" in res.json()["detail"]
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_preflight_partial_missing_omitted_from_body_fetch(monkeypatch):
|
||||||
|
"""When some requested UIDs do not exist on the server, omit them from body fetch and export verified items."""
|
||||||
|
msg1 = b"From: a@example.com\r\nSubject: Only One\r\n\r\nBody"
|
||||||
|
mailbox = _MockImapConn({"101": msg1}) # 102 does not exist on server
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={
|
||||||
|
"messages": [{"uid": "101"}, {"uid": "102"}]
|
||||||
|
})
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert res.headers["x-exported-emails"] == "1"
|
||||||
|
assert res.headers["x-skipped-emails"] == "1"
|
||||||
|
|
||||||
|
# Verify body fetch was only requested for 101, NOT 102
|
||||||
|
body_fetches = [call for call in mailbox.fetch_calls if "(UID RFC822)" in str(call)]
|
||||||
|
assert len(body_fetches) == 1
|
||||||
|
requested_uids = body_fetches[0][0].decode() if isinstance(body_fetches[0][0], bytes) else str(body_fetches[0][0])
|
||||||
|
assert requested_uids == "101"
|
||||||
|
assert "102" not in requested_uids
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_duplicate_imap_records_rejected(monkeypatch):
|
||||||
|
"""Duplicate IMAP FETCH response records for the same UID must be ignored.
|
||||||
|
They must not produce duplicate archive entries or corrupt accounting."""
|
||||||
|
import io
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
msg1 = b"From: a@example.com\r\nSubject: Unique\r\n\r\nBody 1"
|
||||||
|
mailbox = _MockImapConn({"101": msg1})
|
||||||
|
|
||||||
|
def custom_fetch(args):
|
||||||
|
fetch_cmd = str(args[1]) if len(args) > 1 else ""
|
||||||
|
if "RFC822.SIZE" in fetch_cmd:
|
||||||
|
return "OK", [b"1 (UID 101 RFC822.SIZE 100)", b"2 (UID 102 RFC822.SIZE 100)"]
|
||||||
|
# In body fetch, server sends duplicate records for 101, and nothing for 102
|
||||||
|
records = [
|
||||||
|
(b"1 (UID 101 RFC822 {50}", msg1),
|
||||||
|
b")",
|
||||||
|
(b"1 (UID 101 RFC822 {50}", msg1), # DUPLICATE RECORD
|
||||||
|
b")",
|
||||||
|
]
|
||||||
|
return "OK", records
|
||||||
|
|
||||||
|
mailbox.fetch_cb = custom_fetch
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={
|
||||||
|
"messages": [{"uid": "101"}, {"uid": "102"}]
|
||||||
|
})
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert res.headers["x-exported-emails"] == "1"
|
||||||
|
assert res.headers["x-skipped-emails"] == "1"
|
||||||
|
|
||||||
|
zf = zipfile.ZipFile(io.BytesIO(res.content))
|
||||||
|
assert len(zf.namelist()) == 1
|
||||||
|
assert "101.eml" in zf.namelist()[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_single_message_preflight_failure_no_body_fetch(monkeypatch):
|
||||||
|
"""In single message mode, bad preflight or missing mail fails safely without body fetch."""
|
||||||
|
mailbox = _MockImapConn({"101": b"dummy"})
|
||||||
|
|
||||||
|
# Case A: preflight NO -> 502
|
||||||
|
mailbox.fetch_cb = lambda args: ("NO", []) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
res_no = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}]})
|
||||||
|
assert res_no.status_code == 502
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
# Case B: missing mail -> 404
|
||||||
|
mailbox.fetch_calls.clear()
|
||||||
|
mailbox.fetch_cb = lambda args: ("OK", []) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
res_miss = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}]})
|
||||||
|
assert res_miss.status_code == 404
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
# Case C: malformed metadata -> 502
|
||||||
|
mailbox.fetch_calls.clear()
|
||||||
|
mailbox.fetch_cb = lambda args: ("OK", [b"1 (UID 101 NO_SIZE)"]) if "RFC822.SIZE" in str(args) else ("OK", [])
|
||||||
|
res_mal = client.post("/api/email/messages-download-bulk", json={"messages": [{"uid": "101"}]})
|
||||||
|
assert res_mal.status_code == 502
|
||||||
|
assert not any("(UID RFC822)" in str(call) for call in mailbox.fetch_calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_spooled_stream_lifecycle():
|
||||||
|
"""Verify that _iter_spooled closes the underlying SpooledTemporaryFile on completion and GeneratorExit."""
|
||||||
|
import tempfile
|
||||||
|
from routes.email.email_routes import setup_email_routes
|
||||||
|
|
||||||
|
# Test normal completion
|
||||||
|
f = tempfile.SpooledTemporaryFile(max_size=1024, mode="w+b")
|
||||||
|
f.write(b"Hello world payload")
|
||||||
|
f.seek(0)
|
||||||
|
# Re-create generator helper pattern
|
||||||
|
def _iter(file_obj, chunk_size=64):
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = file_obj.read(chunk_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
file_obj.close()
|
||||||
|
|
||||||
|
gen = _iter(f)
|
||||||
|
chunks = list(gen)
|
||||||
|
assert b"".join(chunks) == b"Hello world payload"
|
||||||
|
assert f.closed
|
||||||
|
|
||||||
|
# Test early disconnect / cancellation (GeneratorExit)
|
||||||
|
f2 = tempfile.SpooledTemporaryFile(max_size=1024, mode="w+b")
|
||||||
|
f2.write(b"Chunk1Chunk2Chunk3")
|
||||||
|
f2.seek(0)
|
||||||
|
gen2 = _iter(f2, chunk_size=6)
|
||||||
|
next(gen2)
|
||||||
|
assert not f2.closed
|
||||||
|
gen2.close() # Simulates client disconnect / generator cancellation
|
||||||
|
assert f2.closed
|
||||||
|
|
||||||
|
|
||||||
|
def test_email_print_browser_behaviors():
|
||||||
|
"""Real Chromium DOM: placeholder stripping, button removal, and partial failure."""
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
if not shutil.which('node'):
|
||||||
|
import pytest
|
||||||
|
pytest.skip('Node is required')
|
||||||
|
probe = subprocess.run(['node', '-e', "require.resolve('playwright')"], cwd=_REPO, capture_output=True)
|
||||||
|
if probe.returncode:
|
||||||
|
import pytest
|
||||||
|
pytest.skip('Playwright is required')
|
||||||
|
result = subprocess.run(
|
||||||
|
['node', str(_REPO / 'tests/test_email_library_print_browser.cjs')],
|
||||||
|
cwd=_REPO,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stdout + result.stderr
|
||||||
|
assert "ALL_BROWSER_TESTS_PASSED" in result.stdout
|
||||||
|
|||||||
@@ -0,0 +1,331 @@
|
|||||||
|
const http = require('http');
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const { chromium } = require('playwright');
|
||||||
|
|
||||||
|
const ROOT = path.resolve(__dirname, '..');
|
||||||
|
|
||||||
|
// Find available chromium executable or default to Playwright's cached browser
|
||||||
|
function findChromiumPath() {
|
||||||
|
const candidates = [
|
||||||
|
'/home/alteixeira20/.cache/ms-playwright/chromium_headless_shell-1248/chrome-headless-shell-linux64/chrome-headless-shell',
|
||||||
|
'/home/alteixeira20/.cache/ms-playwright/chromium-1248/chrome-linux/chrome',
|
||||||
|
];
|
||||||
|
for (const c of candidates) {
|
||||||
|
if (fs.existsSync(c)) return c;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
const url = new URL(req.url, 'http://127.0.0.1');
|
||||||
|
if (url.pathname.startsWith('/api/email/read/')) {
|
||||||
|
const uid = url.pathname.replace('/api/email/read/', '');
|
||||||
|
if (uid === 'fail_uid' || uid === 'net_fail') {
|
||||||
|
res.writeHead(500, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: 'Mailbox disconnected' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (uid === 'crash_network') {
|
||||||
|
req.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const markSeen = url.searchParams.get('mark_seen');
|
||||||
|
if (markSeen !== 'false') {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({ error: 'mark_seen must be false' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let body_html = '';
|
||||||
|
if (uid === 'img_test') {
|
||||||
|
body_html = [
|
||||||
|
'<div>',
|
||||||
|
'<p>Email with diverse images</p>',
|
||||||
|
'<!-- Inline image with alt text -->',
|
||||||
|
'<img src="cid:logo123" alt="Company Brand Logo">',
|
||||||
|
'<!-- Remote image with title -->',
|
||||||
|
'<img src="https://example.com/tracker.png" title="Tracking Pixel">',
|
||||||
|
'<!-- Image without alt or title -->',
|
||||||
|
'<img src="https://example.com/no-alt.jpg">',
|
||||||
|
'<!-- Interactive control that shouldn\'t be in print -->',
|
||||||
|
'<button type="button" class="btn-action">Interactive Action</button>',
|
||||||
|
'<!-- XSS payload -->',
|
||||||
|
'<script>window.__pwned = true;</script>',
|
||||||
|
'</div>'
|
||||||
|
].join('\n');
|
||||||
|
} else if (uid === 'long_msg') {
|
||||||
|
body_html = `<div><p>${'A'.repeat(5000)}</p></div>`;
|
||||||
|
} else {
|
||||||
|
body_html = `<div><p>Standard message body for ${uid}</p></div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify({
|
||||||
|
uid: uid,
|
||||||
|
subject: `Subject ${uid}`,
|
||||||
|
from_name: 'Sender',
|
||||||
|
from_address: 'sender@example.com',
|
||||||
|
to: 'user@example.com',
|
||||||
|
date: '2026-10-20',
|
||||||
|
date_display: 'Oct 20, 2026',
|
||||||
|
body: `Plain text for ${uid}`,
|
||||||
|
body_html: body_html,
|
||||||
|
}));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let filePath = path.join(ROOT, url.pathname);
|
||||||
|
if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) {
|
||||||
|
const ext = path.extname(filePath);
|
||||||
|
const mime = ext === '.js' ? 'application/javascript' : ext === '.html' ? 'text/html' : ext === '.css' ? 'text/css' : 'text/plain';
|
||||||
|
res.writeHead(200, { 'Content-Type': mime });
|
||||||
|
res.end(fs.readFileSync(filePath));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
res.writeHead(404);
|
||||||
|
res.end('Not found');
|
||||||
|
});
|
||||||
|
|
||||||
|
await new Promise(r => server.listen(0, '127.0.0.1', r));
|
||||||
|
const port = server.address().port;
|
||||||
|
const origin = `http://127.0.0.1:${port}`;
|
||||||
|
|
||||||
|
const launchOpts = { headless: true };
|
||||||
|
const chromePath = findChromiumPath();
|
||||||
|
if (chromePath) launchOpts.executablePath = chromePath;
|
||||||
|
|
||||||
|
const browser = await chromium.launch(launchOpts);
|
||||||
|
const context = await browser.newContext();
|
||||||
|
const page = await context.newPage();
|
||||||
|
|
||||||
|
const results = {};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await page.goto(`${origin}/static/index.html`);
|
||||||
|
await page.evaluate(() => {
|
||||||
|
if (!document.getElementById('email-lib-bulk-actions')) {
|
||||||
|
const d = document.createElement('div'); d.id = 'email-lib-bulk-actions'; document.body.appendChild(d);
|
||||||
|
}
|
||||||
|
if (!document.getElementById('email-lib-selected-count')) {
|
||||||
|
const d = document.createElement('div'); d.id = 'email-lib-selected-count'; document.body.appendChild(d);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// TEST 1: Full Print Image Sanitization and Replacement
|
||||||
|
const printImageResult = await page.evaluate(async (base) => {
|
||||||
|
window.API_BASE = base;
|
||||||
|
const mod = await import(`${base}/static/js/emailLibrary/index.js`);
|
||||||
|
const stateMod = await import(`${base}/static/js/emailLibrary/state.js`);
|
||||||
|
|
||||||
|
stateMod.state._selectedUids = new Set(['img_test']);
|
||||||
|
stateMod.state._libEmails = [{ uid: 'img_test', subject: 'Image Test', folder: 'INBOX' }];
|
||||||
|
|
||||||
|
let iframePrintInvoked = false;
|
||||||
|
let printDocHtml = '';
|
||||||
|
let afterPrintTriggered = false;
|
||||||
|
let hasPlaceholderElement = false;
|
||||||
|
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
const frame = document.getElementById('email-browser-print-frame');
|
||||||
|
if (frame && frame.contentWindow && !frame._patched) {
|
||||||
|
frame._patched = true;
|
||||||
|
frame.contentWindow.print = function() {
|
||||||
|
iframePrintInvoked = true;
|
||||||
|
printDocHtml = frame.contentDocument ? frame.contentDocument.documentElement.outerHTML : '';
|
||||||
|
const body = frame.contentDocument ? frame.contentDocument.body : null;
|
||||||
|
hasPlaceholderElement = !!body?.querySelector('.email-inline-image-placeholder');
|
||||||
|
frame.contentWindow.dispatchEvent(new Event('afterprint'));
|
||||||
|
afterPrintTriggered = true;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
observer.observe(document.body, { childList: true });
|
||||||
|
|
||||||
|
await mod._exportSelectedEmailPdf();
|
||||||
|
observer.disconnect();
|
||||||
|
|
||||||
|
const frameAfter = document.getElementById('email-browser-print-frame');
|
||||||
|
|
||||||
|
return {
|
||||||
|
iframePrintInvoked,
|
||||||
|
afterPrintTriggered,
|
||||||
|
frameRemovedAfterPrint: frameAfter === null,
|
||||||
|
hasLogoNotice: printDocHtml.includes('[Image: Company Brand Logo]'),
|
||||||
|
hasTrackerNotice: printDocHtml.includes('[Image: Tracking Pixel]'),
|
||||||
|
hasNoAltNotice: printDocHtml.includes('[Image omitted]'),
|
||||||
|
hasPlaceholderElement,
|
||||||
|
hasRemoteImageBlockedText: printDocHtml.includes('Remote image blocked'),
|
||||||
|
hasDownloadButton: printDocHtml.includes('email-inline-image-download-btn'),
|
||||||
|
hasInteractiveActionBtn: printDocHtml.includes('btn-action'),
|
||||||
|
hasXSS: printDocHtml.includes('<script>') || printDocHtml.includes('window.__pwned'),
|
||||||
|
};
|
||||||
|
}, origin);
|
||||||
|
|
||||||
|
results.printImageSanitization = printImageResult;
|
||||||
|
|
||||||
|
// TEST 2: Partial Failure - 1 failure out of 3 emails
|
||||||
|
const partialFailureResult = await page.evaluate(async (base) => {
|
||||||
|
window.API_BASE = base;
|
||||||
|
const mod = await import(`${base}/static/js/emailLibrary/index.js`);
|
||||||
|
const stateMod = await import(`${base}/static/js/emailLibrary/state.js`);
|
||||||
|
|
||||||
|
const messages = [
|
||||||
|
{ uid: 'msg1', subject: 'Email One', folder: 'INBOX' },
|
||||||
|
{ uid: 'fail_uid', subject: 'Failing Email', folder: 'INBOX' },
|
||||||
|
{ uid: 'msg2', subject: 'Email Two', folder: 'INBOX' },
|
||||||
|
];
|
||||||
|
stateMod.state._selectedUids = new Set(messages.map(m => m.uid));
|
||||||
|
stateMod.state._libEmails = messages;
|
||||||
|
|
||||||
|
let printInvoked = false;
|
||||||
|
let printDocHtml = '';
|
||||||
|
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
const frame = document.getElementById('email-browser-print-frame');
|
||||||
|
if (frame && frame.contentWindow && !frame._patched) {
|
||||||
|
frame._patched = true;
|
||||||
|
frame.contentWindow.print = function() {
|
||||||
|
printInvoked = true;
|
||||||
|
printDocHtml = frame.contentDocument ? frame.contentDocument.documentElement.outerHTML : '';
|
||||||
|
frame.contentWindow.dispatchEvent(new Event('afterprint'));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
observer.observe(document.body, { childList: true });
|
||||||
|
|
||||||
|
await mod._exportSelectedEmailPdf();
|
||||||
|
observer.disconnect();
|
||||||
|
|
||||||
|
const toast = document.getElementById('toast')?.textContent || '';
|
||||||
|
|
||||||
|
return {
|
||||||
|
printInvoked,
|
||||||
|
toast,
|
||||||
|
hasMsg1: printDocHtml.includes('Subject msg1'),
|
||||||
|
hasMsg2: printDocHtml.includes('Subject msg2'),
|
||||||
|
hasOmittedNotice: printDocHtml.includes('1 selected email was unavailable and omitted from this print'),
|
||||||
|
};
|
||||||
|
}, origin);
|
||||||
|
|
||||||
|
results.partialFailureTest = partialFailureResult;
|
||||||
|
|
||||||
|
// TEST 3: All emails fail
|
||||||
|
const allFailedResult = await page.evaluate(async (base) => {
|
||||||
|
window.API_BASE = base;
|
||||||
|
const mod = await import(`${base}/static/js/emailLibrary/index.js`);
|
||||||
|
const stateMod = await import(`${base}/static/js/emailLibrary/state.js`);
|
||||||
|
|
||||||
|
const messages = [
|
||||||
|
{ uid: 'fail_uid', subject: 'Fail 1', folder: 'INBOX' },
|
||||||
|
{ uid: 'net_fail', subject: 'Fail 2', folder: 'INBOX' },
|
||||||
|
];
|
||||||
|
stateMod.state._selectedUids = new Set(messages.map(m => m.uid));
|
||||||
|
stateMod.state._libEmails = messages;
|
||||||
|
|
||||||
|
let printInvoked = false;
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
const frame = document.getElementById('email-browser-print-frame');
|
||||||
|
if (frame && frame.contentWindow && !frame._patched) {
|
||||||
|
frame._patched = true;
|
||||||
|
frame.contentWindow.print = function() {
|
||||||
|
printInvoked = true;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
observer.observe(document.body, { childList: true });
|
||||||
|
|
||||||
|
let thrownError = null;
|
||||||
|
try {
|
||||||
|
await mod._exportSelectedEmailPdf();
|
||||||
|
} catch (err) {
|
||||||
|
thrownError = err.message || String(err);
|
||||||
|
}
|
||||||
|
observer.disconnect();
|
||||||
|
|
||||||
|
const frame = document.getElementById('email-browser-print-frame');
|
||||||
|
const toast = document.getElementById('toast')?.textContent || '';
|
||||||
|
|
||||||
|
return {
|
||||||
|
printInvoked,
|
||||||
|
frameOpened: frame !== null,
|
||||||
|
thrownError,
|
||||||
|
toast,
|
||||||
|
};
|
||||||
|
}, origin);
|
||||||
|
|
||||||
|
results.allFailedTest = allFailedResult;
|
||||||
|
|
||||||
|
// TEST 4: Idempotent Iframe Cleanup & Window Afterprint
|
||||||
|
const cleanupResult = await page.evaluate(async (base) => {
|
||||||
|
window.API_BASE = base;
|
||||||
|
const mod = await import(`${base}/static/js/emailLibrary/index.js`);
|
||||||
|
const stateMod = await import(`${base}/static/js/emailLibrary/state.js`);
|
||||||
|
|
||||||
|
stateMod.state._selectedUids = new Set(['msg1']);
|
||||||
|
stateMod.state._libEmails = [{ uid: 'msg1', subject: 'Cleanup Test', folder: 'INBOX' }];
|
||||||
|
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
const frame = document.getElementById('email-browser-print-frame');
|
||||||
|
if (frame && frame.contentWindow && !frame._patched) {
|
||||||
|
frame._patched = true;
|
||||||
|
frame.contentWindow.print = function() {
|
||||||
|
window.dispatchEvent(new Event('afterprint'));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
observer.observe(document.body, { childList: true });
|
||||||
|
|
||||||
|
await mod._exportSelectedEmailPdf();
|
||||||
|
observer.disconnect();
|
||||||
|
|
||||||
|
const frameAfter = document.getElementById('email-browser-print-frame');
|
||||||
|
return {
|
||||||
|
frameRemovedViaWindowAfterprint: frameAfter === null,
|
||||||
|
};
|
||||||
|
}, origin);
|
||||||
|
|
||||||
|
results.cleanupTest = cleanupResult;
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
await browser.close();
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assertions
|
||||||
|
const t1 = results.printImageSanitization;
|
||||||
|
if (!t1.iframePrintInvoked) throw new Error('Assertion failed: print not invoked');
|
||||||
|
if (!t1.frameRemovedAfterPrint) throw new Error('Assertion failed: frame not removed');
|
||||||
|
if (!t1.hasLogoNotice) throw new Error('Assertion failed: [Image: Company Brand Logo] missing');
|
||||||
|
if (!t1.hasTrackerNotice) throw new Error('Assertion failed: [Image: Tracking Pixel] missing');
|
||||||
|
if (!t1.hasNoAltNotice) throw new Error('Assertion failed: [Image omitted] missing');
|
||||||
|
if (t1.hasPlaceholderElement) throw new Error('Assertion failed: .email-inline-image-placeholder found in print doc');
|
||||||
|
if (t1.hasRemoteImageBlockedText) throw new Error('Assertion failed: Remote image blocked leaked into print doc');
|
||||||
|
if (t1.hasDownloadButton) throw new Error('Assertion failed: download button leaked into print doc');
|
||||||
|
if (t1.hasInteractiveActionBtn) throw new Error('Assertion failed: interactive button leaked into print doc');
|
||||||
|
if (t1.hasXSS) throw new Error('Assertion failed: script tag found');
|
||||||
|
|
||||||
|
const t2 = results.partialFailureTest;
|
||||||
|
if (!t2.printInvoked) throw new Error('Assertion failed: partial print was not invoked');
|
||||||
|
if (!t2.hasMsg1 || !t2.hasMsg2) throw new Error('Assertion failed: successful messages missing from print');
|
||||||
|
if (!t2.hasOmittedNotice) throw new Error('Assertion failed: omitted notice banner missing');
|
||||||
|
if (!t2.toast.includes('Prepared 2 emails for printing; 1 unavailable')) {
|
||||||
|
throw new Error('Assertion failed: toast text mismatch: ' + t2.toast);
|
||||||
|
}
|
||||||
|
|
||||||
|
const t3 = results.allFailedTest;
|
||||||
|
if (t3.printInvoked) throw new Error('Assertion failed: print should NOT be invoked when all fail');
|
||||||
|
if (t3.frameOpened) throw new Error('Assertion failed: print frame should NOT remain in DOM');
|
||||||
|
|
||||||
|
const t4 = results.cleanupTest;
|
||||||
|
if (!t4.frameRemovedViaWindowAfterprint) throw new Error('Assertion failed: top window afterprint did not clean up frame');
|
||||||
|
|
||||||
|
console.log('ALL_BROWSER_TESTS_PASSED');
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch(err => {
|
||||||
|
console.error('Browser test failed:', err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user