mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-21 06:12:21 +02:00
fix(email): resolve memory safety, print rendering, and accounting defects
- Use SpooledTemporaryFile with 32 MiB spool threshold for export ZIP archives - Add IMAP size preflight via (UID RFC822.SIZE) and cap batch payload at 10 MiB - Enforce 64 MiB per-message ceiling and fail closed with 500 on corrupted ZIP - Stream spooled archive chunks with generator cleanup on disconnect/cancellation - Bypass ZIP creation for single raw EML message downloads - Memoize account ownership checks per request to eliminate N+1 database queries - Replace interactive image placeholders in print view with clean text notices - Strip interactive buttons, icons, and image actions from print document - Implement partial print success policy printing loaded items and reporting omitted - Harden print iframe lifecycle with window afterprint listener and fallback timer - Correct mid-batch exception accounting so exported plus skipped equals requested - Add executable behavioral tests for limits, mid-batch accounting, and memoization
This commit is contained in:
+237
-101
@@ -25,6 +25,7 @@ import json
|
|||||||
import re
|
import re
|
||||||
import html
|
import html
|
||||||
import io
|
import io
|
||||||
|
import tempfile
|
||||||
import zipfile
|
import zipfile
|
||||||
from urllib.parse import parse_qs, quote, unquote, urlparse
|
from urllib.parse import parse_qs, quote, unquote, urlparse
|
||||||
from html.parser import HTMLParser as _HTMLParser
|
from html.parser import HTMLParser as _HTMLParser
|
||||||
@@ -642,6 +643,11 @@ def _uid_from_fetch_meta(meta_b: bytes) -> str:
|
|||||||
return m.group(1).decode() if m else ""
|
return m.group(1).decode() if m else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _size_from_fetch_meta(meta_b: bytes) -> int | None:
|
||||||
|
m = re.search(rb"\bRFC822\.SIZE\s+(\d+)\b", meta_b, re.IGNORECASE)
|
||||||
|
return int(m.group(1)) if m else None
|
||||||
|
|
||||||
|
|
||||||
def _parse_list_unsubscribe_header(value: str | None) -> list[dict]:
|
def _parse_list_unsubscribe_header(value: str | None) -> list[dict]:
|
||||||
"""Parse RFC List-Unsubscribe entries into safe reviewable actions.
|
"""Parse RFC List-Unsubscribe entries into safe reviewable actions.
|
||||||
|
|
||||||
@@ -4212,15 +4218,16 @@ def setup_email_routes():
|
|||||||
date_span = selected_dates[0] if len(selected_dates) == 1 else f"{selected_dates[0]} - {selected_dates[-1]}"
|
date_span = selected_dates[0] if len(selected_dates) == 1 else f"{selected_dates[0]} - {selected_dates[-1]}"
|
||||||
export_folder = f"emails-{category}" + (f" ({date_span})" if date_span else "")
|
export_folder = f"emails-{category}" + (f" ({date_span})" if date_span else "")
|
||||||
|
|
||||||
zip_buf = io.BytesIO()
|
# 512 MiB safety cap prevents unbounded memory growth when exporting
|
||||||
|
# many large messages.
|
||||||
|
max_total_bytes = 512 * 1024 * 1024
|
||||||
|
max_message_bytes = 64 * 1024 * 1024
|
||||||
|
spool_threshold_bytes = 32 * 1024 * 1024
|
||||||
used_names: set[str] = set()
|
used_names: set[str] = set()
|
||||||
exported = 0
|
exported = 0
|
||||||
skipped = 0
|
skipped = 0
|
||||||
total_bytes = 0
|
total_bytes = 0
|
||||||
# 512 MiB safety cap prevents unbounded memory growth when exporting
|
validated_accounts: set[str] = set()
|
||||||
# many large messages.
|
|
||||||
max_total_bytes = 512 * 1024 * 1024
|
|
||||||
single_export: tuple[str, bytes] | None = None
|
|
||||||
|
|
||||||
def _email_filename(row: dict, uid: str) -> str:
|
def _email_filename(row: dict, uid: str) -> str:
|
||||||
date = str(row.get("date") or "").strip()
|
date = str(row.get("date") or "").strip()
|
||||||
@@ -4281,120 +4288,249 @@ def setup_email_routes():
|
|||||||
logger.debug("Skipping malformed fixture email attachment uid=%s", uid, exc_info=True)
|
logger.debug("Skipping malformed fixture email attachment uid=%s", uid, exc_info=True)
|
||||||
return msg.as_bytes(policy=email_policy.SMTP)
|
return msg.as_bytes(policy=email_policy.SMTP)
|
||||||
|
|
||||||
seen_items: set[tuple[str | None, str, str]] = set()
|
# Optimization: single message exports return raw .eml directly without ZIP compression.
|
||||||
remote_groups: dict[tuple[str | None, str], dict[str, tuple[str, dict]]] = {}
|
if len(messages) == 1:
|
||||||
with zipfile.ZipFile(zip_buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
row = messages[0]
|
||||||
for row in messages:
|
if not isinstance(row, dict):
|
||||||
if not isinstance(row, dict):
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
skipped += 1
|
uid = str(row.get("uid") or "").strip()
|
||||||
continue
|
if not uid:
|
||||||
uid = str(row.get("uid") or "").strip()
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
if not uid:
|
folder = str(row.get("folder") or "INBOX").strip() or "INBOX"
|
||||||
skipped += 1
|
account_id = str(row.get("account_id") or row.get("account") or "").strip() or None
|
||||||
continue
|
if account_id:
|
||||||
folder = str(row.get("folder") or "INBOX").strip() or "INBOX"
|
_assert_owns_account(account_id, owner)
|
||||||
account_id = str(row.get("account_id") or row.get("account") or "").strip() or None
|
|
||||||
if account_id:
|
|
||||||
_assert_owns_account(account_id, owner)
|
|
||||||
|
|
||||||
item_key = (account_id, folder, uid)
|
filename = _email_filename(row, uid)
|
||||||
if item_key in seen_items:
|
message_bytes: bytes | None = None
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
seen_items.add(item_key)
|
|
||||||
|
|
||||||
fixture = _fixture_email_read(uid, folder, owner)
|
fixture = _fixture_email_read(uid, folder, owner)
|
||||||
if fixture is not None and not fixture.get("error"):
|
if fixture is not None and not fixture.get("error"):
|
||||||
filename = _email_filename(row, uid)
|
message_bytes = _fixture_message_bytes(fixture, uid, folder)
|
||||||
unique_name = filename
|
elif re.fullmatch(r"\d+", uid):
|
||||||
suffix = 2
|
|
||||||
while unique_name.lower() in used_names:
|
|
||||||
unique_name = f"{Path(filename).stem}-{suffix}.eml"
|
|
||||||
suffix += 1
|
|
||||||
used_names.add(unique_name.lower())
|
|
||||||
message_bytes = _fixture_message_bytes(fixture, uid, folder)
|
|
||||||
total_bytes += len(message_bytes)
|
|
||||||
if total_bytes > max_total_bytes:
|
|
||||||
raise HTTPException(status_code=413, detail="Export too large")
|
|
||||||
zf.writestr(f"{export_folder}/{unique_name}", message_bytes)
|
|
||||||
single_export = (unique_name, message_bytes)
|
|
||||||
exported += 1
|
|
||||||
continue
|
|
||||||
if not re.fullmatch(r"\d+", uid):
|
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
group = remote_groups.setdefault((account_id, folder), {})
|
|
||||||
group.setdefault(uid, (_email_filename(row, uid), row))
|
|
||||||
|
|
||||||
# Fetch original RFC822 data in batches and store it byte-for-byte.
|
|
||||||
for (account_id, folder), rows_by_uid in remote_groups.items():
|
|
||||||
rows = list(rows_by_uid.items())
|
|
||||||
remaining_rows = len(rows)
|
|
||||||
try:
|
try:
|
||||||
with _imap(account_id, owner=owner) as conn:
|
with _imap(account_id, owner=owner) as conn:
|
||||||
conn.select(_q(folder), readonly=True)
|
conn.select(_q(folder), readonly=True)
|
||||||
for batch_start in range(0, len(rows), 20):
|
uid_b = uid.encode()
|
||||||
batch = rows[batch_start:batch_start + 20]
|
try:
|
||||||
wanted = {uid: filename for uid, (filename, _row) in batch}
|
st_sz, sz_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822.SIZE)")
|
||||||
uid_set = ",".join(wanted).encode()
|
if st_sz == "OK" and sz_data:
|
||||||
status, msg_data = _imap_uid_fetch(conn, uid_set, "(UID RFC822)")
|
for item in sz_data:
|
||||||
if status != "OK" or not msg_data:
|
meta = item[0] if isinstance(item, tuple) else item
|
||||||
skipped += len(batch)
|
if isinstance(meta, bytes):
|
||||||
remaining_rows -= len(batch)
|
sz = _size_from_fetch_meta(meta)
|
||||||
continue
|
if sz is not None and (sz > max_message_bytes or sz > max_total_bytes):
|
||||||
found: set[str] = set()
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
for meta, raw_message in _group_uid_fetch_records(msg_data):
|
except HTTPException:
|
||||||
uid = _uid_from_fetch_meta(meta)
|
raise
|
||||||
if uid not in wanted or not raw_message:
|
except Exception as sz_err:
|
||||||
continue
|
logger.debug("Single message size preflight error: %s", sz_err)
|
||||||
filename = wanted[uid]
|
|
||||||
unique_name = filename
|
st, msg_data = _imap_uid_fetch(conn, uid_b, "(UID RFC822)")
|
||||||
suffix = 2
|
if st == "OK" and msg_data:
|
||||||
while unique_name.lower() in used_names:
|
for meta, raw in _group_uid_fetch_records(msg_data):
|
||||||
unique_name = f"{Path(filename).stem}-{suffix}.eml"
|
if _uid_from_fetch_meta(meta) == uid and raw:
|
||||||
suffix += 1
|
message_bytes = bytes(raw)
|
||||||
used_names.add(unique_name.lower())
|
break
|
||||||
message_bytes = bytes(raw_message)
|
|
||||||
total_bytes += len(message_bytes)
|
|
||||||
if total_bytes > max_total_bytes:
|
|
||||||
raise HTTPException(status_code=413, detail="Export too large")
|
|
||||||
zf.writestr(f"{export_folder}/{unique_name}", message_bytes)
|
|
||||||
single_export = (unique_name, message_bytes)
|
|
||||||
found.add(uid)
|
|
||||||
exported += 1
|
|
||||||
skipped += len(batch) - len(found)
|
|
||||||
remaining_rows -= len(batch)
|
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
skipped += remaining_rows
|
logger.warning("Single email export failed for uid=%s folder=%s: %s", uid, folder, exc)
|
||||||
logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, remaining_rows, exc)
|
|
||||||
|
if not message_bytes:
|
||||||
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
|
|
||||||
|
if len(message_bytes) > max_message_bytes or len(message_bytes) > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
|
||||||
if exported <= 0:
|
|
||||||
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
|
||||||
if len(messages) == 1 and single_export:
|
|
||||||
filename, message_bytes = single_export
|
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
io.BytesIO(message_bytes),
|
io.BytesIO(message_bytes),
|
||||||
media_type="message/rfc822",
|
media_type="message/rfc822",
|
||||||
headers={
|
headers={
|
||||||
"Content-Disposition": f"attachment; filename=\"email.eml\"; filename*=UTF-8''{quote(filename, safe='')}",
|
"Content-Disposition": f"attachment; filename=\"email.eml\"; filename*=UTF-8''{quote(filename, safe='')}",
|
||||||
"X-Exported-Emails": "1",
|
"X-Exported-Emails": "1",
|
||||||
|
"X-Skipped-Emails": "0",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
seen_items: set[tuple[str | None, str, str]] = set()
|
||||||
|
remote_groups: dict[tuple[str | None, str], dict[str, tuple[str, dict]]] = {}
|
||||||
|
spooled_file = tempfile.SpooledTemporaryFile(max_size=spool_threshold_bytes, mode="w+b")
|
||||||
|
response_started = False
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(spooled_file, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||||
|
for row in messages:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
uid = str(row.get("uid") or "").strip()
|
||||||
|
if not uid:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
folder = str(row.get("folder") or "INBOX").strip() or "INBOX"
|
||||||
|
account_id = str(row.get("account_id") or row.get("account") or "").strip() or None
|
||||||
|
if account_id and account_id not in validated_accounts:
|
||||||
|
_assert_owns_account(account_id, owner)
|
||||||
|
validated_accounts.add(account_id)
|
||||||
|
|
||||||
|
item_key = (account_id, folder, uid)
|
||||||
|
if item_key in seen_items:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
seen_items.add(item_key)
|
||||||
|
|
||||||
|
fixture = _fixture_email_read(uid, folder, owner)
|
||||||
|
if fixture is not None and not fixture.get("error"):
|
||||||
|
filename = _email_filename(row, uid)
|
||||||
|
unique_name = filename
|
||||||
|
suffix = 2
|
||||||
|
while unique_name.lower() in used_names:
|
||||||
|
unique_name = f"{Path(filename).stem}-{suffix}.eml"
|
||||||
|
suffix += 1
|
||||||
|
used_names.add(unique_name.lower())
|
||||||
|
message_bytes = _fixture_message_bytes(fixture, uid, folder)
|
||||||
|
if len(message_bytes) > max_message_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
total_bytes += len(message_bytes)
|
||||||
|
if total_bytes > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
try:
|
||||||
|
zf.writestr(f"{export_folder}/{unique_name}", message_bytes)
|
||||||
|
except Exception as zip_err:
|
||||||
|
logger.error("ZIP entry write failed for fixture: %s", zip_err)
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to create export archive")
|
||||||
|
exported += 1
|
||||||
|
continue
|
||||||
|
if not re.fullmatch(r"\d+", uid):
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
group = remote_groups.setdefault((account_id, folder), {})
|
||||||
|
group.setdefault(uid, (_email_filename(row, uid), row))
|
||||||
|
|
||||||
|
# Fetch original RFC822 data in batches and store it byte-for-byte.
|
||||||
|
for (account_id, folder), rows_by_uid in remote_groups.items():
|
||||||
|
rows = list(rows_by_uid.items())
|
||||||
|
remaining_rows = len(rows)
|
||||||
|
try:
|
||||||
|
with _imap(account_id, owner=owner) as conn:
|
||||||
|
conn.select(_q(folder), readonly=True)
|
||||||
|
|
||||||
|
# Size preflight query to catch oversized exports early and bound batch sizes.
|
||||||
|
uid_set_all = ",".join(uid for uid, _ in rows).encode()
|
||||||
|
size_by_uid: dict[str, int] = {}
|
||||||
|
try:
|
||||||
|
st_sz, sz_data = _imap_uid_fetch(conn, uid_set_all, "(UID RFC822.SIZE)")
|
||||||
|
if st_sz == "OK" and sz_data:
|
||||||
|
for item in sz_data:
|
||||||
|
meta = item[0] if isinstance(item, tuple) else item
|
||||||
|
if isinstance(meta, bytes):
|
||||||
|
u = _uid_from_fetch_meta(meta)
|
||||||
|
s = _size_from_fetch_meta(meta)
|
||||||
|
if u and s is not None:
|
||||||
|
size_by_uid[u] = s
|
||||||
|
except Exception as sz_err:
|
||||||
|
logger.debug("IMAP size preflight query failed or unsupported: %s", sz_err)
|
||||||
|
|
||||||
|
if size_by_uid:
|
||||||
|
for u, s in size_by_uid.items():
|
||||||
|
if s > max_message_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
if total_bytes + sum(size_by_uid.values()) > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
|
||||||
|
# Form batches with at most 20 messages and capped at 10 MiB payload.
|
||||||
|
batches: list[list[tuple[str, tuple[str, dict]]]] = []
|
||||||
|
cur_batch: list[tuple[str, tuple[str, dict]]] = []
|
||||||
|
cur_batch_bytes = 0
|
||||||
|
max_batch_bytes = 10 * 1024 * 1024
|
||||||
|
for item in rows:
|
||||||
|
uid_val = item[0]
|
||||||
|
sz = size_by_uid.get(uid_val, 0)
|
||||||
|
if cur_batch and (
|
||||||
|
len(cur_batch) >= 20 or
|
||||||
|
(sz > 0 and cur_batch_bytes + sz > max_batch_bytes)
|
||||||
|
):
|
||||||
|
batches.append(cur_batch)
|
||||||
|
cur_batch = []
|
||||||
|
cur_batch_bytes = 0
|
||||||
|
cur_batch.append(item)
|
||||||
|
cur_batch_bytes += sz
|
||||||
|
if cur_batch:
|
||||||
|
batches.append(cur_batch)
|
||||||
|
|
||||||
|
for batch in batches:
|
||||||
|
wanted = {uid: filename for uid, (filename, _row) in batch}
|
||||||
|
uid_set = ",".join(wanted).encode()
|
||||||
|
status, msg_data = _imap_uid_fetch(conn, uid_set, "(UID RFC822)")
|
||||||
|
if status != "OK" or not msg_data:
|
||||||
|
skipped += len(batch)
|
||||||
|
remaining_rows -= len(batch)
|
||||||
|
continue
|
||||||
|
found: set[str] = set()
|
||||||
|
for meta, raw_message in _group_uid_fetch_records(msg_data):
|
||||||
|
uid = _uid_from_fetch_meta(meta)
|
||||||
|
if uid not in wanted or not raw_message:
|
||||||
|
continue
|
||||||
|
filename = wanted[uid]
|
||||||
|
unique_name = filename
|
||||||
|
suffix = 2
|
||||||
|
while unique_name.lower() in used_names:
|
||||||
|
unique_name = f"{Path(filename).stem}-{suffix}.eml"
|
||||||
|
suffix += 1
|
||||||
|
used_names.add(unique_name.lower())
|
||||||
|
message_bytes = bytes(raw_message)
|
||||||
|
if len(message_bytes) > max_message_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
total_bytes += len(message_bytes)
|
||||||
|
if total_bytes > max_total_bytes:
|
||||||
|
raise HTTPException(status_code=413, detail="Export too large")
|
||||||
|
try:
|
||||||
|
zf.writestr(f"{export_folder}/{unique_name}", message_bytes)
|
||||||
|
except Exception as zip_err:
|
||||||
|
logger.error("ZIP entry write failed for remote message: %s", zip_err)
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to create export archive")
|
||||||
|
found.add(uid)
|
||||||
|
exported += 1
|
||||||
|
remaining_rows -= 1
|
||||||
|
missing = len(batch) - len(found)
|
||||||
|
skipped += len(batch) - len(found)
|
||||||
|
remaining_rows -= missing
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
skipped += remaining_rows
|
||||||
|
logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, remaining_rows, exc)
|
||||||
|
|
||||||
|
if exported <= 0:
|
||||||
|
raise HTTPException(status_code=404, detail="No selected emails could be exported")
|
||||||
|
|
||||||
|
spooled_file.seek(0)
|
||||||
|
zip_name = _safe_attachment_zip_name(f"{export_folder}.zip", "selected-emails.zip")
|
||||||
|
logger.info("Bulk email export complete owner=%s selected=%s exported=%s skipped=%s", owner, len(messages), exported, skipped)
|
||||||
|
|
||||||
|
def _iter_spooled(f, chunk_size=64 * 1024):
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
chunk = f.read(chunk_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
response_started = True
|
||||||
|
return StreamingResponse(
|
||||||
|
_iter_spooled(spooled_file),
|
||||||
|
media_type="application/zip",
|
||||||
|
headers={
|
||||||
|
"Content-Disposition": f'attachment; filename="{zip_name}"',
|
||||||
|
"X-Exported-Emails": str(exported),
|
||||||
"X-Skipped-Emails": str(skipped),
|
"X-Skipped-Emails": str(skipped),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
zip_buf.seek(0)
|
finally:
|
||||||
zip_name = _safe_attachment_zip_name(f"{export_folder}.zip", "selected-emails.zip")
|
if not response_started:
|
||||||
logger.info("Bulk email export complete owner=%s selected=%s exported=%s skipped=%s", owner, len(messages), exported, skipped)
|
spooled_file.close()
|
||||||
return StreamingResponse(
|
|
||||||
zip_buf,
|
|
||||||
media_type="application/zip",
|
|
||||||
headers={
|
|
||||||
"Content-Disposition": f'attachment; filename="{zip_name}"',
|
|
||||||
"X-Exported-Emails": str(exported),
|
|
||||||
"X-Skipped-Emails": str(skipped),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
@router.get("/inline-image/{uid}")
|
@router.get("/inline-image/{uid}")
|
||||||
async def inline_image(
|
async def inline_image(
|
||||||
|
|||||||
@@ -1132,32 +1132,70 @@ async function _printSelectedEmails(messages) {
|
|||||||
while (nextIndex < messages.length) {
|
while (nextIndex < messages.length) {
|
||||||
const index = nextIndex++;
|
const index = nextIndex++;
|
||||||
const message = messages[index];
|
const message = messages[index];
|
||||||
const account = message.account_id ? `&account_id=${encodeURIComponent(message.account_id)}` : '';
|
try {
|
||||||
const url = `${API_BASE}/api/email/read/${encodeURIComponent(message.uid)}?folder=${encodeURIComponent(message.folder || 'INBOX')}${account}&mark_seen=false&full=true`;
|
const account = message.account_id ? `&account_id=${encodeURIComponent(message.account_id)}` : '';
|
||||||
const response = await fetch(url, { credentials: 'same-origin' });
|
const url = `${API_BASE}/api/email/read/${encodeURIComponent(message.uid)}?folder=${encodeURIComponent(message.folder || 'INBOX')}${account}&mark_seen=false&full=true`;
|
||||||
if (!response.ok) throw new Error(`Could not load "${message.subject || 'email'}" (HTTP ${response.status})`);
|
const response = await fetch(url, { credentials: 'same-origin' });
|
||||||
const data = await response.json();
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
if (data?.error) throw new Error(data.error);
|
const data = await response.json();
|
||||||
loaded[index] = { message, data };
|
if (data?.error) throw new Error(data.error);
|
||||||
|
loaded[index] = { ok: true, message, data };
|
||||||
|
} catch (err) {
|
||||||
|
loaded[index] = { ok: false, message, error: err?.message || 'Failed to load' };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
await Promise.all(Array.from({ length: Math.min(4, messages.length) }, readWorker));
|
await Promise.all(Array.from({ length: Math.min(4, messages.length) }, readWorker));
|
||||||
|
|
||||||
|
const successful = loaded.filter(item => item && item.ok);
|
||||||
|
const omitted = loaded.filter(item => !item || !item.ok);
|
||||||
|
|
||||||
|
if (successful.length === 0) {
|
||||||
|
throw new Error('None of the selected emails could be loaded for printing');
|
||||||
|
}
|
||||||
|
|
||||||
// Build sanitized HTML for the print frame -- scripts and event handlers
|
// Build sanitized HTML for the print frame -- scripts and event handlers
|
||||||
// cannot execute inside the sandboxed iframe.
|
// cannot execute inside the sandboxed iframe.
|
||||||
const articles = [];
|
const articles = [];
|
||||||
for (const { message, data } of loaded) {
|
if (omitted.length > 0) {
|
||||||
|
const count = omitted.length;
|
||||||
|
articles.push(
|
||||||
|
`<div class="email-export-pdf-omitted-notice">Note: ${count} selected email${count === 1 ? ' was' : 's were'} unavailable and omitted from this print.</div>`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const { message, data } of successful) {
|
||||||
const subject = String(data.subject || message.subject || '(no subject)');
|
const subject = String(data.subject || message.subject || '(no subject)');
|
||||||
const from = data.from_name && data.from_address
|
const from = data.from_name && data.from_address
|
||||||
? `${data.from_name} <${data.from_address}>`
|
? `${data.from_name} <${data.from_address}>`
|
||||||
: data.from_name || data.from_address;
|
: data.from_name || data.from_address;
|
||||||
|
|
||||||
const safeBody = new DOMParser().parseFromString(_safeRenderEmailBody(data), 'text/html');
|
const safeBody = new DOMParser().parseFromString(_safeRenderEmailBody(data), 'text/html');
|
||||||
safeBody.querySelectorAll('img').forEach(img => {
|
|
||||||
const placeholder = safeBody.createElement('span');
|
// Replace interactive image placeholder components with clean print-safe notices
|
||||||
placeholder.textContent = img.alt ? `[Image: ${img.alt}]` : '[Inline image omitted]';
|
safeBody.querySelectorAll('.email-inline-image-placeholder').forEach(ph => {
|
||||||
img.replaceWith(placeholder);
|
const title = ph.querySelector('.email-inline-image-title')?.textContent?.trim();
|
||||||
|
const ariaLabel = ph.getAttribute('aria-label')?.trim();
|
||||||
|
const label = title || ariaLabel || '';
|
||||||
|
const isGeneric = !label || /^inline image$|^remote image$|^image unavailable$|^image$/i.test(label);
|
||||||
|
const span = safeBody.createElement('span');
|
||||||
|
span.className = 'email-export-image-notice';
|
||||||
|
span.textContent = isGeneric ? '[Image omitted]' : `[Image: ${label}]`;
|
||||||
|
ph.replaceWith(span);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Replace any remaining raw <img> tags
|
||||||
|
safeBody.querySelectorAll('img').forEach(img => {
|
||||||
|
const alt = (img.getAttribute('alt') || img.getAttribute('title') || '').trim();
|
||||||
|
const span = safeBody.createElement('span');
|
||||||
|
span.className = 'email-export-image-notice';
|
||||||
|
span.textContent = alt ? `[Image: ${alt}]` : '[Image omitted]';
|
||||||
|
img.replaceWith(span);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Strip interactive controls (buttons, inputs, actions) from the print document
|
||||||
|
safeBody.querySelectorAll('button, input, select, textarea, .email-inline-image-actions, .email-inline-image-btn').forEach(el => el.remove());
|
||||||
|
|
||||||
safeBody.querySelectorAll('details').forEach(detail => { detail.open = true; });
|
safeBody.querySelectorAll('details').forEach(detail => { detail.open = true; });
|
||||||
const bodyHtml = safeBody.body.innerHTML;
|
const bodyHtml = safeBody.body.innerHTML;
|
||||||
|
|
||||||
@@ -1192,11 +1230,14 @@ async function _printSelectedEmails(messages) {
|
|||||||
'.email-export-pdf-body a{color:#135f9b;text-decoration:underline}',
|
'.email-export-pdf-body a{color:#135f9b;text-decoration:underline}',
|
||||||
'.email-export-pdf-body blockquote{margin-left:12px;padding-left:10px;border-left:2px solid #ccc;color:#555}',
|
'.email-export-pdf-body blockquote{margin-left:12px;padding-left:10px;border-left:2px solid #ccc;color:#555}',
|
||||||
'.email-export-pdf-body pre{white-space:pre-wrap}',
|
'.email-export-pdf-body pre{white-space:pre-wrap}',
|
||||||
|
'.email-inline-image-placeholder,.email-inline-image-actions,.email-inline-image-btn{display:none !important}',
|
||||||
|
'.email-export-image-notice{display:inline-block;color:#666;font-style:italic;font-size:11px;margin:4px 0}',
|
||||||
|
'.email-export-pdf-omitted-notice{margin:0 0 16px;padding:8px 12px;background:#f8f9fa;border:1px solid #e2e8f0;border-radius:4px;font-size:11px;color:#64748b}',
|
||||||
'@page{margin:10mm}',
|
'@page{margin:10mm}',
|
||||||
].join('\n');
|
].join('\n');
|
||||||
|
|
||||||
const safeBaseName = messages.length === 1
|
const safeBaseName = successful.length === 1
|
||||||
? String(loaded[0]?.data?.subject || messages[0]?.subject || 'email').replace(/[^\p{L}\p{N} ._-]+/gu, '_').trim().slice(0, 100) || 'email'
|
? String(successful[0]?.data?.subject || successful[0]?.message?.subject || 'email').replace(/[^\p{L}\p{N} ._-]+/gu, '_').trim().slice(0, 100) || 'email'
|
||||||
: 'selected-emails';
|
: 'selected-emails';
|
||||||
|
|
||||||
// Use a sandboxed same-origin iframe following the established document.js
|
// Use a sandboxed same-origin iframe following the established document.js
|
||||||
@@ -1209,6 +1250,17 @@ async function _printSelectedEmails(messages) {
|
|||||||
frame.setAttribute('sandbox', 'allow-same-origin allow-modals');
|
frame.setAttribute('sandbox', 'allow-same-origin allow-modals');
|
||||||
frame.style.cssText = 'position:fixed;left:-10000px;top:0;width:800px;height:600px;border:0;';
|
frame.style.cssText = 'position:fixed;left:-10000px;top:0;width:800px;height:600px;border:0;';
|
||||||
document.body.appendChild(frame);
|
document.body.appendChild(frame);
|
||||||
|
|
||||||
|
let cleanedUp = false;
|
||||||
|
const cleanupFrame = () => {
|
||||||
|
if (cleanedUp) return;
|
||||||
|
cleanedUp = true;
|
||||||
|
try {
|
||||||
|
frame.remove();
|
||||||
|
} catch (_) {}
|
||||||
|
window.removeEventListener('afterprint', cleanupFrame);
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const printDoc = frame.contentDocument;
|
const printDoc = frame.contentDocument;
|
||||||
printDoc.open();
|
printDoc.open();
|
||||||
@@ -1223,11 +1275,17 @@ async function _printSelectedEmails(messages) {
|
|||||||
container.innerHTML = articles.join('');
|
container.innerHTML = articles.join('');
|
||||||
printDoc.body.appendChild(container);
|
printDoc.body.appendChild(container);
|
||||||
await printDoc.fonts.ready;
|
await printDoc.fonts.ready;
|
||||||
frame.contentWindow.addEventListener('afterprint', () => frame.remove(), { once: true });
|
frame.contentWindow?.addEventListener('afterprint', cleanupFrame, { once: true });
|
||||||
|
window.addEventListener('afterprint', cleanupFrame, { once: true });
|
||||||
|
setTimeout(cleanupFrame, 60000);
|
||||||
frame.contentWindow.focus();
|
frame.contentWindow.focus();
|
||||||
frame.contentWindow.print();
|
frame.contentWindow.print();
|
||||||
|
if (omitted.length > 0) {
|
||||||
|
const msg = `Printed ${successful.length} email${successful.length === 1 ? '' : 's'}; ${omitted.length} unavailable`;
|
||||||
|
showToast(msg);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
frame.remove();
|
cleanupFrame();
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -411,3 +411,161 @@ def test_messages_download_bulk_enforces_account_ownership_guard(monkeypatch):
|
|||||||
|
|
||||||
assert res.status_code == 404
|
assert res.status_code == 404
|
||||||
assert "Account not found" in res.text
|
assert "Account not found" in res.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_mid_batch_failure_accounting(monkeypatch):
|
||||||
|
"""When an exception occurs mid-batch after partial progress, already-exported
|
||||||
|
records must not be double-counted as skipped."""
|
||||||
|
class MidBatchFailImap:
|
||||||
|
def select(self, folder, readonly=True):
|
||||||
|
return "OK", [b"1"]
|
||||||
|
def uid(self, command, *args):
|
||||||
|
records = []
|
||||||
|
for i in range(1, 5):
|
||||||
|
records.append((f"{i} (UID {i} RFC822 {{{10}}}".encode(), f"Subject: Msg {i}\r\n\r\nBody {i}".encode()))
|
||||||
|
class BadRecord:
|
||||||
|
def __bytes__(self):
|
||||||
|
raise RuntimeError("Corrupt memory buffer")
|
||||||
|
records.append((b"5 (UID 5 RFC822 {10}", BadRecord()))
|
||||||
|
return "OK", records
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from routes.email import email_routes as routes
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def mock_imap(account_id=None, owner=""):
|
||||||
|
yield MidBatchFailImap()
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes, "_start_poller", lambda: None)
|
||||||
|
monkeypatch.setattr(routes, "_POOL_HOOKS", {})
|
||||||
|
monkeypatch.setattr(routes, "_imap", mock_imap)
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(routes.setup_email_routes())
|
||||||
|
app.dependency_overrides[routes.require_owner] = lambda: "test-user"
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
rows = [{"uid": str(i)} for i in range(1, 11)]
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": rows})
|
||||||
|
assert res.status_code == 200
|
||||||
|
exp = int(res.headers.get("x-exported-emails"))
|
||||||
|
skp = int(res.headers.get("x-skipped-emails"))
|
||||||
|
assert exp == 4
|
||||||
|
assert skp == 6
|
||||||
|
assert exp + skp == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_size_cap_413_boundary(monkeypatch):
|
||||||
|
"""Cumulative export size exceeding max_total_bytes returns HTTP 413."""
|
||||||
|
class LargeMessageImap:
|
||||||
|
def select(self, folder, readonly=True):
|
||||||
|
return "OK", [b"1"]
|
||||||
|
def uid(self, command, *args):
|
||||||
|
raw_uids = args[0] if isinstance(args[0], str) else args[0].decode()
|
||||||
|
if "(UID RFC822.SIZE)" in str(args):
|
||||||
|
records = []
|
||||||
|
for u in raw_uids.split(","):
|
||||||
|
records.append(f"{u} (UID {u} RFC822.SIZE 10485760)".encode())
|
||||||
|
return "OK", records
|
||||||
|
records = []
|
||||||
|
big_chunk = b"X" * (10 * 1024 * 1024)
|
||||||
|
for u in raw_uids.split(","):
|
||||||
|
records.append((f"{u} (UID {u} RFC822".encode(), big_chunk))
|
||||||
|
return "OK", records
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from routes.email import email_routes as routes
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def mock_imap(account_id=None, owner=""):
|
||||||
|
yield LargeMessageImap()
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes, "_start_poller", lambda: None)
|
||||||
|
monkeypatch.setattr(routes, "_POOL_HOOKS", {})
|
||||||
|
monkeypatch.setattr(routes, "_imap", mock_imap)
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(routes.setup_email_routes())
|
||||||
|
app.dependency_overrides[routes.require_owner] = lambda: "test-user"
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
# 52 messages of 10 MB = 520 MB > 512 MiB limit
|
||||||
|
payload_52 = [{"uid": str(i + 1), "subject": f"Big {i}"} for i in range(52)]
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": payload_52})
|
||||||
|
assert res.status_code == 413
|
||||||
|
assert res.json()["detail"] == "Export too large"
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_per_message_limit_413(monkeypatch):
|
||||||
|
"""A single oversized message (>64 MiB) triggers HTTP 413."""
|
||||||
|
class HugeSingleMsgImap:
|
||||||
|
def select(self, folder, readonly=True):
|
||||||
|
return "OK", [b"1"]
|
||||||
|
def uid(self, command, *args):
|
||||||
|
return "OK", [b"1 (UID 101 RFC822.SIZE 73400320)"]
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from routes.email import email_routes as routes
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def mock_imap(account_id=None, owner=""):
|
||||||
|
yield HugeSingleMsgImap()
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes, "_start_poller", lambda: None)
|
||||||
|
monkeypatch.setattr(routes, "_POOL_HOOKS", {})
|
||||||
|
monkeypatch.setattr(routes, "_imap", mock_imap)
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(routes.setup_email_routes())
|
||||||
|
app.dependency_overrides[routes.require_owner] = lambda: "test-user"
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={
|
||||||
|
"messages": [{"uid": "101", "subject": "Huge Email"}]
|
||||||
|
})
|
||||||
|
assert res.status_code == 413
|
||||||
|
assert res.json()["detail"] == "Export too large"
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_account_ownership_memoization(monkeypatch):
|
||||||
|
"""Account ownership validation is memoized per distinct account_id to eliminate N+1 DB queries."""
|
||||||
|
owns_calls = []
|
||||||
|
def counting_assert_owns(account_id, owner):
|
||||||
|
owns_calls.append(account_id)
|
||||||
|
|
||||||
|
raw_rfc822 = b"From: sender@example.com\r\nSubject: Test\r\n\r\nBody"
|
||||||
|
msgs = {str(i): raw_rfc822 for i in range(1, 26)}
|
||||||
|
mailbox = _MockImapConn(msgs)
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox, assert_owns_fn=counting_assert_owns)
|
||||||
|
|
||||||
|
# 25 messages all referencing the same account_id
|
||||||
|
payload = [{"uid": str(i), "account_id": "acc-shared"} for i in range(1, 26)]
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={"messages": payload})
|
||||||
|
assert res.status_code == 200
|
||||||
|
assert len(owns_calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_messages_download_bulk_corrupt_zip_fails_closed(monkeypatch):
|
||||||
|
"""If ZIP construction fails (e.g. disk error), fail closed with 500 instead of returning corrupt archive."""
|
||||||
|
import zipfile
|
||||||
|
raw_rfc822 = b"From: sender@example.com\r\nSubject: Test\r\n\r\nBody"
|
||||||
|
mailbox = _MockImapConn({"101": raw_rfc822, "102": raw_rfc822})
|
||||||
|
|
||||||
|
orig_writestr = zipfile.ZipFile.writestr
|
||||||
|
def failing_writestr(self, zinfo_or_arcname, data=None, *args, **kwargs):
|
||||||
|
raise OSError("Disk write failed: No space left on device")
|
||||||
|
|
||||||
|
monkeypatch.setattr(zipfile.ZipFile, "writestr", failing_writestr)
|
||||||
|
client = _create_test_app(monkeypatch, mailbox=mailbox)
|
||||||
|
|
||||||
|
res = client.post("/api/email/messages-download-bulk", json={
|
||||||
|
"messages": [{"uid": "101"}, {"uid": "102"}]
|
||||||
|
})
|
||||||
|
assert res.status_code == 500
|
||||||
|
assert "Failed to create export archive" in res.json()["detail"]
|
||||||
|
|||||||
Reference in New Issue
Block a user