From 6cf676dfe29fc6d43e3abd012f4f36d58edcbc3d Mon Sep 17 00:00:00 2001 From: pewdiepie-archdaemon Date: Thu, 15 Oct 2026 12:02:06 +0000 Subject: [PATCH] feat(email): export selected messages as email or PDF --- routes/email/email_routes.py | 190 ++++++++++++++++++++++- static/js/emailLibrary/index.js | 165 +++++++++++++++++++- static/js/emailLibrary/menus.js | 46 +++++- tests/test_email_library_bulk_actions.py | 31 +++- 4 files changed, 415 insertions(+), 17 deletions(-) diff --git a/routes/email/email_routes.py b/routes/email/email_routes.py index 172790a6c..99b429f6e 100644 --- a/routes/email/email_routes.py +++ b/routes/email/email_routes.py @@ -26,12 +26,14 @@ import re import html import io import zipfile -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse from html.parser import HTMLParser as _HTMLParser import logging import uuid -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path +from email.message import EmailMessage +from email import policy as email_policy from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart @@ -4185,6 +4187,190 @@ def setup_email_routes(): headers={"Content-Disposition": f'attachment; filename="{zip_name}"'}, ) + @router.post("/messages-download-bulk") + async def download_bulk_messages(request: Request, owner: str = Depends(require_owner)): + """Export selected original email messages as .eml files in a ZIP.""" + try: + payload = await request.json() + except Exception: + raise HTTPException(status_code=400, detail="Invalid JSON body") + messages = payload.get("messages") if isinstance(payload, dict) else None + if not isinstance(messages, list) or not messages: + raise HTTPException(status_code=400, detail="No emails selected") + if len(messages) > 250: + raise HTTPException(status_code=400, detail="Too many emails selected") + + category = str(payload.get("category") or "selected").strip().lower() + category = re.sub(r"[^a-z0-9_-]+", "-", category).strip("-") or "selected" + selected_dates = sorted({ + str(row.get("date") or "").strip() + for row in messages if isinstance(row, dict) + and re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(row.get("date") or "").strip()) + }) + date_span = "" + if selected_dates: + date_span = selected_dates[0] if len(selected_dates) == 1 else f"{selected_dates[0]} - {selected_dates[-1]}" + export_folder = f"emails-{category}" + (f" ({date_span})" if date_span else "") + + zip_buf = io.BytesIO() + used_names: set[str] = set() + exported = 0 + skipped = 0 + single_export: tuple[str, bytes] | None = None + + def _email_filename(row: dict, uid: str) -> str: + date = str(row.get("date") or "").strip() + date = date if re.fullmatch(r"\d{4}-\d{2}-\d{2}", date) else "" + subject = str(row.get("subject") or "email").strip() + subject = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]+', "_", subject) + subject = re.sub(r"\s+", " ", subject).strip(" .")[:100] or "email" + safe_uid = re.sub(r"[^A-Za-z0-9_-]+", "_", uid).strip("_") or "message" + prefix = f"{date} - " if date else "" + return _safe_attachment_zip_name(f"{prefix}{subject} - {safe_uid}.eml", "email.eml") + + def _fixture_message_bytes(fixture: dict, uid: str, folder: str) -> bytes: + msg = EmailMessage() + sender_name = str(fixture.get("from_name") or "").strip() + sender_address = str(fixture.get("from_address") or "").strip() + sender = email.utils.formataddr((sender_name, sender_address)) if sender_address else sender_name + for key, value in ( + ("From", sender), + ("To", fixture.get("to") or owner), + ("Subject", fixture.get("subject") or "(no subject)"), + ("Message-ID", fixture.get("message_id") or f""), + ): + clean_value = re.sub(r"[\r\n]+", " ", str(value or "")).strip() + if clean_value: + msg[key] = clean_value + raw_date = str(fixture.get("date") or "").strip() + if raw_date: + try: + parsed_date = datetime.fromisoformat(raw_date.replace("Z", "+00:00")) + if parsed_date.tzinfo is None: + parsed_date = parsed_date.replace(tzinfo=timezone.utc) + except Exception: + try: + parsed_date = email.utils.parsedate_to_datetime(raw_date) + except Exception: + parsed_date = None + if parsed_date is not None: + msg["Date"] = email.utils.format_datetime(parsed_date) + msg.set_content(str(fixture.get("body") or "")) + for attachment in fixture.get("attachments") or []: + try: + index = int(attachment.get("index")) + source = _fixture_attachment_source(uid, index, owner, folder) + if not source: + continue + _row, raw_attachment = source + content_type = str(raw_attachment.get("content_type") or "application/octet-stream") + maintype, _, subtype = content_type.partition("/") + if not subtype: + maintype, subtype = "application", "octet-stream" + msg.add_attachment( + str(raw_attachment.get("content") or "").encode("utf-8"), + maintype=maintype, + subtype=subtype, + filename=str(raw_attachment.get("filename") or attachment.get("filename") or f"attachment-{index}"), + ) + except Exception: + logger.debug("Skipping malformed fixture email attachment uid=%s", uid, exc_info=True) + return msg.as_bytes(policy=email_policy.SMTP) + + remote_groups: dict[tuple[str | None, str], dict[str, tuple[str, dict]]] = {} + with zipfile.ZipFile(zip_buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for row in messages: + if not isinstance(row, dict): + skipped += 1 + continue + uid = str(row.get("uid") or "").strip() + if not uid: + skipped += 1 + continue + folder = str(row.get("folder") or "INBOX").strip() or "INBOX" + account_id = str(row.get("account_id") or row.get("account") or "").strip() or None + fixture = _fixture_email_read(uid, folder, owner) + if fixture is not None and not fixture.get("error"): + filename = _email_filename(row, uid) + unique_name = filename + suffix = 2 + while unique_name.lower() in used_names: + unique_name = f"{Path(filename).stem}-{suffix}.eml" + suffix += 1 + used_names.add(unique_name.lower()) + message_bytes = _fixture_message_bytes(fixture, uid, folder) + zf.writestr(f"{export_folder}/{unique_name}", message_bytes) + single_export = (unique_name, message_bytes) + exported += 1 + continue + if not re.fullmatch(r"\d+", uid): + skipped += 1 + continue + group = remote_groups.setdefault((account_id, folder), {}) + group.setdefault(uid, (_email_filename(row, uid), row)) + + # Fetch original RFC822 data in batches and store it byte-for-byte. + for (account_id, folder), rows_by_uid in remote_groups.items(): + rows = list(rows_by_uid.items()) + try: + with _imap(account_id, owner=owner) as conn: + conn.select(_q(folder), readonly=True) + for batch_start in range(0, len(rows), 20): + batch = rows[batch_start:batch_start + 20] + wanted = {uid: filename for uid, (filename, _row) in batch} + uid_set = ",".join(wanted).encode() + status, msg_data = _imap_uid_fetch(conn, uid_set, "(UID RFC822)") + if status != "OK" or not msg_data: + skipped += len(batch) + continue + found: set[str] = set() + for meta, raw_message in _group_uid_fetch_records(msg_data): + uid = _uid_from_fetch_meta(meta) + if uid not in wanted or not raw_message: + continue + filename = wanted[uid] + unique_name = filename + suffix = 2 + while unique_name.lower() in used_names: + unique_name = f"{Path(filename).stem}-{suffix}.eml" + suffix += 1 + used_names.add(unique_name.lower()) + message_bytes = bytes(raw_message) + zf.writestr(f"{export_folder}/{unique_name}", message_bytes) + single_export = (unique_name, message_bytes) + found.add(uid) + exported += 1 + skipped += len(batch) - len(found) + except Exception as exc: + skipped += len(rows) + logger.warning("Bulk email export skipped mailbox folder=%s account=%s count=%s: %s", folder, account_id, len(rows), exc) + + if exported <= 0: + raise HTTPException(status_code=404, detail="No selected emails could be exported") + if len(messages) == 1 and single_export: + filename, message_bytes = single_export + return StreamingResponse( + io.BytesIO(message_bytes), + media_type="message/rfc822", + headers={ + "Content-Disposition": f"attachment; filename=\"email.eml\"; filename*=UTF-8''{quote(filename, safe='')}", + "X-Exported-Emails": "1", + "X-Skipped-Emails": str(skipped), + }, + ) + zip_buf.seek(0) + zip_name = _safe_attachment_zip_name(f"{export_folder}.zip", "selected-emails.zip") + logger.info("Bulk email export complete owner=%s selected=%s exported=%s skipped=%s", owner, len(messages), exported, skipped) + return StreamingResponse( + zip_buf, + media_type="application/zip", + headers={ + "Content-Disposition": f'attachment; filename="{zip_name}"', + "X-Exported-Emails": str(exported), + "X-Skipped-Emails": str(skipped), + }, + ) + @router.get("/inline-image/{uid}") async def inline_image( uid: str, diff --git a/static/js/emailLibrary/index.js b/static/js/emailLibrary/index.js index 82f76b0c5..7de542fe8 100644 --- a/static/js/emailLibrary/index.js +++ b/static/js/emailLibrary/index.js @@ -1122,7 +1122,130 @@ function _filenameFromContentDisposition(header) { return bare ? bare[1].trim() : ''; } -export async function _exportSelectedAttachments() { +let _emailHtml2PdfReady = null; +async function _loadEmailHtml2Pdf() { + if (window.html2pdf) return; + if (!_emailHtml2PdfReady) { + _emailHtml2PdfReady = new Promise((resolve, reject) => { + const script = document.createElement('script'); + script.src = '/static/lib/html2pdf.bundle.min.js'; + script.onload = resolve; + script.onerror = () => reject(new Error('Could not load PDF exporter')); + document.head.appendChild(script); + }); + } + try { + await _emailHtml2PdfReady; + } catch (error) { + _emailHtml2PdfReady = null; + throw error; + } +} + +async function _downloadSelectedEmailPdf(messages) { + if (messages.length > 20) throw new Error('PDF export supports up to 20 emails at a time'); + await _loadEmailHtml2Pdf(); + + const loaded = new Array(messages.length); + let nextIndex = 0; + const readWorker = async () => { + while (nextIndex < messages.length) { + const index = nextIndex++; + const message = messages[index]; + const account = message.account_id ? `&account_id=${encodeURIComponent(message.account_id)}` : ''; + const url = `${API_BASE}/api/email/read/${encodeURIComponent(message.uid)}?folder=${encodeURIComponent(message.folder || 'INBOX')}${account}&mark_seen=false&full=true`; + const response = await fetch(url, { credentials: 'same-origin' }); + if (!response.ok) throw new Error(`Could not load "${message.subject || 'email'}" (HTTP ${response.status})`); + const data = await response.json(); + if (data?.error) throw new Error(data.error); + loaded[index] = { message, data }; + } + }; + await Promise.all(Array.from({ length: Math.min(4, messages.length) }, readWorker)); + + const page = document.createElement('div'); + page.className = 'email-export-pdf'; + page.style.cssText = 'width:794px;box-sizing:border-box;padding:38px;background:#fff;color:#171717;font:12px/1.55 Arial,Helvetica,sans-serif;'; + const style = document.createElement('style'); + style.textContent = ` + .email-export-pdf-message { page-break-after: always; break-after: page; } + .email-export-pdf-message:last-child { page-break-after: auto; break-after: auto; } + .email-export-pdf-message h1 { margin: 0 0 18px; font-size: 20px; line-height: 1.3; overflow-wrap: anywhere; } + .email-export-pdf-fields { margin: 0 0 20px; padding: 0 0 14px; border-bottom: 1px solid #d8d8d8; } + .email-export-pdf-field { display: flex; gap: 10px; margin: 3px 0; overflow-wrap: anywhere; } + .email-export-pdf-label { width: 54px; flex: 0 0 54px; color: #666; } + .email-export-pdf-body { overflow-wrap: anywhere; word-break: break-word; } + .email-export-pdf-body img { max-width: 100%; } + .email-export-pdf-body a { color: #135f9b; text-decoration: underline; } + .email-export-pdf-body blockquote { margin-left: 12px; padding-left: 10px; border-left: 2px solid #ccc; color: #555; } + .email-export-pdf-body pre { white-space: pre-wrap; } + `; + page.appendChild(style); + + const addField = (parent, label, value) => { + if (!value) return; + const row = document.createElement('div'); + row.className = 'email-export-pdf-field'; + const labelEl = document.createElement('span'); + labelEl.className = 'email-export-pdf-label'; + labelEl.textContent = label; + const valueEl = document.createElement('span'); + valueEl.textContent = value; + row.append(labelEl, valueEl); + parent.appendChild(row); + }; + + for (const { message, data } of loaded) { + const article = document.createElement('article'); + article.className = 'email-export-pdf-message'; + const subject = String(data.subject || message.subject || '(no subject)'); + const title = document.createElement('h1'); + title.textContent = subject; + article.appendChild(title); + + const fields = document.createElement('div'); + fields.className = 'email-export-pdf-fields'; + const from = data.from_name && data.from_address + ? `${data.from_name} <${data.from_address}>` + : data.from_name || data.from_address; + addField(fields, 'From', from); + addField(fields, 'To', data.to); + addField(fields, 'Cc', data.cc); + addField(fields, 'Date', data.date_display || data.date || message.date); + article.appendChild(fields); + + const body = document.createElement('div'); + body.className = 'email-export-pdf-body'; + const safeBody = new DOMParser().parseFromString(_safeRenderEmailBody(data), 'text/html'); + safeBody.querySelectorAll('img').forEach(img => { + const placeholder = safeBody.createElement('span'); + placeholder.textContent = img.alt ? `[Image: ${img.alt}]` : '[Inline image omitted]'; + img.replaceWith(placeholder); + }); + body.innerHTML = safeBody.body.innerHTML; + body.querySelectorAll('details').forEach(detail => { detail.open = true; }); + article.appendChild(body); + page.appendChild(article); + } + + const safeBaseName = messages.length === 1 + ? String(loaded[0]?.data?.subject || messages[0]?.subject || 'email').replace(/[^\p{L}\p{N} ._-]+/gu, '_').trim().slice(0, 100) || 'email' + : 'selected-emails'; + try { + await window.html2pdf().set({ + margin: 10, + filename: `${safeBaseName}.pdf`, + image: { type: 'jpeg', quality: 0.92 }, + html2canvas: { scale: 1.6, useCORS: false, backgroundColor: '#fff' }, + jsPDF: { unit: 'mm', format: 'a4', orientation: 'portrait' }, + pagebreak: { mode: ['css', 'legacy'] }, + }).from(page).save(); + } finally { + page.remove(); + } +} + +async function _exportSelectedBundle(kind) { const messages = _selectedEmailExportPayload(); if (!messages.length) { showToast('Select emails first'); @@ -1150,8 +1273,15 @@ export async function _exportSelectedAttachments() { if (deleteBtn) deleteBtn.disabled = true; if (cancelBtn) cancelBtn.disabled = true; if (selectAll) selectAll.disabled = true; - if (countEl) countEl.textContent = `Exporting attachments from ${messages.length}…`; - const res = await fetch(`${API_BASE}/api/email/attachments-download-bulk`, { + const isEmailExport = kind === 'emails'; + const isPdfExport = kind === 'pdf'; + if (countEl) countEl.textContent = `${isPdfExport ? 'Creating PDF' : 'Exporting'} ${isPdfExport ? 'from' : isEmailExport ? 'emails from' : 'attachments from'} ${messages.length}…`; + if (isPdfExport) { + await _downloadSelectedEmailPdf(messages); + showToast(`Exported ${messages.length} email${messages.length === 1 ? '' : 's'} as PDF`); + return; + } + const res = await fetch(`${API_BASE}/api/email/${isEmailExport ? 'messages' : 'attachments'}-download-bulk`, { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, @@ -1169,12 +1299,21 @@ export async function _exportSelectedAttachments() { throw new Error(detail || `HTTP ${res.status}`); } const blob = await res.blob(); - const filename = _filenameFromContentDisposition(res.headers.get('Content-Disposition')) || 'selected-email-attachments.zip'; + const filename = _filenameFromContentDisposition(res.headers.get('Content-Disposition')) + || (isEmailExport + ? (messages.length === 1 ? 'selected-email.eml' : 'selected-emails.zip') + : 'selected-email-attachments.zip'); _downloadBlob(blob, filename); - showToast(`Exported attachments from ${messages.length} selected email${messages.length === 1 ? '' : 's'}`); + const exported = Number(res.headers.get('X-Exported-Emails')) || messages.length; + const skipped = Number(res.headers.get('X-Skipped-Emails')) || 0; + showToast(skipped + ? `Exported ${exported} email${exported === 1 ? '' : 's'}; ${skipped} unavailable` + : isEmailExport + ? `Exported ${exported} email${exported === 1 ? '' : 's'} as ${exported === 1 ? 'an .eml file' : '.eml files'}` + : `Exported attachments from ${messages.length} selected email${messages.length === 1 ? '' : 's'}`); } catch (err) { - console.error('Failed to export selected email attachments:', err); - showToast(err?.message || 'Failed to export attachments'); + console.error(`Failed to export selected email ${kind}:`, err); + showToast(err?.message || (kind === 'emails' ? 'Failed to export emails' : 'Failed to export attachments')); } finally { if (busySpinner) busySpinner.destroy(); if (actionsBtn) { @@ -1190,6 +1329,18 @@ export async function _exportSelectedAttachments() { } } +export async function _exportSelectedEmails() { + return _exportSelectedBundle('emails'); +} + +export async function _exportSelectedEmailPdf() { + return _exportSelectedBundle('pdf'); +} + +export async function _exportSelectedAttachments() { + return _exportSelectedBundle('attachments'); +} + // URL-suffix helper — appends &account_id=... when an account is actively selected. // Every email route call in this file goes through here so switching accounts // is a single-variable flip. diff --git a/static/js/emailLibrary/menus.js b/static/js/emailLibrary/menus.js index da859ecad..3b03f4aba 100644 --- a/static/js/emailLibrary/menus.js +++ b/static/js/emailLibrary/menus.js @@ -25,6 +25,8 @@ import { _animateEmailCardRemoval, _clearDoneResponseTagsLocal, _emailMutationQuery, + _exportSelectedEmailPdf, + _exportSelectedEmails, _exportSelectedAttachments, _findSiblingEmailCard, _fitEmailDropdown, @@ -585,7 +587,10 @@ export function _showBulkActionsMenu(anchor) { const _unreadIco = ''; const _doneIco = ''; const _exportIco = ''; + const _emailIco = ''; + const _pdfIco = ''; const items = [ + { label: 'Export Emails', icon: _exportIco, submenu: true }, { label: 'Export Attachments', icon: _exportIco, action: () => _exportSelectedAttachments() }, { label: 'Done', icon: _doneIco, action: () => _bulkAction('done') }, { label: 'Mark Read', icon: _readIco, action: () => _bulkAction('read') }, @@ -594,8 +599,45 @@ export function _showBulkActionsMenu(anchor) { for (const a of items) { const it = document.createElement('div'); it.className = 'dropdown-item-compact' + (a.danger ? ' dropdown-item-danger' : ''); - it.innerHTML = `${a.icon}${a.label}`; - it.addEventListener('click', (e) => { e.stopPropagation(); close(); a.action(); }); + it.className += a.submenu ? ' email-bulk-submenu-trigger' : ''; + it.innerHTML = `${a.icon}${a.label}${a.submenu ? '›' : ''}`; + it.addEventListener('click', (e) => { + e.stopPropagation(); + if (a.submenu) { + dropdown.replaceChildren(); + const back = document.createElement('div'); + back.className = 'dropdown-item-compact email-bulk-submenu-back'; + back.innerHTML = '‹Export Emails'; + back.addEventListener('click', (ev) => { + ev.stopPropagation(); + close(); + _showBulkActionsMenu(anchor); + }); + dropdown.appendChild(back); + const divider = document.createElement('div'); + divider.className = 'dropdown-divider'; + dropdown.appendChild(divider); + const options = [ + { label: 'Email file (.eml)', icon: _emailIco, action: _exportSelectedEmails }, + { label: 'PDF', icon: _pdfIco, action: _exportSelectedEmailPdf }, + ]; + for (const option of options) { + const child = document.createElement('div'); + child.className = 'dropdown-item-compact'; + child.innerHTML = `${option.icon}${option.label}`; + child.addEventListener('click', (ev) => { + ev.stopPropagation(); + close(); + option.action(); + }); + dropdown.appendChild(child); + } + _fitEmailDropdown(dropdown, anchor.getBoundingClientRect()); + return; + } + close(); + a.action(); + }); dropdown.appendChild(it); } // Mobile-only Cancel — matches the per-card and sidebar dropdowns. diff --git a/tests/test_email_library_bulk_actions.py b/tests/test_email_library_bulk_actions.py index 4e9930678..64e3f46c1 100644 --- a/tests/test_email_library_bulk_actions.py +++ b/tests/test_email_library_bulk_actions.py @@ -41,18 +41,18 @@ def test_email_bulk_read_unread_checks_backend_success_before_syncing_cache(): assert "_libCacheWriteBack()" in src -def test_email_bulk_export_attachments_is_ui_only_selected_context(): +def test_email_bulk_exports_messages_and_attachments_from_selected_context(): frontend = email_library_source() backend = _EMAIL_ROUTES.read_text(encoding="utf-8") - export_src = frontend[ - frontend.index("async function _exportSelectedAttachments()"): - frontend.index("\n}\n\n// URL-suffix helper", frontend.index("async function _exportSelectedAttachments()")) + 3 - ] + export_src = _function_source("_exportSelectedBundle") payload_src = _function_source("_selectedEmailExportPayload") + assert "Export Emails" in frontend + assert "Email file (.eml)" in frontend + assert "_exportSelectedEmailPdf" in frontend assert "Export Attachments" in frontend assert "All in view" in frontend - assert "/api/email/attachments-download-bulk" in export_src + assert "${isEmailExport ? 'messages' : 'attachments'}-download-bulk" in export_src assert "method: 'POST'" in export_src assert "res.blob()" in export_src assert "account_id: String(em.account_id || state._libAccountId || '')" in payload_src @@ -63,6 +63,25 @@ def test_email_bulk_export_attachments_is_ui_only_selected_context(): assert "_fixture_email_read(uid, folder, owner)" in backend assert "_is_likely_signature_image_attachment(att)" in backend + assert '@router.post("/messages-download-bulk")' in backend + assert '"(UID RFC822)"' in backend + assert "msg.as_bytes(policy=email_policy.SMTP)" in backend + assert 'media_type="message/rfc822"' in backend + assert '"X-Exported-Emails"' in backend + + +def test_email_bulk_pdf_export_reads_without_marking_read_and_sanitizes_body(): + frontend = email_library_source() + pdf_src = _function_source("_downloadSelectedEmailPdf") + menu_src = js_function_source("_showBulkActionsMenu", frontend) + + assert "mark_seen=false&full=true" in pdf_src + assert "_safeRenderEmailBody(data)" in pdf_src + assert "new DOMParser()" in pdf_src + assert "window.html2pdf()" in pdf_src + assert "up to 20 emails" in pdf_src + assert "PDF" in menu_src + def test_fixture_tag_filter_supports_receipt_tag_flow(): backend = _EMAIL_ROUTES.read_text(encoding="utf-8")