mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-03 03:32:25 +01:00
fix(research): typeset math in visual reports with the shared renderer (#6929)
This commit is contained in:
red person
@@ -0,0 +1,87 @@
|
||||
"""Preserve report TeX for markdown.js's existing deferred KaTeX renderer."""
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from markdown.blockprocessors import BlockProcessor
|
||||
from markdown.extensions import Extension
|
||||
from markdown.inlinepatterns import InlineProcessor
|
||||
from markdown.util import AtomicString
|
||||
|
||||
|
||||
# Keep delimiter and currency rules aligned with mathBlocks in markdown.js.
|
||||
_ESCAPED_MATH = r"\\\[([\s\S]*?)\\\]|\\\(([^\n]*?)\\\)"
|
||||
_DOLLAR_MATH = (
|
||||
r"\$\$([\s\S]*?)\$\$|"
|
||||
r"(?<![\$\d])\$(?!\$)(?=\S)([^\$\n]+?)(?<=\S)\$(?!\$|\d)"
|
||||
)
|
||||
|
||||
|
||||
def _placeholder(tex, display):
|
||||
node = ElementTree.Element("span", {
|
||||
"class": "ody-math-pending", "data-display": str(display).lower(),
|
||||
})
|
||||
node.text = AtomicString(tex)
|
||||
return node
|
||||
|
||||
|
||||
class _HtmlContext(HTMLParser):
|
||||
"""Inline HTML is still raw when math runs, including tag attributes."""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.protected = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in {"code", "pre", "script", "style", "textarea"}:
|
||||
self.protected.append(tag)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag in self.protected:
|
||||
del self.protected[self.protected.index(tag):]
|
||||
|
||||
|
||||
class _InlineMath(InlineProcessor):
|
||||
def handleMatch(self, match, data):
|
||||
prefix = data[:match.start()]
|
||||
if "<" in prefix:
|
||||
context = _HtmlContext()
|
||||
context.feed(prefix)
|
||||
if context.protected or re.match(r"<(?:/?[A-Za-z]|[!?])", context.rawdata.lstrip()):
|
||||
return None, None, None
|
||||
# Backticks run before this processor. Preserve a backslash-escaped
|
||||
# dollar too; math must run before Markdown consumes \( and \[.
|
||||
if prefix.endswith("\\") and (len(prefix) - len(prefix.rstrip("\\"))) % 2:
|
||||
return None, None, None
|
||||
tex = next(group for group in match.groups() if group is not None)
|
||||
display = match.group(1) is not None
|
||||
return _placeholder(tex, display), match.start(), match.end()
|
||||
|
||||
|
||||
class _DisplayMath(BlockProcessor):
|
||||
def test(self, parent, block):
|
||||
return re.match(r"^ {0,3}(\$\$|\\\[)", block) is not None
|
||||
|
||||
def run(self, parent, blocks):
|
||||
opening = re.match(r"^ {0,3}(\$\$|\\\[)", blocks[0])
|
||||
closing = "$$" if opening.group(1) == "$$" else "\\]"
|
||||
source = blocks[0][opening.end():]
|
||||
consumed = 1
|
||||
while closing not in source:
|
||||
if consumed == len(blocks):
|
||||
return False # Leave incomplete input to ordinary Markdown.
|
||||
source += "\n\n" + blocks[consumed]
|
||||
consumed += 1
|
||||
tex, _, tail = source.partition(closing)
|
||||
del blocks[:consumed]
|
||||
if tail.strip():
|
||||
blocks.insert(0, tail)
|
||||
parent.append(_placeholder(tex, True))
|
||||
|
||||
|
||||
class ReportMathExtension(Extension):
|
||||
def extendMarkdown(self, md):
|
||||
# Backticks (190) and indented code (80) retain precedence.
|
||||
md.inlinePatterns.register(_InlineMath(_ESCAPED_MATH, md), "report_math_escaped", 185)
|
||||
# Parse links before dollar math so URL destinations stay literal.
|
||||
md.inlinePatterns.register(_InlineMath(_DOLLAR_MATH, md), "report_math_dollar", 155)
|
||||
md.parser.blockprocessors.register(_DisplayMath(md.parser), "report_math", 79)
|
||||
+28
-3
@@ -23,6 +23,7 @@ from typing import Dict, List, Optional, Tuple
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.research_utils import strip_thinking
|
||||
from src.report_math import ReportMathExtension
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import markdown
|
||||
@@ -47,6 +48,7 @@ for _t in ("td", "th"):
|
||||
_REPORT_ALLOWED_ATTRS.setdefault(_t, set()).add("align")
|
||||
_REPORT_ALLOWED_ATTRS.setdefault("a", set()).update({"href", "title", "target", "rel"})
|
||||
_REPORT_ALLOWED_ATTRS.setdefault("img", set()).update({"src", "alt", "title"})
|
||||
_REPORT_ALLOWED_ATTRS.setdefault("span", set()).add("data-display")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -79,7 +81,7 @@ def _md_to_html(md_text: str) -> str:
|
||||
md_text = _autolink_urls(md_text)
|
||||
result = markdown.markdown(
|
||||
md_text,
|
||||
extensions=["extra", "codehilite", "toc", "tables", "sane_lists", "md_in_html"],
|
||||
extensions=["extra", "codehilite", "toc", "tables", "sane_lists", "md_in_html", ReportMathExtension()],
|
||||
extension_configs={
|
||||
"codehilite": {"css_class": "code", "guess_lang": False},
|
||||
"toc": {"marker": "", "toc_depth": "2-3"},
|
||||
@@ -1021,6 +1023,8 @@ body[class^="standard-report-"] .report-section:nth-of-type(even) {{
|
||||
.content code {{ font-family: var(--font-mono); font-size: 0.86em; background: var(--bg-surface-alt); padding: 0.15em 0.4em; border-radius: 4px; }}
|
||||
.content pre {{ background: var(--bg-surface-alt); border: 1px solid var(--border); border-radius: var(--radius); padding: 1.25rem 1.5rem; overflow-x: auto; margin: 1.25rem 0; font-size: 0.86rem; line-height: 1.6; }}
|
||||
.content pre code {{ background: none; padding: 0; }}
|
||||
/* Match the chat renderer's handling of wide display equations. */
|
||||
.content .katex-display {{ overflow-x: auto; overflow-y: hidden; }}
|
||||
.content table {{ width: 100%; border-collapse: collapse; margin: 1.25rem 0; font-size: 0.9rem; border-radius: var(--radius); overflow: hidden; box-shadow: var(--shadow-sm); }}
|
||||
.content th {{ text-align: left; padding: 0.7rem 1rem; background: var(--accent-bg); font-weight: 600; border-bottom: 2px solid var(--border-strong); }}
|
||||
.content td {{ padding: 0.6rem 1rem; border-bottom: 1px solid var(--border); vertical-align: top; }}
|
||||
@@ -1434,6 +1438,12 @@ body[class^="standard-report-"] .report-section:nth-of-type(even) {{
|
||||
|
||||
<script>
|
||||
(function() {{
|
||||
// Reuse the chat renderer and vendored assets only for reports with math.
|
||||
if (document.querySelector('.ody-math-pending')) {{
|
||||
import('/static/js/markdown.js').then(function(markdown) {{
|
||||
return markdown.renderMath(document);
|
||||
}}).catch(function(err) {{ console.warn('Report math render error:', err); }});
|
||||
}}
|
||||
// ESC closes the report tab. window.close() works when the tab was
|
||||
// opened via window.open() (which is how the panel launches it). If the
|
||||
// browser blocks self-close (rare — e.g. report opened by direct URL),
|
||||
@@ -2835,11 +2845,23 @@ def generate_visual_report(
|
||||
spare_images = section_pool[_consumed:]
|
||||
|
||||
# Build TOC
|
||||
math_labels = {}
|
||||
if 'ody-math-pending' in report_html:
|
||||
label_soup = BeautifulSoup(report_html, "html.parser")
|
||||
for heading in label_soup.select('h2[id], h3[id]'):
|
||||
if not heading.select_one('.ody-math-pending'):
|
||||
continue
|
||||
for child in list(heading.find_all(True)):
|
||||
if 'ody-math-pending' in child.get('class', []):
|
||||
child['data-display'] = 'false'
|
||||
else:
|
||||
child.unwrap()
|
||||
math_labels[heading['id']] = heading.decode_contents()
|
||||
toc_lines = []
|
||||
for h in headings:
|
||||
depth_class = f"depth-{h['level']}"
|
||||
toc_lines.append(
|
||||
f'<a href="#{h["slug"]}" class="{depth_class}">{html.escape(h["text"])}</a>'
|
||||
f'<a href="#{h["slug"]}" class="{depth_class}">{math_labels.get(h["slug"], html.escape(h["text"]))}</a>'
|
||||
)
|
||||
toc_html = "\n ".join(toc_lines) if toc_lines else ""
|
||||
|
||||
@@ -2848,8 +2870,11 @@ def generate_visual_report(
|
||||
options = []
|
||||
for h in headings:
|
||||
prefix = "\u2014 " if h["level"] == 3 else ""
|
||||
text = h["text"]
|
||||
if h["slug"] in math_labels:
|
||||
text = BeautifulSoup(math_labels[h["slug"]], "html.parser").get_text()
|
||||
options.append(
|
||||
f'<option value="{h["slug"]}">{prefix}{html.escape(h["text"])}</option>'
|
||||
f'<option value="{h["slug"]}">{prefix}{html.escape(text)}</option>'
|
||||
)
|
||||
mobile_toc_html = (
|
||||
'<div class="mobile-section-nav">'
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Research reports preserve TeX for the shared, vendored math renderer."""
|
||||
from bs4 import BeautifulSoup
|
||||
import pytest
|
||||
|
||||
from src.visual_report import _md_to_html, generate_visual_report
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source,tex,display", [
|
||||
(r"Value $x_i^2$.", "x_i^2", "false"),
|
||||
(r"Value \(x_i^2\).", "x_i^2", "false"),
|
||||
(r"Value $$\sum_i x_i$$.", r"\sum_i x_i", "true"),
|
||||
(r"Value \[\sum_i x_i\].", r"\sum_i x_i", "true"),
|
||||
("$$a_i\n\nb_i$$", "a_i\n\nb_i", "true"),
|
||||
("\\[a_i\n\nb_i\\]", "a_i\n\nb_i", "true"),
|
||||
])
|
||||
def test_math_source_survives_markdown(source, tex, display):
|
||||
soup = BeautifulSoup(_md_to_html(source), "html.parser")
|
||||
pending = soup.select(".ody-math-pending")
|
||||
assert len(pending) == 1
|
||||
assert pending[0].get_text() == tex
|
||||
assert pending[0]["data-display"] == display
|
||||
assert pending[0].find("em") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", [
|
||||
"`$x_i$`", "```text\n$x_i$\n```", "~~~text\n$x_i$\n~~~",
|
||||
" $x_i$", "<code>$x_i$</code>", "<pre>$x_i$</pre>",
|
||||
"Prices range from $5 to $10.", "An escaped \\$5 costs less than $10.",
|
||||
])
|
||||
def test_code_and_currency_are_not_math(source):
|
||||
soup = BeautifulSoup(_md_to_html(source), "html.parser")
|
||||
assert not soup.select(".ody-math-pending")
|
||||
assert "$" in soup.get_text()
|
||||
|
||||
|
||||
def test_html_attributes_are_not_rewritten_as_math():
|
||||
soup = BeautifulSoup(_md_to_html('<a title="$literal$">Value $x_i$</a>'), "html.parser")
|
||||
assert soup.a["title"] == "$literal$"
|
||||
assert soup.select_one("a .ody-math-pending").get_text() == "x_i"
|
||||
|
||||
|
||||
def test_link_destinations_stay_literal_while_link_labels_can_contain_math():
|
||||
soup = BeautifulSoup(_md_to_html('[$x_i$](https://example.com/$literal$)'), "html.parser")
|
||||
assert soup.a["href"] == "https://example.com/$literal$"
|
||||
assert soup.select_one("a .ody-math-pending").get_text() == "x_i"
|
||||
|
||||
|
||||
def test_math_source_is_inert_and_report_sanitizer_stays_active():
|
||||
soup = BeautifulSoup(_md_to_html(
|
||||
r'$\text{<img src=x onerror="alert(1)">}$'
|
||||
'\n\n<script>alert(2)</script><a href="javascript:alert(3)">link</a>'
|
||||
), "html.parser")
|
||||
assert soup.select_one(".ody-math-pending").get_text() == r'\text{<img src=x onerror="alert(1)">}'
|
||||
assert soup.find("img") is None
|
||||
assert soup.find("script") is None
|
||||
assert not soup.select('a[href^="javascript:"]')
|
||||
|
||||
|
||||
def test_heading_math_reaches_body_and_toc_without_changing_anchor_contract():
|
||||
soup = BeautifulSoup(generate_visual_report(
|
||||
"Formula report", "# Formula report\n\n## Formula $x_i$\n\nValue $x_i$.",
|
||||
session_id="rp-math",
|
||||
), "html.parser")
|
||||
link = soup.select_one(".toc-sidebar nav a")
|
||||
heading = soup.find(id=link["href"][1:])
|
||||
assert heading["id"] == "formula-xi"
|
||||
for node in (link, heading):
|
||||
assert node.select_one(".ody-math-pending").get_text() == "x_i"
|
||||
assert soup.select_one("#mobile-section-select option").get_text() == "Formula x_i"
|
||||
|
||||
|
||||
def test_unclosed_display_math_stays_readable():
|
||||
soup = BeautifulSoup(_md_to_html("$$unfinished\n\nA later paragraph."), "html.parser")
|
||||
assert not soup.select(".ody-math-pending")
|
||||
assert "unfinished" in soup.get_text()
|
||||
assert "A later paragraph." in soup.get_text()
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Exercise generated report HTML with the real vendored KaTeX in Chromium."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from src.visual_report import generate_visual_report
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [1440, 390])
|
||||
@pytest.mark.parametrize("mode", ["math", "plain", "blocked"])
|
||||
def test_report_math_in_browser(tmp_path, width, mode):
|
||||
if not shutil.which("node"):
|
||||
pytest.skip("node not installed")
|
||||
markdown = (
|
||||
"# Math report\n\n" + r"## Formula $x_i$" + "\n\n"
|
||||
+ r"Inline \(x_i^2\) and $a < b$." + "\n\n"
|
||||
+ r"\[\sum_{i=1}^{3} x_i\]" + "\n\n"
|
||||
+ r"Invalid: $\frac{1}$. Unsafe: $\href{javascript:alert(1)}{bad}$."
|
||||
+ "\n\n`$literal$` costs $5 to $10."
|
||||
+ "\n\n$$" + "+".join(["x_i"] * 30) + "$$"
|
||||
) if mode != "plain" else "## Findings\n\nNo math here."
|
||||
document = tmp_path / "report.html"
|
||||
document.write_text(generate_visual_report("Math report", markdown), encoding="utf-8")
|
||||
script = r"""
|
||||
const fs = require('node:fs'), assert = require('node:assert/strict');
|
||||
let chromium;
|
||||
try { ({chromium} = require('playwright')); }
|
||||
catch (error) { if (error.code === 'MODULE_NOT_FOUND') process.exit(77); throw error; }
|
||||
(async () => {
|
||||
const browser = await chromium.launch({headless:true});
|
||||
try {
|
||||
const page = await browser.newPage({viewport:{width:Number(process.argv[2]),height:900}});
|
||||
const origin = process.env.ODYSSEUS_TEST_STATIC_ORIGIN;
|
||||
const errors = [], requests = [], failures = [];
|
||||
page.on('pageerror', e => errors.push(e.message));
|
||||
page.on('request', r => requests.push(r.url()));
|
||||
page.on('requestfailed', r => failures.push(r.url()));
|
||||
page.on('dialog', async d => { errors.push('Unexpected dialog'); await d.dismiss(); });
|
||||
const mode = process.argv[3];
|
||||
if (mode === 'blocked') await page.route('**/static/lib/katex/katex.min.js', r => r.abort());
|
||||
await page.route(origin + '/report-math-test', r => r.fulfill({
|
||||
contentType:'text/html', body:fs.readFileSync(process.argv[1], 'utf8')
|
||||
}));
|
||||
await page.goto(origin + '/report-math-test');
|
||||
if (mode === 'math') {
|
||||
await page.locator('.content .katex-display').first().waitFor({state:'attached'});
|
||||
await page.waitForFunction(() => !document.querySelector('.ody-math-pending'));
|
||||
assert.equal(await page.locator('.toc-sidebar .katex').count(), 1);
|
||||
assert.equal(await page.locator('.content .katex-error').count(), 1);
|
||||
assert.equal(await page.locator('.content code').innerText(), '$literal$');
|
||||
assert((await page.locator('.content').innerText()).includes('$5 to $10'));
|
||||
assert.equal(await page.locator('a[href^="javascript:"]').count(), 0);
|
||||
assert.equal(requests.filter(u => u.endsWith('/static/lib/katex/katex.min.js')).length, 1);
|
||||
assert.equal(requests.filter(u => u.endsWith('/static/lib/katex/katex.min.css')).length, 1);
|
||||
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth), false);
|
||||
assert.deepEqual(failures, []);
|
||||
} else if (mode === 'blocked') {
|
||||
await page.waitForFunction(() => performance.getEntriesByType('resource').some(e => e.name.includes('katex.min.js')));
|
||||
assert((await page.locator('.content .ody-math-pending').first().innerText()).includes('x_i'));
|
||||
assert.equal(await page.locator('.katex').count(), 0);
|
||||
} else {
|
||||
await page.waitForLoadState('networkidle');
|
||||
assert.equal(requests.some(u => u.includes('/static/lib/katex/')), false);
|
||||
assert.equal(requests.some(u => u.includes('/static/js/markdown.js')), false);
|
||||
}
|
||||
assert.deepEqual(errors, []);
|
||||
assert(requests.every(u => u.startsWith(origin + '/')), JSON.stringify(requests));
|
||||
console.log(JSON.stringify({mode, width:process.argv[2], requests:requests.length}));
|
||||
} finally { await browser.close(); }
|
||||
})().catch(e => { console.error(e); process.exitCode=1; });
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["node", "-e", script, str(document), str(width), mode],
|
||||
cwd=Path(__file__).resolve().parents[1], env=os.environ.copy(),
|
||||
capture_output=True, text=True, timeout=60,
|
||||
)
|
||||
if result.returncode == 77:
|
||||
pytest.skip("playwright not installed")
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert json.loads(result.stdout)["mode"] == mode
|
||||
Reference in New Issue
Block a user