From 31685b050a2bd9ce7e5e9acda8dd14dc48916142 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:45:09 +0000 Subject: [PATCH] fix(llm): support scientific and decimal literals in degenerate guard Preserve complete scientific notation and signed floating-point literals across arbitrary streaming chunk boundaries in _DegenerateStreamGuard. Buffer pending tokens until delimiter characters arrive to avoid splitting numbers into incomplete or arbitrary alphanumeric tokens, while maintaining the 2048-number runaway backstop and detection of repeated phrases and word loops. --- src/llm_core.py | 49 +++++++++++---- tests/test_llm_core_fallback.py | 106 +++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 26 deletions(-) diff --git a/src/llm_core.py b/src/llm_core.py index d8f8ac6d5..df3da8a02 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -422,7 +422,18 @@ def _stream_delta_event(text: str, *, thinking: bool = False) -> str: return f"data: {json.dumps(payload)}\n\n" -_DEGENERATE_WORD_RE = re.compile(r"[A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff]+") +_WORD_CHARS = r"A-Za-z0-9_\u0370-\u03ff\u0400-\u04ff" +_DELIM_RE = re.compile(rf"[^{_WORD_CHARS}+\-.]") +_NUMERIC_PATTERN = rf"(?{_NUMERIC_PATTERN})|(?P[{_WORD_CHARS}]+)") + + +def _is_numeric_token(token: str) -> bool: + if token.isdigit(): + return True + return _NUMERIC_FULLMATCH_RE.fullmatch(token) is not None class _DegenerateStreamGuard: @@ -444,24 +455,38 @@ class _DegenerateStreamGuard: self.recent_tokens: List[str] = [] self.periodic_tokens: List[str] = [] self.total_chars = 0 + self._buffer = "" def check(self, text: str) -> Optional[str]: if not text: return None self.total_chars += len(text) - # Keep one-character tokens. Structured output often varies through - # numeric selectors or coordinates; dropping those tokens can make - # valid CSS/SVG rows look like an exact repeated phrase. - tokens = [t.lower() for t in _DEGENERATE_WORD_RE.findall(text)] + self._buffer += text + + m_delims = list(_DELIM_RE.finditer(self._buffer)) + if m_delims: + split_idx = m_delims[-1].end() + head = self._buffer[:split_idx] + self._buffer = self._buffer[split_idx:] + elif len(self._buffer) >= 256: + head = self._buffer + self._buffer = "" + else: + return None + + tokens = [] + for m in _TOKEN_RE.finditer(head): + tokens.append((m.group(0).lower(), m.lastgroup == "NUM")) if not tokens: return None - for token in tokens: + + for token, is_num in tokens: if token == self.last_token: self.same_run += 1 else: self.last_token = token self.same_run = 1 - self.numeric_run = self.numeric_run + 1 if token.isdigit() else 0 + self.numeric_run = self.numeric_run + 1 if is_num else 0 self.recent_tokens.append(token) self.periodic_tokens.append(token) if len(self.recent_tokens) > 96: @@ -470,19 +495,19 @@ class _DegenerateStreamGuard: self.periodic_tokens = self.periodic_tokens[-2048:] reason = None - if self.same_run >= 28 and self.total_chars >= 100 and not self.last_token.isdigit(): + if self.same_run >= 28 and self.total_chars >= 100 and not _is_numeric_token(self.last_token): reason = f"repeated '{self.last_token}' {self.same_run} times" elif len(self.recent_tokens) >= 72: top = max(set(self.recent_tokens), key=self.recent_tokens.count) count = self.recent_tokens.count(top) - if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78 and not top.isdigit(): + if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78 and not _is_numeric_token(top): reason = f"repeated '{top}' {count}/{len(self.recent_tokens)} recent tokens" if ( not reason and len(self.recent_tokens) >= 48 # A window that is mostly numbers is matrix/table data; its # periodicity is real. Mixed loops still hit the exact-block check. - and sum(t.isdigit() for t in self.recent_tokens) * 2 < len(self.recent_tokens) + and sum(_is_numeric_token(t) for t in self.recent_tokens) * 2 < len(self.recent_tokens) ): # Detect a periodic suffix instead of counting repeated n-grams. # Reused property names are normal in CSS/SVG/JSON; a collapsed @@ -518,9 +543,9 @@ class _DegenerateStreamGuard: if history[previous:previous + anchor_size] != anchor: continue period = current_anchor - previous - if period < 24 or len(history) < period * 2: + if period < 24 or len(history) < period * 2 or self.numeric_run >= period: continue - if history[-2 * period:-period] == history[-period:] and not all(t.isdigit() for t in history[-period:]): + if history[-2 * period:-period] == history[-period:] and not all(_is_numeric_token(t) for t in history[-period:]): reason = f"repeated an exact {period}-token block twice" break diff --git a/tests/test_llm_core_fallback.py b/tests/test_llm_core_fallback.py index f235fee43..680f4aefd 100644 --- a/tests/test_llm_core_fallback.py +++ b/tests/test_llm_core_fallback.py @@ -1093,39 +1093,117 @@ def test_degenerate_stream_guard_allows_long_rows_with_changing_values(): def _latex_matrix(cell): rows = (" & ".join(cell(i, j) for j in range(8)) for i in range(8)) - return "$$\\begin{pmatrix}\n" + " \\\\\n".join(rows) + "\n\\end{pmatrix}$$" + return "$$\\begin{pmatrix}\n" + " \\\\\n".join(rows) + "\n\\end{pmatrix}$$\n" @pytest.mark.parametrize("text", [ _latex_matrix(lambda i, j: "0"), _latex_matrix(lambda i, j: "1" if i == j else "0"), - "buf = [" + ", ".join("0" for _ in range(64)) + "]", + _latex_matrix(lambda i, j: "2e10"), + _latex_matrix(lambda i, j: "1e-5"), + _latex_matrix(lambda i, j: "1.25e-3"), + _latex_matrix(lambda i, j: "-0.125"), + _latex_matrix(lambda i, j: "0.0"), + _latex_matrix(lambda i, j: "3.14"), + "buf = [" + ", ".join("0" for _ in range(64)) + "]\n", + "buf = [" + ", ".join("1.25e-3" for _ in range(64)) + "]\n", + "buf = [" + ", ".join("-0.125" for _ in range(64)) + "]\n", + "t,v1,v2,v3\n" + "\n".join(f"{i},1.25e-3,-0.125,2e10" for i in range(25)) + "\n", + "\n".join(f"{i}. Solve: $\\frac{{{i}}}{{{i+1}}}x + \\frac{{1}}{{{i+2}}} = \\frac{{{i+3}}}{{{i+4}}}$" for i in range(1, 20)) + "\n", + "" + "".join(f'' for i in range(4) for j in range(4)) + "\n", + "{" + ", ".join(f'"key_{i}": "translation_{i}"' for i in range(40)) + "}\n", ]) -@pytest.mark.parametrize("chunk", [1, 6, 64]) +@pytest.mark.parametrize("chunk", [1, 3, 6, 8, 16, 32, 64, 128]) def test_degenerate_stream_guard_allows_numeric_data(text, chunk): - """#5560: zero/identity matrices and padded buffers repeat numbers by - design; they are not token collapse.""" + """#5560: legitimate matrices, scientific numbers, tables, and structured data + repeat values or formatting by design; they are not token collapse.""" guard = llm_core._DegenerateStreamGuard("math-model") assert all(guard.check(text[i:i + chunk]) is None for i in range(0, len(text), chunk)) -def test_degenerate_stream_guard_still_stops_an_endless_number_run(): +@pytest.mark.parametrize("chunk", [1, 6, 64, 3000]) +def test_degenerate_stream_guard_allows_numeric_data_full_and_char_stream(chunk): + text = _latex_matrix(lambda i, j: "1.25e-3") + guard = llm_core._DegenerateStreamGuard("math-model") + + assert all(guard.check(text[i:i + chunk]) is None for i in range(0, len(text), chunk)) + + +@pytest.mark.parametrize("loop_text", [ + "0 " * 3000, + "1.25e-3 " * 3000, + "1 2 " * 1500, +]) +@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 3000]) +def test_degenerate_stream_guard_still_stops_an_endless_number_run(loop_text, chunk): guard = llm_core._DegenerateStreamGuard("looping-model") - chunk = guard.check("0 " * 3000) - - assert chunk is not None - assert "numbers in a row" in chunk + stopped = False + for i in range(0, len(loop_text), chunk): + res = guard.check(loop_text[i:i + chunk]) + if res is not None: + assert "numbers in a row" in res + stopped = True + break + assert stopped -def test_degenerate_stream_guard_still_stops_a_mixed_word_number_loop(): +@pytest.mark.parametrize("loop_text", [ + "row 0 0 0 0 " * 300, + "Step 1 2 3 " * 200, +]) +@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 1500]) +def test_degenerate_stream_guard_still_stops_a_mixed_word_number_loop(loop_text, chunk): guard = llm_core._DegenerateStreamGuard("looping-model") - chunk = guard.check("row 0 0 0 0 " * 300) + stopped = False + for i in range(0, len(loop_text), chunk): + res = guard.check(loop_text[i:i + chunk]) + if res is not None: + assert "repeated" in res + stopped = True + break + assert stopped - assert chunk is not None - assert "repeated" in chunk + +@pytest.mark.parametrize("loop_text", [ + "Summer " * 80, + "also be a software developer mode " * 30, +]) +@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 500]) +def test_degenerate_stream_guard_still_stops_repeated_words_and_phrases(loop_text, chunk): + guard = llm_core._DegenerateStreamGuard("looping-model") + + stopped = False + for i in range(0, len(loop_text), chunk): + res = guard.check(loop_text[i:i + chunk]) + if res is not None: + assert "repeated" in res + stopped = True + break + assert stopped + + +@pytest.mark.parametrize("bad_token", [ + "1face ", + "1e ", + "25e ", + "0x12 ", +]) +@pytest.mark.parametrize("chunk", [1, 6, 32, 64, 250]) +def test_degenerate_stream_guard_stops_alphanumeric_repetition_resembling_numbers(bad_token, chunk): + guard = llm_core._DegenerateStreamGuard("looping-model") + text = bad_token * 50 + + stopped = False + for i in range(0, len(text), chunk): + res = guard.check(text[i:i + chunk]) + if res is not None: + assert "repeated" in res + stopped = True + break + assert stopped def test_terminal_stream_retries_degenerate_generation_before_emitting(monkeypatch):