From 516f046fef04599d9cf808f307927a4e293e8f6e Mon Sep 17 00:00:00 2001 From: Vykos Date: Sat, 31 Oct 2026 14:24:32 +0100 Subject: [PATCH] fix(chat): label the vision description injected for text-only models (#6223) When the chat model cannot see images, the separate vision call's description is appended to the user's message. Nothing marked it as machine-derived, so the chat model read it as the user's own prose and answered the description instead of the question. The label goes below the `[Image: ]` marker line because the client strips the block by matching that exact prefix; renaming the marker would leave the raw description rendered in the user's bubble. Fixes #5573 Co-authored-by: Claude --- src/chat_handler.py | 8 +- tests/test_vision_description_provenance.py | 132 ++++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 tests/test_vision_description_provenance.py diff --git a/src/chat_handler.py b/src/chat_handler.py index 5ba244a34..0e72f55b3 100644 --- a/src/chat_handler.py +++ b/src/chat_handler.py @@ -282,7 +282,13 @@ class ChatHandler: _sync_upload_vision_to_gallery(file_info, owner, vl_desc) except Exception: pass - enhanced_message = f"{enhanced_message}\n\n[Image: {file_info['name']}]\n{vl_desc}" + enhanced_message = f"{enhanced_message}\n\n[Image: {file_info['name']}]\n" + if vl_desc: + # A separate vision call wrote this, but the chat model sees + # one message and answers the prose as if the user wrote it + # (#5573). Label it below the marker line — the client strips + # the block by matching `[Image: ]\n`. + enhanced_message += f"[Image description — context only, not part of the user's message]:\n{vl_desc}" # Surface the description to the client live so it renders as a # collapsible "image description" on the user bubble (not just # after a refresh that re-parses the stored message). diff --git a/tests/test_vision_description_provenance.py b/tests/test_vision_description_provenance.py new file mode 100644 index 000000000..b65399f35 --- /dev/null +++ b/tests/test_vision_description_provenance.py @@ -0,0 +1,132 @@ +"""Issue #5573 — the vision model's description must not read as the user's own words. + +For a text-only chat model the description comes from a separate vision call and +is appended to the user's message. Unlabelled, the chat model answers the prose +instead of the image — reviewing the user's "writing" back to them. +""" + +import importlib +from types import SimpleNamespace + +import pytest + +DESCRIPTION = "A chocolate cake, plated with a mirror glaze and gold leaf." +QUESTION = "What am I looking at?" + + +class _UploadHandler: + def __init__(self, file_info): + self._file_info = file_info + + def resolve_upload(self, att_id, owner=None): + return self._file_info if att_id == self._file_info["id"] else None + + def is_image_file(self, *_args, **_kwargs): + return True + + +def _no_vision_call(*_args, **_kwargs): + raise AssertionError("the vision model must not be called on this path") + + +@pytest.fixture +def vision_env(monkeypatch, tmp_path): + """Text-only chat model with one image attachment, no DB and no real vision call. + + Other tests drop src.chat_handler from sys.modules, so resolve it here rather + than at import time — otherwise the patches land on a different module object + than the one under test. + """ + module = importlib.import_module("src.chat_handler") + monkeypatch.setattr(module, "UPLOAD_DIR", str(tmp_path)) + monkeypatch.setattr(module, "model_supports_vision", lambda *a, **k: False) + monkeypatch.setattr(module, "_sync_upload_vision_to_gallery", lambda *a, **k: None) + monkeypatch.setattr(module, "build_user_content", lambda text, *a, **k: text) + monkeypatch.setattr( + importlib.import_module("src.settings"), "get_setting", lambda key, default=None: default + ) + return SimpleNamespace(module=module, tmp_path=tmp_path, file_info={ + "id": "img-1", "name": "cake.jpg", "path": str(tmp_path / "cake.jpg"), + "mime": "image/jpeg", "size": 1, "hash": "h", + }) + + +async def _preprocess(env): + handler = env.module.ChatHandler( + session_manager=None, memory_manager=None, chat_processor=None, + research_handler=None, preset_manager=None, + upload_handler=_UploadHandler(env.file_info), + ) + sess = SimpleNamespace(model="chat-model", endpoint_url="", owner="user", id="s1") + enhanced, *_ = await handler.preprocess_message( + QUESTION, ["img-1"], sess, auto_opened_docs=[], + ) + return enhanced + + +def _describe(env, monkeypatch, text): + monkeypatch.setattr(env.module, "analyze_image_with_vl_result", + lambda *a, **k: {"text": text, "model": "vl-model"}) + + +def _label_above(enhanced, description): + """The line the description is introduced by.""" + return enhanced.split(description)[0].rstrip("\n").rsplit("\n", 1)[-1] + + +@pytest.mark.asyncio +async def test_description_is_marked_as_not_the_users_message(vision_env, monkeypatch): + _describe(vision_env, monkeypatch, DESCRIPTION) + enhanced = await _preprocess(vision_env) + + assert QUESTION in enhanced, "the user's own words must survive untouched" + label = _label_above(enhanced, DESCRIPTION) + assert label.startswith("[") and "user" in label.lower(), ( + "the description is appended as bare prose, so the chat model reads it as " + f"the user's own writing. Got:\n{enhanced}" + ) + + +@pytest.mark.asyncio +async def test_cached_description_is_labelled_too(vision_env, monkeypatch): + """The cache holds an auto description or a hand-corrected one, and the label + has to be true either way — it is a description of the image regardless.""" + cache = vision_env.tmp_path / ".vision" + cache.mkdir() + (cache / "img-1.txt").write_text(DESCRIPTION, encoding="utf-8") + monkeypatch.setattr(vision_env.module, "analyze_image_with_vl_result", _no_vision_call) + + enhanced = await _preprocess(vision_env) + + label = _label_above(enhanced, DESCRIPTION) + assert label.startswith("[") and "user" in label.lower(), ( + f"a cached description is injected with no label. Got:\n{enhanced}" + ) + + +@pytest.mark.asyncio +async def test_client_side_image_marker_is_preserved(vision_env, monkeypatch): + """chatRenderer.js and sessions.js strip the block by matching `[Image: ]\\n`. + + Renaming that marker leaves the raw description rendered in the user's bubble. + """ + _describe(vision_env, monkeypatch, DESCRIPTION) + assert "[Image: cake.jpg]\n" in await _preprocess(vision_env) + + +@pytest.mark.asyncio +async def test_no_dangling_label_when_the_vision_model_returns_nothing(vision_env, monkeypatch): + _describe(vision_env, monkeypatch, "") + enhanced = await _preprocess(vision_env) + assert enhanced.endswith("[Image: cake.jpg]\n"), ( + "an empty description must leave the bare marker, trailing newline included " + f"— the client's strip regex needs it. Got:\n{enhanced!r}" + ) + + +@pytest.mark.asyncio +async def test_vision_capable_model_path_is_unchanged(vision_env, monkeypatch): + monkeypatch.setattr(vision_env.module, "model_supports_vision", lambda *a, **k: True) + monkeypatch.setattr(vision_env.module, "analyze_image_with_vl_result", _no_vision_call) + + assert await _preprocess(vision_env) == f"{QUESTION}\n\n[Image attached: cake.jpg]"