mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-09 16:32:21 +02:00
Léo
137daab16e
The compact preview reduces every pre-execution tool error to a fixed allowlist so exception text cannot reach the client. Messages our own validators build per call are not on that list, so the model got "The tool call could not be validated..." instead of the pdf_extract path hint, the email uid provenance error, the artifact-Python target message, or which schema argument was wrong. That is the default path for the tool-profile models, Qwen3.5-9B included. Each validator site now records its message in a per-call validator_error before raising, and only that text is returned verbatim. A schema failure gets a hint rebuilt from the offered schema and the call's own arguments (missing property, expected type, allowed values), never from the caught ValidationError. Everything else, including all execution failures, still goes through _public_preview_tool_error.
267 lines
11 KiB
Python
267 lines
11 KiB
Python
"""Exception details stay in server logs across both chat SSE transports."""
|
|
|
|
import json
|
|
import logging
|
|
from uuid import uuid4
|
|
|
|
import jsonschema
|
|
import pytest
|
|
from starlette.responses import StreamingResponse
|
|
|
|
from src import agent_runs
|
|
from src.tool_policy import ToolPolicy
|
|
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
|
|
from src.turn_contract import resolve_full_inventory_contract
|
|
from tests.runtime_evidence_helpers import authoritative_executor
|
|
|
|
|
|
SENSITIVE = (
|
|
"TAKEOVER_SECRET_827_828 /srv/private/credentials.json "
|
|
"https://internal.example/debug?token=PRIVATE_TOKEN "
|
|
'{"request_body":"PRIVATE_BODY"}'
|
|
)
|
|
|
|
|
|
async def _client_chunks(generator, detached):
|
|
session = "redaction-" + uuid4().hex
|
|
run = None
|
|
try:
|
|
if detached:
|
|
run = agent_runs.start(session, generator)
|
|
await run.task
|
|
generator = agent_runs.subscribe(session, run)
|
|
response = StreamingResponse(generator, media_type="text/event-stream")
|
|
return [chunk async for chunk in response.body_iterator]
|
|
finally:
|
|
if run is not None:
|
|
if run.evict_task:
|
|
run.evict_task.cancel()
|
|
agent_runs._RUNS.pop(session, None)
|
|
|
|
|
|
def _events(chunks):
|
|
return [
|
|
json.loads(chunk.split("data: ", 1)[1])
|
|
for chunk in chunks if "data: " in chunk and "[DONE]" not in chunk
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("detached", [False, True], ids=["direct-827", "detached-828"])
|
|
@pytest.mark.parametrize("boundary", ["calendar", "explicit"])
|
|
async def test_native_preemptive_exception_detail_stays_server_side(
|
|
monkeypatch, caplog, detached, boundary,
|
|
):
|
|
import src.agent_loop as module
|
|
|
|
monkeypatch.setattr(module, "get_setting", lambda key, default=None: default)
|
|
monkeypatch.setattr(module, "get_mcp_manager", lambda: None)
|
|
monkeypatch.setattr(module, "blocked_tools_for_owner", lambda owner: set())
|
|
monkeypatch.setattr(module, "estimate_tokens", lambda *args, **kwargs: 10)
|
|
monkeypatch.setattr(module, "_agent_route_tool_mode", lambda *args, **kwargs: (True, False, False))
|
|
monkeypatch.setattr(module, "_build_system_prompt", lambda messages, *args, **kwargs: (list(messages), []))
|
|
monkeypatch.setattr(module, "_required_safe_read_operation", lambda contract: None)
|
|
|
|
async def execute(*args, **kwargs):
|
|
raise RuntimeError(SENSITIVE)
|
|
|
|
async def stream(*args, **kwargs):
|
|
yield 'data: {"delta":"The requested tool failed."}\n\n'
|
|
yield "data: [DONE]\n\n"
|
|
|
|
monkeypatch.setattr(module, "execute_tool_block", authoritative_executor(execute))
|
|
monkeypatch.setattr(module, "stream_llm_with_fallback", stream)
|
|
tool = "manage_calendar" if boundary == "calendar" else "list_sessions"
|
|
if boundary == "calendar":
|
|
instruction = "What meetings do I have tomorrow?"
|
|
fallback = "preemptive_calendar_lookup"
|
|
else:
|
|
monkeypatch.setattr(module, "_parse_simple_calendar_tool_request", lambda *args: None)
|
|
instruction = "List all chats."
|
|
fallback = "preemptive_explicit_admin_session"
|
|
caplog.set_level(logging.WARNING)
|
|
chunks = await _client_chunks(module.stream_agent_loop(
|
|
"http://model.test/v1", "test-model",
|
|
[{"role": "user", "content": instruction}],
|
|
relevant_tools={tool}, owner="fixture", max_rounds=1, _is_teacher_run=True,
|
|
), detached)
|
|
tool_event = next((e for e in _events(chunks) if e.get("type") == "tool_output"), None)
|
|
assert tool_event is not None, _events(chunks)
|
|
assert tool_event["fallback"] == fallback
|
|
assert tool_event["exit_code"] == 1
|
|
assert tool_event["output"]
|
|
assert "TAKEOVER_SECRET" not in "".join(chunks)
|
|
assert "/srv/private" not in "".join(chunks)
|
|
assert "PRIVATE_TOKEN" not in "".join(chunks)
|
|
assert "PRIVATE_BODY" not in "".join(chunks)
|
|
assert tool_event["error_category"] == "tool_execution_error"
|
|
assert SENSITIVE in caplog.text
|
|
|
|
|
|
def _preview_provider(monkeypatch, payloads):
|
|
import src.clean_agent_preview as module
|
|
replies = iter(payloads)
|
|
|
|
class Response:
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *args): pass
|
|
def raise_for_status(self): pass
|
|
async def aiter_lines(self):
|
|
yield "data: " + json.dumps(next(replies))
|
|
yield "data: [DONE]"
|
|
|
|
class Client:
|
|
def __init__(self, **kwargs): pass
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *args): pass
|
|
def stream(self, *args, **kwargs): return Response()
|
|
|
|
monkeypatch.setattr(module.httpx, "AsyncClient", Client)
|
|
return module
|
|
|
|
|
|
def _preview_generator(module, schemas=()):
|
|
policy = ToolPolicy()
|
|
contract = resolve_full_inventory_contract(schemas=list(schemas), policy=policy)
|
|
return module.stream_preview(
|
|
endpoint_url="http://model.test", model="test", headers={},
|
|
messages=[{"role": "user", "content": "List my notes."}],
|
|
turn_contract=contract, session_id="fixture-redaction", owner="fixture",
|
|
disabled_tools=set(), tool_policy=policy,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("detached", [False, True], ids=["direct-827", "detached-828"])
|
|
@pytest.mark.parametrize("shape", ["string", "message", "detail"])
|
|
async def test_preview_provider_detail_stays_server_side(monkeypatch, caplog, detached, shape):
|
|
error = SENSITIVE if shape == "string" else {shape: SENSITIVE}
|
|
module = _preview_provider(monkeypatch, [{"error": error}])
|
|
caplog.set_level(logging.WARNING)
|
|
chunks = await _client_chunks(_preview_generator(module), detached)
|
|
assert chunks[-1].startswith("event: error\n")
|
|
payload = _events(chunks)[-1]
|
|
assert payload["status"] == 502
|
|
assert "provider" in payload["error"]
|
|
assert SENSITIVE not in "".join(chunks)
|
|
assert "/srv/private" not in "".join(chunks)
|
|
assert "PRIVATE_TOKEN" not in "".join(chunks)
|
|
assert payload["error_category"] == "provider_stream_error"
|
|
assert SENSITIVE in caplog.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("detached", [False, True], ids=["direct-827", "detached-828"])
|
|
@pytest.mark.parametrize("failure", ["execution", "schema", "json", "arguments"])
|
|
async def test_preview_tool_exception_detail_stays_server_side(
|
|
monkeypatch, caplog, detached, failure,
|
|
):
|
|
module = _preview_provider(monkeypatch, [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call-1", "function": {
|
|
"name": "manage_notes", "arguments": '{"action":"list"}',
|
|
}}]}}]},
|
|
{"choices": [{"delta": {"content": "The requested call failed."}}]},
|
|
] * 8)
|
|
|
|
async def execute(*args, **kwargs):
|
|
raise ValueError(SENSITIVE)
|
|
|
|
def fail(*args, **kwargs):
|
|
if failure == "schema":
|
|
raise jsonschema.ValidationError(SENSITIVE, instance={"private": SENSITIVE})
|
|
if failure == "json":
|
|
raise json.JSONDecodeError(SENSITIVE, SENSITIVE, 0)
|
|
raise ValueError(SENSITIVE)
|
|
|
|
if failure == "execution":
|
|
monkeypatch.setattr(module, "execute_tool_block", execute)
|
|
elif failure == "schema":
|
|
monkeypatch.setattr(module.jsonschema, "validate", fail)
|
|
else:
|
|
monkeypatch.setattr(module, "normalize_preview_call_args", fail)
|
|
schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s["function"]["name"] == "manage_notes")
|
|
caplog.set_level(logging.WARNING)
|
|
chunks = await _client_chunks(_preview_generator(module, [schema]), detached)
|
|
tool_event = next(e for e in _events(chunks) if e.get("type") == "tool_output")
|
|
assert tool_event["error"] is True
|
|
assert tool_event["exit_code"] == 1
|
|
assert tool_event["output"]
|
|
assert "TAKEOVER_SECRET" not in "".join(chunks)
|
|
assert "/srv/private" not in "".join(chunks)
|
|
assert "PRIVATE_TOKEN" not in "".join(chunks)
|
|
assert "PRIVATE_BODY" not in "".join(chunks)
|
|
assert tool_event["error_category"] == (
|
|
"tool_execution_error" if failure == "execution" else "invalid_tool_arguments"
|
|
)
|
|
assert SENSITIVE in caplog.text
|
|
|
|
|
|
@pytest.mark.parametrize("message", [
|
|
"Tool is not offered or permitted.",
|
|
"Tool arguments must be a JSON object.",
|
|
"This operation is outside the preview safety policy. No change was made.",
|
|
"Resolve the named recipient with resolve_contact before drafting. Never invent an email address.",
|
|
"The calendar read has not succeeded yet. Obtain the requested calendar evidence before creating the dependent email draft.",
|
|
])
|
|
def test_curated_domain_errors_remain_useful(message):
|
|
from src.clean_agent_preview import _public_preview_tool_error
|
|
assert _public_preview_tool_error(ValueError(message)) == message
|
|
assert message not in _public_preview_tool_error(ValueError(message), execution_attempted=True)
|
|
|
|
|
|
@pytest.mark.parametrize("detail", [
|
|
SENSITIVE,
|
|
"Tool is not offered or permitted.\n" + SENSITIVE,
|
|
"Shell access to credential variable " + SENSITIVE,
|
|
"Artifact completion Python must reference the required " + SENSITIVE,
|
|
])
|
|
def test_untrusted_validation_detail_cannot_masquerade_as_curated_guidance(detail):
|
|
from src.clean_agent_preview import _public_preview_tool_error
|
|
public = _public_preview_tool_error(ValueError(detail))
|
|
assert public
|
|
assert "TAKEOVER_SECRET" not in public
|
|
assert "/srv/private" not in public
|
|
assert "PRIVATE_TOKEN" not in public
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("tool, arguments, hint", [
|
|
# Authored by normalized_native_function_argument_error.
|
|
("pdf_extract", {"url": "Attention Is All You Need"},
|
|
"pdf_extract requires a public http(s) PDF URL"),
|
|
# Rebuilt from the schema, not from the caught ValidationError.
|
|
("manage_notes", {"action": "explode"},
|
|
"Argument 'action' must be one of: list, search, view"),
|
|
("python", {},
|
|
"Missing required argument: 'code'."),
|
|
])
|
|
async def test_preview_returns_validator_guidance_to_the_model(monkeypatch, tool, arguments, hint):
|
|
module = _preview_provider(monkeypatch, [
|
|
{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call-1", "function": {
|
|
"name": tool, "arguments": json.dumps(arguments),
|
|
}}]}}]},
|
|
{"choices": [{"delta": {"content": "The requested call failed."}}]},
|
|
] * 8)
|
|
|
|
async def execute(*args, **kwargs):
|
|
raise AssertionError("an invalid call must not execute")
|
|
|
|
monkeypatch.setattr(module, "execute_tool_block", execute)
|
|
schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s["function"]["name"] == tool)
|
|
chunks = await _client_chunks(_preview_generator(module, [schema]), False)
|
|
tool_event = next(e for e in _events(chunks) if e.get("type") == "tool_output")
|
|
assert tool_event["error_category"] == "invalid_tool_arguments"
|
|
assert hint in tool_event["output"]
|
|
assert "could not be validated" not in tool_event["output"]
|
|
|
|
|
|
def test_schema_hint_ignores_the_caught_exception_text():
|
|
from src.clean_agent_preview import _schema_argument_hint
|
|
schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s["function"]["name"] == "manage_notes")
|
|
# A ValidationError raised for arguments that are actually valid yields no
|
|
# hint, so the caller falls back to the curated schema message.
|
|
assert _schema_argument_hint({"action": "list"}, schema["function"]["parameters"]) == ""
|
|
hint = _schema_argument_hint({"action": SENSITIVE}, schema["function"]["parameters"])
|
|
assert "must be one of" in hint
|
|
assert "TAKEOVER_SECRET" not in hint
|