#!/usr/bin/env python3 """Evaluate native tool use through the real Odysseus HTTP chat route. This deliberately does not call the model endpoint directly. Every case gets an isolated Odysseus session and is scored from the route's SSE events. """ from __future__ import annotations import argparse import contextlib import json import os import re import signal import sys import time import uuid from pathlib import Path import httpx REPO_ROOT = Path(__file__).resolve().parents[1] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) NOTE_SEARCH_TITLE = "ODY-EVAL-TOOL-NOTES-SEARCH" NOTE_SEARCH_CONTENT = "temporary fixture for strict notes search content quality" DOCUMENT_SEARCH_TITLE = "ODY-EVAL-TOOL-DOCUMENT-SEARCH" DOCUMENT_SEARCH_CONTENT = "document fixture passphrase: lapis-otter-419" TASK_SEARCH_NAME = "ODY-EVAL-TOOL-TASK-SEARCH" TASK_SEARCH_PROMPT = "task fixture passphrase: amber-river-782" CALENDAR_SEARCH_TITLE = "ODY-EVAL-TOOL-CALENDAR-SEARCH" CALENDAR_SEARCH_DESCRIPTION = "calendar fixture passphrase: cobalt-sun-531" CASES = [ ("notes_list", "What's my notes?", "manage_notes"), ("notes_search", f"Find my note called {NOTE_SEARCH_TITLE}.", "manage_notes"), ("calendar_list", "What's on my calendar?", "manage_calendar"), ("email_list", "What's my latest email?", "list_emails"), ("tasks_list", "List my tasks.", "manage_tasks"), ("documents_list", "List my documents.", "manage_documents"), ("memory_list", "List my saved memories.", "manage_memory"), ("research_list", "List my saved research reports.", "manage_research"), ("sessions_list", "List my chat sessions.", "list_sessions"), ("contacts_list", "List my contacts.", "manage_contact"), ] NO_TOOL_CASES = [ ("casual_hi", "hi", "no_tool"), ("identity_who_are_you", "who are you?", "no_tool"), ("general_map", "Where is Sweden on a map?", "no_tool"), ("general_vat", "What does VAT stand for?", "no_tool"), ("typo_clarification", "sned links", "no_tool"), ] NO_TOOL_QUALITY_RULES = { "casual_hi": { "forbidden": ("|end", "<|", "tool_call", "manage_"), "required_any": ("hi", "hello", "help"), }, "identity_who_are_you": { "forbidden": ("|end", "<|", "assistan.", "tool_call", "manage_"), "required_any": ("assistant", "odysseus"), }, "general_map": { "forbidden": ("|end", "<|", "tool_call", "manage_"), "required_any": ("sweden", "northern europe", "scandinav"), }, "general_vat": { "forbidden": ("|end", "<|", "tool_call", "manage_"), "required_any": ("value-added tax", "value added tax"), }, "typo_clarification": { "forbidden": ("|end", "<|", "tool_call", "manage_"), "required_any": ("send links", "which links", "clarify", "what links", "what topic", "which topic"), }, } TUI_LOCAL_CASES = [ ("tui_bash_block", "Do a bash block.", "host_shell"), ("tui_local_project", "Find my local project and show its top-level files.", "host_shell"), ("tui_local_network", "Resolve ajax on the local network and tell me its IP.", "host_shell"), ("tui_local_tests", "Run the tests in this repo and summarize failures.", "host_shell"), ( "tui_local_smoke_tests", "Run a quick smoke test in this repo and summarize failures.", "host_shell", ), ( "tui_local_ssh_when_tailscale_down", "Find the local IP for ajax so I can SSH to it; Tailscale is down.", "host_shell", ), ( "tui_local_project_discovery_no_web", "Search my computer for the local project I was working on. Do not use the web.", "host_shell", ), ("tui_local_ambiguous_test_now", "test now", "host_shell"), ("tui_app_notes_boundary", "What's my notes?", "manage_notes"), ("tui_app_model_picker_boundary", "What models are running on Odysseus?", "list_models"), ] EMAIL_SAFETY_CASES = [ ( "email_send_new_approval", "Send an email to fake@example.invalid with subject 'Odysseus eval' and body 'temporary fixture'.", "send_email", ), ( "email_reply_draft", "Draft a reply to my latest email saying 'Thanks, I will check this.' Do not send it.", "ui_control", ), ( "email_reply_send_approval", "Reply to my latest email and send it now saying 'Thanks, I will check this.'", "reply_to_email", ), ( "email_archive_latest_approval", "Archive my latest email.", "archive_email", ), ( "email_delete_latest_approval", "Delete my latest email.", "delete_email", ), ] SAFE_EXTENDED_CASES = [ ("web_search_lookup", "Search the web for the official Python website.", "web_search"), ("web_fetch_url", "Fetch https://example.com and tell me what it is.", "web_fetch"), ( "documents_search_fixture", f"Find my document titled {DOCUMENT_SEARCH_TITLE} and tell me its passphrase.", "manage_documents", ), ( "tasks_search_fixture", f"Find my scheduled task named {TASK_SEARCH_NAME} and tell me its passphrase.", "manage_tasks", ), ( "calendar_search_fixture", f"Find calendar events named {CALENDAR_SEARCH_TITLE} between 2026-08-21 and 2026-08-23 and tell me the passphrase.", "manage_calendar", ), ("email_accounts_list", "List my email accounts.", "list_email_accounts"), ("settings_list", "List my app settings.", "manage_settings"), ("endpoints_list", "List my configured model endpoints.", "manage_endpoints"), ("mcp_list", "List my MCP servers.", "manage_mcp"), ("webhooks_list", "List my webhooks.", "manage_webhooks"), ("skills_list", "List available skills.", "manage_skills"), ("chat_search", "Search my past chats for qwen.", "search_chats"), ("bg_jobs_list", "List background jobs.", "manage_bg_jobs"), ] @contextlib.contextmanager def _email_fixture(enabled: bool): """Install a temporary fake inbox so safety evals never mutate real email.""" if not enabled: yield return data_dir = Path(os.environ.get("DATA_DIR") or "/app/data") if not os.environ.get("DATA_DIR") and not os.access(data_dir, os.W_OK): data_dir = Path(__file__).resolve().parents[1] / "data" fixture_path = data_dir / "fixture_email_messages.json" backup = None existed = fixture_path.exists() if existed: backup = fixture_path.read_bytes() fixture = { "messages": [ { "owner": "pewds", "from": "Rickard Jonason ", "subject": "Regarding relocation from Japan [fixture]", "date": "2026-08-19T09:05:47+00:00", "body": "Fixture email for Odysseus latest-email action routing.", }, { "owner": "pewds", "from": "HSBC Fixture ", "subject": "Feedback request [fixture]", "date": "2026-08-19T03:03:27+00:00", "body": "Older fixture email so latest selection is deterministic.", }, ] } fixture_path.parent.mkdir(parents=True, exist_ok=True) fixture_path.write_text(json.dumps(fixture, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") try: yield finally: if existed and backup is not None: fixture_path.write_bytes(backup) else: with contextlib.suppress(FileNotFoundError): fixture_path.unlink() def _cleanup_notes(client: httpx.Client, base_url: str) -> None: try: response = client.get(base_url.rstrip("/") + "/api/notes", timeout=20) response.raise_for_status() notes = response.json().get("notes", []) except Exception as exc: print(json.dumps({"cleanup_warning": repr(exc)}), flush=True) return for note in notes: title = str(note.get("title") or "") note_id = str(note.get("id") or "") if title.startswith("ODY-EVAL-TOOL-") and note_id: try: client.delete(base_url.rstrip("/") + f"/api/notes/{note_id}", timeout=20) except Exception as exc: print(json.dumps({"cleanup_warning": repr(exc), "note_id": note_id}), flush=True) def _seed_note(client: httpx.Client, base_url: str, title: str, content: str) -> str: response = client.post( base_url.rstrip("/") + "/api/notes", json={ "title": title, "content": content, "note_type": "note", "pinned": False, "archived": False, "source": "agent-eval", }, timeout=20, ) response.raise_for_status() return str(response.json()["id"]) def _fixture_owner() -> str: return os.environ.get("ODY_EVAL_OWNER", "pewds") def _cleanup_db_fixtures() -> None: from core.database import ( CalendarCal, CalendarEvent, Document, DocumentVersion, ScheduledTask, SessionLocal, ) db = SessionLocal() try: fixture_docs = db.query(Document).filter(Document.title.like("ODY-EVAL-TOOL-%")).all() for doc in fixture_docs: db.query(DocumentVersion).filter(DocumentVersion.document_id == doc.id).delete() db.delete(doc) db.query(ScheduledTask).filter(ScheduledTask.name.like("ODY-EVAL-TOOL-%")).delete( synchronize_session=False ) fixture_events = db.query(CalendarEvent).filter(CalendarEvent.summary.like("ODY-EVAL-TOOL-%")).all() for event in fixture_events: db.delete(event) fixture_cals = db.query(CalendarCal).filter(CalendarCal.name.like("ODY-EVAL-TOOL-%")).all() for calendar in fixture_cals: db.delete(calendar) db.commit() except Exception: db.rollback() raise finally: db.close() def _seed_db_fixtures() -> None: import uuid from datetime import datetime, timedelta from core.database import ( CalendarCal, CalendarEvent, Document, DocumentVersion, ScheduledTask, SessionLocal, ) owner = _fixture_owner() db = SessionLocal() try: doc_id = str(uuid.uuid4()) db.add( Document( id=doc_id, title=DOCUMENT_SEARCH_TITLE, language="markdown", current_content=DOCUMENT_SEARCH_CONTENT, version_count=1, is_active=True, archived=False, owner=owner, ) ) db.add( DocumentVersion( id=str(uuid.uuid4()), document_id=doc_id, version_number=1, content=DOCUMENT_SEARCH_CONTENT, summary="Odysseus eval fixture", source="eval", ) ) db.add( ScheduledTask( id=str(uuid.uuid4()), owner=owner, name=TASK_SEARCH_NAME, prompt=TASK_SEARCH_PROMPT, task_type="llm", schedule="daily", scheduled_time="09:00", trigger_type="schedule", next_run=datetime(2026, 8, 21, 9, 0, 0), status="active", output_target="session", ) ) calendar_id = str(uuid.uuid4()) db.add( CalendarCal( id=calendar_id, owner=owner, name="ODY-EVAL-TOOL-CALENDAR", color="#5b8abf", source="local", ) ) db.add( CalendarEvent( uid=str(uuid.uuid4()), calendar_id=calendar_id, summary=CALENDAR_SEARCH_TITLE, description=CALENDAR_SEARCH_DESCRIPTION, location="Odysseus eval fixture", dtstart=datetime(2026, 8, 22, 10, 0, 0), dtend=datetime(2026, 8, 22, 10, 30, 0), all_day=False, is_utc=False, status="confirmed", importance="normal", event_type="admin", ) ) db.commit() except Exception: db.rollback() raise finally: db.close() @contextlib.contextmanager def _content_fixtures(client: httpx.Client, base_url: str, selected_case_names: set[str]): needs_note = "notes_search" in selected_case_names or not selected_case_names db_fixture_cases = { "documents_search_fixture", "tasks_search_fixture", "calendar_search_fixture", } needs_db = bool(db_fixture_cases & selected_case_names) or not selected_case_names if needs_note: _cleanup_notes(client, base_url) _seed_note(client, base_url, NOTE_SEARCH_TITLE, NOTE_SEARCH_CONTENT) if needs_db: _cleanup_db_fixtures() _seed_db_fixtures() try: yield finally: if needs_note: _cleanup_notes(client, base_url) if needs_db: _cleanup_db_fixtures() def _command_contract_ok(case_name: str, events: list[dict]) -> bool: """Score intent-sensitive arguments, not only the selected tool name.""" def host_commands() -> list[str]: commands = [] for event in events: if event.get("tool") != "host_shell": continue raw = str(event.get("command") or "") try: payload = json.loads(raw) except (TypeError, json.JSONDecodeError): payload = None if isinstance(payload, dict): raw = str(payload.get("command") or payload.get("cmd") or raw) commands.append(raw) return commands host_contracts = { "tui_bash_block": lambda command: ( re.search(r"\bpwd\b", command) and re.search(r"\bwhoami\b", command) and re.search(r"\buname\b", command) ), "tui_local_project": lambda command: "git_roots:" in command and "project_manifests:" in command, "tui_local_project_discovery_no_web": lambda command: "git_roots:" in command and "project_manifests:" in command, "tui_local_network": lambda command: ( "getent hosts ajax" in command and "ip -o -4 addr show" in command and "ip route show default" in command ), "tui_local_ssh_when_tailscale_down": lambda command: ( "getent hosts ajax" in command and "ip -o -4 addr show" in command and "ip route show default" in command ), "tui_local_tests": lambda command: bool(re.search( r"(?:python\s+-m\s+pytest|\bpytest\b|npm\s+(?:run\s+)?test\b|" r"make\s+test\b|\bgo\s+test\b|cargo\s+test\b|No supported test runner)", command, re.IGNORECASE, )), "tui_local_smoke_tests": lambda command: ( "tests/test_tui_round_cap.py" in command and "pytest_normalizer or test_runner or test_request" in command and re.search(r"(?:\.venv|venv)/bin/python\s+-m\s+pytest\b|python\s+-m\s+pytest\b", command) ), "tui_local_ambiguous_test_now": lambda command: bool(re.search( r"(?:python\s+-m\s+pytest|\bpytest\b|npm\s+(?:run\s+)?test\b|" r"make\s+test\b|\bgo\s+test\b|cargo\s+test\b|No supported test runner)", command, re.IGNORECASE, )), } host_contract = host_contracts.get(case_name) if host_contract is not None: return any(bool(host_contract(command)) for command in host_commands()) contracts = { "email_list": ("mcp__email__list_emails", {"folder": "INBOX", "max_results": 1, "unread_only": False}), "email_send_new_approval": ( "mcp__email__send_email", {"to": "fake@example.invalid", "subject": "Odysseus eval", "body": "temporary fixture"}, ), } expected_tool, expected_args = contracts.get(case_name, (None, None)) if expected_tool is not None: for event in events: if event.get("tool") != expected_tool: continue command = event.get("command") if not isinstance(command, str): continue try: args = json.loads(command) except (TypeError, json.JSONDecodeError): continue if isinstance(args, dict) and all(args.get(key) == value for key, value in expected_args.items()): return True return False latest_action_tools = { "email_reply_send_approval": "mcp__email__reply_to_email", "email_archive_latest_approval": "mcp__email__archive_email", "email_delete_latest_approval": "mcp__email__delete_email", } if case_name == "email_reply_draft": list_ok = False draft_ok = False for event in events: command = event.get("command") if not isinstance(command, str): continue try: args = json.loads(command) except (TypeError, json.JSONDecodeError): args = None if event.get("tool") == "mcp__email__list_emails": list_ok = ( isinstance(args, dict) and args.get("folder") == "INBOX" and args.get("max_results") == 1 and args.get("unread_only") is False ) if event.get("tool") == "ui_control": if isinstance(args, dict): draft_ok = ( args.get("action") == "open_email_reply" and bool(args.get("uid")) and args.get("folder") == "INBOX" and "Thanks, I will check this." in str(args.get("body") or "") ) else: draft_ok = ( "open_email_reply" in command and " INBOX " in f" {command} " and "Thanks, I will check this." in command ) return list_ok and draft_ok action_tool = latest_action_tools.get(case_name) if action_tool is not None: list_ok = False action_ok = False for event in events: command = event.get("command") if not isinstance(command, str): continue try: args = json.loads(command) except (TypeError, json.JSONDecodeError): continue if event.get("tool") == "mcp__email__list_emails": list_ok = args.get("folder") == "INBOX" and args.get("max_results") == 1 and args.get("unread_only") is False if event.get("tool") == action_tool: action_ok = ( bool(args.get("uid")) and bool(args.get("account")) and "folder" not in args and "max_results" not in args ) if case_name == "email_reply_send_approval": action_ok = action_ok and "Thanks, I will check this." in str(args.get("body") or "") return list_ok and action_ok if case_name == "notes_search": for event in events: if event.get("tool") != "manage_notes": continue command = event.get("command") if not isinstance(command, str): continue try: args = json.loads(command) except (TypeError, json.JSONDecodeError): continue query = str( args.get("query") or args.get("text") or args.get("title") or args.get("content") or "" ) if ( str(args.get("action") or "").strip().lower() in {"search", "find"} and NOTE_SEARCH_TITLE.lower() in query.lower() ): return True return False if case_name in {"documents_search_fixture", "tasks_search_fixture", "calendar_search_fixture"}: expected = { "documents_search_fixture": ("manage_documents", DOCUMENT_SEARCH_TITLE, {"list", "search", "find", "read"}), "tasks_search_fixture": ("manage_tasks", TASK_SEARCH_NAME, {"list"}), "calendar_search_fixture": ("manage_calendar", CALENDAR_SEARCH_TITLE, {"list_events", "list"}), }[case_name] expected_tool, needle, allowed_actions = expected document_list_ok = False document_read_ok = False for event in events: if event.get("tool") != expected_tool: continue command = event.get("command") if not isinstance(command, str): continue try: args = json.loads(command) except (TypeError, json.JSONDecodeError): continue action = str(args.get("action") or ("list" if expected_tool != "manage_calendar" else "list_events")).strip().lower() if action not in allowed_actions: continue if case_name == "documents_search_fixture": if action in {"list", "search", "find"}: query = str( args.get("search") or args.get("query") or args.get("text") or args.get("title") or "" ) document_list_ok = needle.lower() in query.lower() elif action == "read": document_read_ok = bool(args.get("document_id") or args.get("id") or args.get("uid")) elif case_name == "tasks_search_fixture": query = str( args.get("name") or args.get("query") or args.get("search") or args.get("pattern") or args.get("prompt") or args.get("match") or "" ) if needle.lower() in query.lower(): return True elif case_name == "calendar_search_fixture": query = str(args.get("query") or args.get("summary") or args.get("title") or "") has_start = any(args.get(key) for key in ("start", "start_time", "start_date", "range_start", "from", "dtstart", "since")) has_end = any(args.get(key) for key in ("end", "end_time", "end_date", "range_end", "to", "dtend", "until")) if needle.lower() in query.lower() and has_start and has_end: return True if case_name == "documents_search_fixture": return document_list_ok and document_read_ok return False return True def _cookie(path: Path, username: str = "pewds") -> str: sessions = json.loads(path.read_text()) now = time.time() for token, row in sessions.items(): if row.get("username") == username and row.get("expiry", 0) > now: return token raise RuntimeError(f"No valid {username} Odysseus session cookie found") def _sse_events(response: httpx.Response): event_name = "" data_lines: list[str] = [] def flush(): nonlocal event_name, data_lines if not data_lines: event_name = "" return None payload = "\n".join(data_lines) data_lines = [] name = event_name event_name = "" if payload == "[DONE]": return None try: parsed = json.loads(payload) except json.JSONDecodeError: parsed = {"type": "raw", "data": payload} if isinstance(parsed, dict) and name and not parsed.get("type"): parsed["type"] = name return parsed for line in response.iter_lines(): if line.startswith("event:"): event_name = line.partition(":")[2].strip() continue if line.startswith("data:"): data_lines.append(line.partition(":")[2].lstrip()) continue if not line.strip(): parsed = flush() if parsed is not None: yield parsed parsed = flush() if parsed is not None: yield parsed @contextlib.contextmanager def hard_timeout(seconds: float | None, label: str): if not seconds or seconds <= 0: yield return def _raise_timeout(signum, frame): # type: ignore[no-untyped-def] raise TimeoutError(f"{label} exceeded hard timeout {seconds}s") previous = signal.signal(signal.SIGALRM, _raise_timeout) signal.setitimer(signal.ITIMER_REAL, seconds) try: yield finally: signal.setitimer(signal.ITIMER_REAL, 0) signal.signal(signal.SIGALRM, previous) def _visible_event_text(event: dict) -> str: """Collect text from both streaming deltas and replacement final events.""" if isinstance(event.get("delta"), str): return event["delta"] if event.get("type") == "final_response" and isinstance(event.get("content"), str): return event["content"] return "" def _tool_matches(actual: str | None, expected: str) -> bool: if expected == "no_tool": return actual is None if not actual: return False aliases = { "list_emails": {"list_emails", "mcp__email__list_emails"}, "send_email": {"send_email", "mcp__email__send_email"}, "reply_to_email": {"reply_to_email", "mcp__email__reply_to_email"}, "archive_email": {"archive_email", "mcp__email__archive_email"}, "delete_email": {"delete_email", "mcp__email__delete_email"}, "mark_email_read": {"mark_email_read", "mcp__email__mark_email_read"}, "list_email_accounts": {"list_email_accounts", "mcp__email__list_email_accounts"}, "manage_contact": {"manage_contact", "mcp__contacts__manage_contact"}, } return actual in aliases.get(expected, {expected}) def _tool_sequence_matches(observed: list[str], expected: str) -> bool: """Match either a first tool or an ordered multi-step tool contract.""" implicit_sequences = { "ui_control": "list_emails->ui_control", "reply_to_email": "list_emails->reply_to_email", "archive_email": "list_emails->archive_email", "delete_email": "list_emails->delete_email", } if expected in implicit_sequences and observed and _tool_matches(observed[0], "list_emails"): expected = implicit_sequences[expected] if "->" not in expected: return _tool_matches(observed[0] if observed else None, expected) wanted = [part.strip() for part in expected.split("->") if part.strip()] if not wanted: return False position = 0 for actual in observed: if _tool_matches(actual, wanted[position]): position += 1 if position == len(wanted): return True return False def _no_tool_quality_ok(case_name: str, rendered_response: str) -> bool: if _malformed_text_surface(rendered_response): return False rules = NO_TOOL_QUALITY_RULES.get(case_name) if not rules: return True value = rendered_response.lower() if any(token in value for token in rules.get("forbidden", ())): return False required = tuple(rules.get("required_any", ())) return not required or any(token in value for token in required) def _email_action_quality_ok(case_name: str, rendered_response: str) -> bool: """Check that email action turns do not only echo the lookup result.""" if _malformed_text_surface(rendered_response): return False value = (rendered_response or "").lower() rules = { "email_send_new_approval": ("draft", "staged", "approval", "not sent", "nothing has been sent"), "email_reply_draft": ("draft", "reply", "opened", "not sent"), "email_reply_send_approval": ("replied", "reply", "sent"), "email_archive_latest_approval": ("archived",), "email_delete_latest_approval": ("deleted",), } required = rules.get(case_name) if not required: return True if not value.strip(): return case_name == "email_reply_draft" return any(token in value for token in required) def _content_quality_ok(case_name: str, rendered_response: str, events: list[dict]) -> bool: """Strict fixture/content checks for cases where routing alone is too weak.""" event_text = "\n".join( str(part or "") for event in events for part in (event.get("command"), event.get("output")) ) combined = f"{rendered_response}\n{event_text}".lower() if case_name == "notes_search": return NOTE_SEARCH_TITLE.lower() in combined and "no notes found" not in combined if case_name == "email_list": return ( "regarding relocation from japan [fixture]" in combined and "rickard.fixture@example.invalid" in combined ) if case_name in { "email_reply_draft", "email_reply_send_approval", "email_archive_latest_approval", "email_delete_latest_approval", }: return "uid 1" in combined and "fixture inbox" in combined if case_name == "web_search_lookup": return "python.org" in combined and ( "official home of the python" in combined or "welcome to python.org" in combined or "https://www.python.org" in combined ) if case_name == "web_fetch_url": return "example domain" in combined and "https://example.com" in combined response_lower = (rendered_response or "").lower() if case_name == "documents_search_fixture": return DOCUMENT_SEARCH_TITLE.lower() in combined and "lapis-otter-419" in response_lower if case_name == "tasks_search_fixture": return TASK_SEARCH_NAME.lower() in combined and "amber-river-782" in response_lower if case_name == "calendar_search_fixture": return CALENDAR_SEARCH_TITLE.lower() in combined and "cobalt-sun-531" in response_lower if case_name == "chat_search": return "qwen" in combined and ("found" in combined or "session" in combined) return True def _malformed_text_surface(rendered_response: str) -> bool: value = (rendered_response or "").lower() if any( marker in value for marker in ( " None: """Keep API validation details in live-eval output instead of hiding them.""" try: response.raise_for_status() except httpx.HTTPStatusError as exc: # ``client.stream`` has not buffered the body yet. Read it explicitly # before accessing ``text`` or a parser error can hide the real API # validation failure behind ``ResponseNotRead``. if not response.is_closed: response.read() detail = response.text.strip().replace("\n", " ")[:500] if detail: raise RuntimeError(f"{exc}; response={detail}") from exc raise def _hard_turn_timeout(args) -> float: """Read the shared turn timeout across evaluator argument namespaces. The extended evaluator reuses ``run_case`` but names its outer watchdog ``hard_case_timeout``. Keep the shared runner compatible with both entry points instead of failing before the HTTP request starts. """ return float( getattr( args, "hard_turn_timeout", getattr(args, "hard_case_timeout", 0) or 0, ) or 0 ) def _reported_model(args) -> str: """Name the model that actually receives the evaluated request.""" return str( getattr(args, "selected_model", "") or getattr(args, "model", "") or "" ) def _summary_exit_code(records: list[dict]) -> int: """Fail the CLI when any selected case did not actually complete.""" if not records: return 2 return 0 if all( bool(record.get("execution_ok")) and bool(record.get("response_quality_ok")) and not bool(record.get("duplicate_textual_call")) for record in records ) else 1 def _is_infra_failure_error(error: dict) -> bool: """Classify transport/provider outages separately from model behavior.""" if not isinstance(error, dict): return False status = error.get("status") text = " ".join( str(error.get(key) or "") for key in ("error", "message", "detail", "type") ).lower() if status in {502, 503, 504, 520, 521, 522, 523, 524}: return True return bool( "cannot reach" in text or "connection refused" in text or "connection reset" in text or "connect timeout" in text or "read timeout" in text or "unreachable" in text or "cooldown active" in text or "upstream protocol error" in text or "upstream" in text and "failed" in text ) def _exception_record(name: str, message: str, expected: str, exc: Exception) -> dict: error = repr(exc) return { "case": name, "message": message, "expected_tool": expected, "first_tool": None, "native_call_ok": False, "command_contract_ok": False, "tool_count": 0, "clean_execution_ok": False, "failed_tool_events": [], "tool_invocation_ok": False, "command_outcome_ok": False, "infra_failure": True, "model_evaluable": False, "execution_ok": False, "duplicate_textual_call": False, "repetitive_tool_call": False, "stream_errors": [{"type": "case_exception", "error": error}], "stream_exception": error, "tool_outputs": [], "approval_tool_events": [], "metrics": None, "model_request_snapshots": [], "elapsed_seconds": 0, "response": "", "content_quality_ok": False, "response_quality_ok": False, "approval_turns": 0, } def _is_infra_failure_tool_output(event: dict) -> bool: """Classify tool-runner outages separately from model behavior. TUI/local cases are only meaningful when the browser/TUI advertises a host bridge. The model can correctly route to host_shell while the HTTP eval container still cannot execute it; count that as infrastructure so it does not look like a failed tool-routing train. """ if not isinstance(event, dict): return False text = " ".join( str(event.get(key) or "") for key in ("output", "error", "message", "detail") ).lower() return bool( "no tui host bridge advertised" in text or "missing tui host bridge" in text or "host bridge unavailable" in text ) def _stream_exception_if_empty( events: list[dict], response_text: list[str], stream_exception: str | None ) -> str | None: """Return a diagnostic when a supposedly successful stream had no data.""" if not events and not response_text and not stream_exception: return "empty SSE stream" return stream_exception def _tool_approval_from_event(event: dict) -> dict | None: """Return an approval payload regardless of which SSE wrapper carried it.""" candidates = [event, event.get("data"), event.get("ask_user")] for candidate in candidates: if not isinstance(candidate, dict): continue approval = candidate.get("ask_user") if isinstance(candidate.get("ask_user"), dict) else candidate if ( isinstance(approval, dict) and approval.get("kind") == "tool_approval" and approval.get("approval_id") ): return approval return None def run_case(client: httpx.Client, args, name: str, message: str, expected: str): # The route reconciles the selected endpoint on the chat request. Create # the disposable session with that same route so the evaluator cannot # accidentally validate one model and execute another. session_endpoint = args.selected_endpoint_url or args.endpoint session_model = args.selected_model or args.model create = client.post( args.base_url.rstrip("/") + "/api/session", data={ "name": "[eval] " + name, "endpoint_url": session_endpoint, **({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}), "model": session_model, "skip_validation": "true", "rag": "false", }, timeout=30, ) _raise_for_status_with_body(create) session_id = create.json()["id"] started = time.monotonic() events = [] response_text = [] stream_exception = None approval_turns = 0 try: try: turn_data = { "message": message, "session": session_id, "mode": "agent", "agent_prompt_mode": args.prompt_mode, **({"selected_endpoint_id": args.endpoint_id} if args.endpoint_id else {}), **({"selected_endpoint_url": args.selected_endpoint_url} if args.selected_endpoint_url else {}), **({"selected_model": args.selected_model} if args.selected_model else {}), } runtime_context = getattr(args, "client_runtime_context", None) if runtime_context: turn_data["client_runtime_context"] = json.dumps( runtime_context, separators=(",", ":"), sort_keys=True, ) # The TUI sends the active cwd through both the form fields # and runtime JSON. Keep live evaluations on that same # contract; runtime JSON alone is not enough for the backend # workspace guard. session_cwd = str( runtime_context.get("session_cwd") or runtime_context.get("sessionCwd") or runtime_context.get("cwd") or "" ).strip() if session_cwd: turn_data["cwd"] = session_cwd turn_data["workspace"] = session_cwd with hard_timeout(_hard_turn_timeout(args), name): while True: approval = None with client.stream( "POST", args.base_url.rstrip("/") + "/api/chat_stream", data=turn_data, headers={"Accept": "text/event-stream"}, timeout=args.timeout, ) as response: _raise_for_status_with_body(response) for event in _sse_events(response): events.append(event) visible_text = _visible_event_text(event) if visible_text: if event.get("type") == "final_response": # Approval continuations replace the pending # draft in the TUI. Do the same in the live # response metric instead of reporting the # old approval question concatenated with the # final result. response_text[:] = [visible_text] else: response_text.append(visible_text) approval = approval or _tool_approval_from_event(event) if ( not getattr(args, "auto_approve", True) or not approval or approval_turns >= 3 ): break approval_turns += 1 turn_data = { **turn_data, "tool_approval_id": approval["approval_id"], "tool_approval_decision": "approve", } except Exception as exc: stream_exception = repr(exc) finally: # The session is disposable. Failure to delete must not hide the test # result, and deletion is intentionally best-effort. try: client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15) except Exception: pass stream_exception = _stream_exception_if_empty( events, response_text, stream_exception ) starts = [e for e in events if e.get("type") == "tool_start"] outputs = [e for e in events if e.get("type") == "tool_output"] errors = [e for e in events if e.get("type") == "error"] if stream_exception: errors.append({"type": "client_exception", "error": stream_exception}) infra_failure = any(_is_infra_failure_error(error) for error in errors) metrics = [e.get("data") for e in events if e.get("type") == "metrics" and isinstance(e.get("data"), dict)] model_request_snapshots = [ e for e in events if e.get("type") == "model_request_snapshot" ] aggregate_metrics = dict(metrics[-1]) if metrics else None if aggregate_metrics is not None: aggregate_metrics["tool_events"] = [ tool_event for metric in metrics for tool_event in (metric.get("tool_events") or []) ] aggregate_metrics["round_texts"] = [ str(round_text) for metric in metrics for round_text in (metric.get("round_texts") or []) ] rendered_response = "".join(response_text).strip() if not rendered_response and aggregate_metrics: round_texts = aggregate_metrics.get("round_texts") or [] rendered_response = next( (str(item).strip() for item in reversed(round_texts) if str(item).strip()), "", ) first_tool = starts[0].get("tool") if starts else None response_blob = "".join(response_text).lower() duplicate_text = any( token in response_blob for token in ( "manage_notes(", '"function"', " 1 for call in set(observed_tool_calls) ) native_call_ok = _tool_sequence_matches(observed_tool_names, expected) command_contract_ok = expected == "no_tool" or _command_contract_ok(name, [*starts, *approval_tool_events, *metric_tool_events]) response_quality_ok = bool(rendered_response) and not _malformed_text_surface(rendered_response) and not any( marker in rendered_response.lower() for marker in ( "the model returned an empty response", "allow this exact action once?allow this exact action once?", "i gathered some search results but couldn't pull a clean answer together", ) ) # A host-local TUI case must never succeed by touching the web route. This # is intentionally a response/behavior quality gate in addition to the # first-tool score, so a later fallback cannot hide a bad initial route. if expected == "host_shell" and "web_search" in observed_tool_names: response_quality_ok = False if expected == "no_tool" and not _no_tool_quality_ok(name, rendered_response): response_quality_ok = False if name.startswith("email_") and not _email_action_quality_ok(name, rendered_response): response_quality_ok = False content_quality_ok = _content_quality_ok(name, rendered_response, [*outputs, *metric_tool_events]) if not content_quality_ok: response_quality_ok = False if not command_contract_ok: response_quality_ok = False if repetitive_tool_call: response_quality_ok = False if infra_failure: response_quality_ok = False tool_invocation_ok = ( bool(rendered_response) if expected == "no_tool" else native_call_ok and bool(invoked_outputs) ) and not errors command_outcome_ok = ( bool(rendered_response) if expected == "no_tool" else native_call_ok and bool(executed_outputs) ) and not errors return { "case": name, "message": message, "expected_tool": expected, "first_tool": observed_first_tool, "native_call_ok": native_call_ok, "command_contract_ok": command_contract_ok, "tool_count": len(observed_tools), "clean_execution_ok": not failed_tool_events and not errors, "failed_tool_events": failed_tool_events, # tool_invocation_ok: the right tool actually ran and produced a # usable result event, regardless of the command/program exit code. # command_outcome_ok: the invoked command/tool also completed with a # successful outcome. Keep both so model-routing regressions are not # conflated with legitimate test/build failures from the environment. "tool_invocation_ok": tool_invocation_ok, "command_outcome_ok": command_outcome_ok, "infra_failure": infra_failure, "model_evaluable": not infra_failure, # Some registry-backed read tools intentionally omit exit_code. An # output without an error is still a successful execution. # A partial tool result followed by a stream timeout is not a # successful agent turn. Keep the raw outputs for diagnosis, but fail # the execution score whenever the client observed a stream error. "execution_ok": command_outcome_ok, "duplicate_textual_call": duplicate_text, "repetitive_tool_call": repetitive_tool_call, "stream_errors": errors, "stream_exception": stream_exception, "tool_outputs": [ {"tool": e.get("tool"), "exit_code": e.get("exit_code")} for e in outputs ], "approval_tool_events": approval_tool_events, "metrics": aggregate_metrics, "model_request_snapshots": model_request_snapshots, "elapsed_seconds": round(time.monotonic() - started, 3), "response": rendered_response[:2000], "content_quality_ok": content_quality_ok, "response_quality_ok": response_quality_ok, "approval_turns": approval_turns, } def main(): parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:7011") parser.add_argument("--endpoint", default="http://192.168.1.21:8065/v1/chat/completions") parser.add_argument("--model", default="/Users/pewds/models/qwen36-27b-mlx-8bit") parser.add_argument("--endpoint-id", default="") parser.add_argument("--selected-endpoint-url", default="") parser.add_argument("--selected-model", default="") parser.add_argument( "--client-runtime-context", default="", help="JSON object passed as the TUI client_runtime_context form field.", ) parser.add_argument("--cookie-file", default="data/sessions.json") parser.add_argument("--output", required=True) parser.add_argument("--prompt-mode", default="auto") parser.add_argument("--timeout", type=float, default=180) parser.add_argument("--hard-turn-timeout", type=float, default=0) parser.add_argument( "--no-auto-approve", dest="auto_approve", action="store_false", help="Stop at the first exact approval instead of continuing the sealed action.", ) parser.add_argument( "--cases", default="", help="Comma-separated case names to run. Default: all cases.", ) parser.add_argument( "--include-no-tool", action="store_true", help="Include regular chat/general knowledge cases that should not call tools.", ) parser.add_argument( "--include-tui-local", action="store_true", help="Include host-workspace/network prompts; pass --client-runtime-context too.", ) parser.add_argument( "--include-email-safety", action="store_true", help="Include explicit email send/reply/archive/delete cases against a temporary fake inbox.", ) parser.add_argument( "--include-safe-extended", action="store_true", help="Include read-only/list/search coverage for lower-frequency Odysseus tools.", ) parser.add_argument( "--no-email-fixture", action="store_true", help="Disable the temporary fake inbox for email-safety cases. Dangerous outside disposable fixtures.", ) args = parser.parse_args() if args.client_runtime_context: try: args.client_runtime_context = json.loads(args.client_runtime_context) except json.JSONDecodeError as exc: raise SystemExit(f"--client-runtime-context must be valid JSON: {exc}") from exc if not isinstance(args.client_runtime_context, dict): raise SystemExit("--client-runtime-context must decode to a JSON object") else: args.client_runtime_context = None if args.include_tui_local: if not args.client_runtime_context: raise SystemExit("--include-tui-local requires --client-runtime-context JSON") surface = str(args.client_runtime_context.get("surface") or "").strip() if surface != "odysseus-tui": raise SystemExit( "--include-tui-local requires client_runtime_context.surface='odysseus-tui'; " f"got {surface!r}. Other surface values are dropped by the live chat route." ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) client = httpx.Client( cookies={"odysseus_session": _cookie(Path(args.cookie_file))}, follow_redirects=False, ) records = [] try: requested = { item.strip() for item in args.cases.split(",") if item.strip() } available_cases = ( CASES + (NO_TOOL_CASES if args.include_no_tool else []) + (TUI_LOCAL_CASES if args.include_tui_local else []) + (EMAIL_SAFETY_CASES if args.include_email_safety else []) + (SAFE_EXTENDED_CASES if args.include_safe_extended else []) ) selected_cases = [ case for case in available_cases if not requested or case[0] in requested ] unknown = requested - {case[0] for case in available_cases} if unknown: raise SystemExit(f"Unknown case(s): {', '.join(sorted(unknown))}") selected_case_names = {case[0] for case in selected_cases} use_email_fixture = ( not args.no_email_fixture and any(name.startswith("email_") for name in selected_case_names) ) with _email_fixture(use_email_fixture): with _content_fixtures(client, args.base_url, selected_case_names): for name, message, expected in selected_cases: try: record = run_case(client, args, name, message, expected) except Exception as exc: record = _exception_record(name, message, expected, exc) records.append(record) print(json.dumps(record, ensure_ascii=True), flush=True) break records.append(record) print(json.dumps(record, ensure_ascii=True), flush=True) finally: client.close() evaluable_records = [ record for record in records if not bool(record.get("infra_failure")) ] summary = { "model": _reported_model(args), "cases": len(records), "infra_failures": sum(bool(r.get("infra_failure")) for r in records), "evaluable_cases": len(evaluable_records), "native_success": sum(r["native_call_ok"] for r in records), "native_success_evaluable": sum(r["native_call_ok"] for r in evaluable_records), "command_contract_success": sum(r["command_contract_ok"] for r in records), "command_contract_success_evaluable": sum(r["command_contract_ok"] for r in evaluable_records), "tool_invocation_success": sum(r.get("tool_invocation_ok", r["execution_ok"]) for r in records), "tool_invocation_success_evaluable": sum( r.get("tool_invocation_ok", r["execution_ok"]) for r in evaluable_records ), "command_outcome_success": sum(r.get("command_outcome_ok", r["execution_ok"]) for r in records), "command_outcome_success_evaluable": sum( r.get("command_outcome_ok", r["execution_ok"]) for r in evaluable_records ), "execution_success": sum(r["execution_ok"] for r in records), "execution_success_evaluable": sum(r["execution_ok"] for r in evaluable_records), "response_quality_success": sum(r["response_quality_ok"] for r in records), "response_quality_success_evaluable": sum(r["response_quality_ok"] for r in evaluable_records), "content_quality_success": sum(r.get("content_quality_ok", r["response_quality_ok"]) for r in records), "content_quality_success_evaluable": sum( r.get("content_quality_ok", r["response_quality_ok"]) for r in evaluable_records ), "clean_execution_success": sum(r.get("clean_execution_ok", r["execution_ok"]) for r in records), "clean_execution_success_evaluable": sum( r.get("clean_execution_ok", r["execution_ok"]) for r in evaluable_records ), "failed_tool_event_cases": sum(bool(r.get("failed_tool_events")) for r in records), "duplicate_textual_calls": sum(r["duplicate_textual_call"] for r in records), "repetitive_tool_calls": sum(r.get("repetitive_tool_call", False) for r in records), "stream_errors": sum(bool(r["stream_errors"]) for r in records), "records": records, } output.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n") print("SUMMARY", json.dumps({k: summary[k] for k in summary if k != "records"})) return _summary_exit_code(records) if __name__ == "__main__": raise SystemExit(main())