#!/usr/bin/env python3 """Focused real-route eval for implicit Odysseus memory questions. This uses the same Odysseus HTTP chat route as the broader harness evals, then adds memory-specific scoring so the next SFT slice can target the actual gap: natural "what do you remember/know about me" questions that should route to manage_memory before answering. """ from __future__ import annotations import argparse import json import re import time from pathlib import Path from typing import Any import httpx try: from scripts.eval_odysseus_tool_use import _cookie, run_case except ModuleNotFoundError: from eval_odysseus_tool_use import _cookie, run_case CASES: list[dict[str, Any]] = [ { "case": "memory_what_remember_nationality", "message": "what do you remember about my nationality?", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("nationality",), }, { "case": "memory_what_know_about_me", "message": "what do you know about me?", "expected_tool": "manage_memory", "expected_action": "list", }, { "case": "memory_saved_swedish", "message": "do you have anything saved about me being Swedish?", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("swedish",), }, { "case": "memory_family", "message": "what have I told you about my family?", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("family",), }, { "case": "memory_preferences", "message": "what do you remember about my preferences?", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("preference", "preferences"), }, { "case": "memory_location", "message": "what do you know about where I live?", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("live", "location", "reside", "residence"), }, { "case": "memory_explicit_control", "message": "search my saved memories for nationality", "expected_tool": "manage_memory", "expected_action": "search", "query_terms": ("nationality",), }, { "case": "chat_hi_control", "message": "hi", "expected_tool": "no_tool", "expected_action": "", }, { "case": "chat_identity_control", "message": "who are you?", "expected_tool": "no_tool", "expected_action": "", }, ] BAD_SURFACE_PATTERNS = ( r"\bdon['\u2019]?\s+have\b", r"\bi don['\u2019]?\b", r"\bdon['\u2019]?\s+retain\b", r"\bdon['\u2019]?\s+remember\b", r"\bdon'\b", r"\babou\b", r"\blis\b", r"\bfirs\b", r"\btha\b", r"\bwh\b", ) def _parse_command(raw: Any) -> tuple[str, str]: """Return action/query-ish text from a tool command payload.""" if isinstance(raw, dict): action = str(raw.get("action") or "").strip() query = str(raw.get("query") or raw.get("text") or raw.get("command") or "").strip() return action, query text = str(raw or "").strip() if not text: return "", "" try: parsed = json.loads(text) except json.JSONDecodeError: parsed = None if isinstance(parsed, dict): return _parse_command(parsed) lines = [line.strip() for line in text.splitlines() if line.strip()] if not lines: return "", "" action = lines[0] query_lines = [ line for line in lines[1:] if not line.startswith(" bool: value = response or "" return any(re.search(pattern, value, re.IGNORECASE) for pattern in BAD_SURFACE_PATTERNS) def annotate(record: dict[str, Any], case: dict[str, Any]) -> dict[str, Any]: metrics = record.get("metrics") or {} tool_events = metrics.get("tool_events") or [] memory_events = [event for event in tool_events if event.get("tool") == "manage_memory"] first_memory_action = "" first_memory_query = "" if memory_events: first_memory_action, first_memory_query = _parse_command(memory_events[0].get("command")) expected_tool = case["expected_tool"] expected_action = case.get("expected_action") or "" response = str(record.get("response") or "") no_tool = expected_tool == "no_tool" action_ok = no_tool or first_memory_action == expected_action query_terms = tuple(str(term).lower() for term in case.get("query_terms") or ()) query_lower = first_memory_query.lower() query_ok = no_tool or not query_terms or any(term in query_lower for term in query_terms) tool_ok = ( (record.get("tool_count") == 0 and no_tool) or (record.get("first_tool") == expected_tool) ) no_premature_denial = no_tool or not ( record.get("tool_count") == 0 and re.search(r"\b(i\s+)?do\s+not\b|\bi don['\u2019]?t\b|\bno saved memor", response, re.I) ) surface_ok = bool(response) and not _bad_surface(response) success = bool( tool_ok and action_ok and query_ok and no_premature_denial and surface_ok and not record.get("infra_failure") and not record.get("stream_errors") ) record.update( { "expected_action": expected_action, "first_memory_action": first_memory_action, "first_memory_query": first_memory_query, "memory_tool_ok": bool(tool_ok), "memory_action_ok": bool(action_ok), "memory_query_ok": bool(query_ok), "no_premature_memory_denial": bool(no_premature_denial), "memory_surface_ok": bool(surface_ok), "focused_success": success, "input_tokens": metrics.get("input_tokens"), "output_tokens": metrics.get("output_tokens"), "tokens_per_second": metrics.get("tokens_per_second"), } ) return record def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--base-url", default="http://127.0.0.1:7011") parser.add_argument("--endpoint", default="http://127.0.0.1:18051/v1") parser.add_argument("--endpoint-id", default="8b80db2d") parser.add_argument("--selected-endpoint-url", default="http://host.docker.internal:18051/v1") parser.add_argument("--model", default="qwen35-9b-tool-router-v31-recovery-from-base") parser.add_argument("--selected-model", default="qwen35-9b-tool-router-v31-recovery-from-base") parser.add_argument("--cookie-file", default="data/sessions.json") parser.add_argument("--prompt-mode", default="agent") parser.add_argument("--timeout", type=float, default=120.0) parser.add_argument("--hard-turn-timeout", type=float, default=60.0) parser.add_argument("--output", required=True) parser.add_argument("--cases", default="") parser.set_defaults(auto_approve=True, client_runtime_context=None) args = parser.parse_args() selected = {item.strip() for item in args.cases.split(",") if item.strip()} cases = [case for case in CASES if not selected or case["case"] in selected] unknown = selected - {case["case"] for case in CASES} if unknown: raise SystemExit(f"Unknown case(s): {', '.join(sorted(unknown))}") output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) records: list[dict[str, Any]] = [] with httpx.Client( cookies={"odysseus_session": _cookie(Path(args.cookie_file))}, follow_redirects=False, timeout=args.timeout + 20, ) as client: for case in cases: record = run_case( client, args, case["case"], case["message"], case["expected_tool"], ) record = annotate(record, case) records.append(record) print( json.dumps( { key: record.get(key) for key in ( "case", "message", "expected_tool", "expected_action", "first_tool", "first_memory_action", "first_memory_query", "memory_tool_ok", "memory_action_ok", "memory_query_ok", "no_premature_memory_denial", "memory_surface_ok", "focused_success", "input_tokens", "output_tokens", "elapsed_seconds", "response", ) }, ensure_ascii=True, ), flush=True, ) summary = { "model": args.selected_model or args.model, "created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "cases": len(records), "focused_success": sum(bool(r.get("focused_success")) for r in records), "memory_tool_success": sum(bool(r.get("memory_tool_ok")) for r in records), "memory_action_success": sum(bool(r.get("memory_action_ok")) for r in records), "memory_query_success": sum(bool(r.get("memory_query_ok")) for r in records), "surface_success": sum(bool(r.get("memory_surface_ok")) for r in records), "infra_failures": sum(bool(r.get("infra_failure")) for r in records), "stream_errors": sum(bool(r.get("stream_errors")) for r in records), "records": records, } output.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n") print("SUMMARY", json.dumps({k: v for k, v in summary.items() if k != "records"})) return 0 if summary["focused_success"] == summary["cases"] else 1 if __name__ == "__main__": raise SystemExit(main())