mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-11 18:52:21 +02:00
277 lines
10 KiB
Python
277 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Evaluate Qwen tool-routing rows through Odysseus streaming + parser code.
|
|
|
|
This is intentionally below the full chat HTTP route: it does not execute tools
|
|
or mutate user data. It uses the same Odysseus LLM request path and production
|
|
text parser that the agent loop uses after a local model streams text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import time
|
|
import uuid
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from src.llm_core import stream_llm
|
|
from src.tool_parsing import parse_tool_blocks
|
|
from src.tool_schemas import function_call_to_tool_block
|
|
|
|
|
|
def _sse_payloads(chunk: str) -> list[dict[str, Any]]:
|
|
payloads = []
|
|
for line in str(chunk or "").splitlines():
|
|
if not line.startswith("data: "):
|
|
continue
|
|
data = line[6:]
|
|
if data == "[DONE]":
|
|
continue
|
|
try:
|
|
payloads.append(json.loads(data))
|
|
except json.JSONDecodeError:
|
|
payloads.append({"type": "raw", "data": data})
|
|
return payloads
|
|
|
|
|
|
def _expected_block(row: dict[str, Any]):
|
|
call = row["messages"][-1]["tool_calls"][0]["function"]
|
|
return function_call_to_tool_block(call["name"], json.dumps(call.get("arguments") or {}))
|
|
|
|
|
|
def _expected_name(row: dict[str, Any]) -> str:
|
|
return row["messages"][-1]["tool_calls"][0]["function"]["name"]
|
|
|
|
|
|
def _same_tool_content(actual: str | None, expected: str | None) -> bool:
|
|
if actual == expected:
|
|
return True
|
|
if actual is None or expected is None:
|
|
return False
|
|
try:
|
|
actual_value = json.loads(actual)
|
|
expected_value = json.loads(expected)
|
|
except (TypeError, json.JSONDecodeError):
|
|
return False
|
|
return actual_value == expected_value
|
|
|
|
|
|
def _row_messages(row: dict[str, Any], mode: str) -> list[dict[str, str]]:
|
|
messages = row["messages"]
|
|
user = messages[1]["content"]
|
|
if mode == "row_system":
|
|
return [
|
|
{"role": "system", "content": messages[0]["content"]},
|
|
{"role": "user", "content": user},
|
|
]
|
|
if mode == "zero":
|
|
return [{"role": "user", "content": user}]
|
|
if mode == "tiny":
|
|
return [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"Use Odysseus native tool-call tags for explicit tool requests. "
|
|
"Make exactly one call, then stop."
|
|
),
|
|
},
|
|
{"role": "user", "content": user},
|
|
]
|
|
if mode == "compact_map":
|
|
return [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"You are Odysseus. For explicit requests, emit exactly one "
|
|
"native tool call, then stop. Use this map: "
|
|
"manage_notes=notes/checklists; "
|
|
"manage_documents=document library; "
|
|
"manage_calendar=calendar events; "
|
|
"manage_tasks=scheduled/recurring tasks; "
|
|
"manage_memory=saved memories; "
|
|
"search_chats=past chats; "
|
|
"read_file=explicit workspace paths; "
|
|
"mcp__email__list_emails=inbox/latest email; "
|
|
"mcp__email__search_emails=email subject/sender/topic search; "
|
|
"mcp__email__read_email=known email id."
|
|
),
|
|
},
|
|
{"role": "user", "content": user},
|
|
]
|
|
if mode == "compact_map_v2":
|
|
return [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"You are Odysseus. For explicit requests, emit exactly one "
|
|
"native tool call, then stop. Use: manage_notes for notes "
|
|
"and checklists; manage_documents for the document library; "
|
|
"manage_calendar for calendar events; manage_tasks for "
|
|
"scheduled or recurring tasks; manage_memory for saved "
|
|
"memories; search_chats for past chats; read_file for "
|
|
"explicit workspace paths. Email: use mcp__email__list_emails "
|
|
"with folder INBOX and max_results 20 when asked to find/read "
|
|
"an email by subject; use mcp__email__search_emails with "
|
|
"max_results 10 for mail search by sender/topic; use "
|
|
"mcp__email__read_email only with a known email id."
|
|
),
|
|
},
|
|
{"role": "user", "content": user},
|
|
]
|
|
if mode == "compact_map_v3":
|
|
return [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"Odysseus tools. Emit one native tool call, then stop. "
|
|
"manage_notes: notes/checklists. manage_documents: document "
|
|
"library. manage_calendar: calendar events. manage_tasks: "
|
|
"scheduled/recurring tasks. manage_memory: saved memories. "
|
|
"search_chats: past chats. read_file: workspace path. Email: "
|
|
"subject find+read -> mcp__email__list_emails {folder:INBOX,max_results:20}; "
|
|
"sender/topic search -> mcp__email__search_emails {max_results:10}; "
|
|
"known id -> mcp__email__read_email."
|
|
),
|
|
},
|
|
{"role": "user", "content": user},
|
|
]
|
|
raise ValueError(f"Unknown mode: {mode}")
|
|
|
|
|
|
def _select_rows(path: Path, per_group: int) -> list[dict[str, Any]]:
|
|
groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
with path.open() as f:
|
|
for line in f:
|
|
row = json.loads(line)
|
|
groups[_expected_name(row)].append(row)
|
|
selected = []
|
|
for name in sorted(groups):
|
|
selected.extend(groups[name][:per_group])
|
|
return selected
|
|
|
|
|
|
async def _run_one(args, row: dict[str, Any]) -> dict[str, Any]:
|
|
expected = _expected_block(row)
|
|
messages = _row_messages(row, args.mode)
|
|
started = time.monotonic()
|
|
text_parts: list[str] = []
|
|
stream_events: list[dict[str, Any]] = []
|
|
error = None
|
|
try:
|
|
async for chunk in stream_llm(
|
|
args.base_url,
|
|
args.model,
|
|
messages,
|
|
temperature=args.temperature,
|
|
max_tokens=args.max_tokens,
|
|
timeout=args.timeout,
|
|
tools=None,
|
|
session_id="tool-groups-" + uuid.uuid4().hex,
|
|
):
|
|
for payload in _sse_payloads(chunk):
|
|
stream_events.append(payload)
|
|
if isinstance(payload.get("delta"), str):
|
|
text_parts.append(payload["delta"])
|
|
elif payload.get("type") == "error":
|
|
error = payload
|
|
except Exception as exc: # noqa: BLE001 - eval should record failures
|
|
error = {"error": repr(exc)}
|
|
text = "".join(text_parts)
|
|
blocks = parse_tool_blocks(text, skip_fenced=True)
|
|
actual = blocks[0] if blocks else None
|
|
exact = bool(
|
|
expected
|
|
and actual
|
|
and actual.tool_type == expected.tool_type
|
|
and _same_tool_content(actual.content, expected.content)
|
|
)
|
|
tool_ok = bool(expected and actual and actual.tool_type == expected.tool_type)
|
|
return {
|
|
"group": expected.tool_type if expected else _expected_name(row),
|
|
"user": row["messages"][1]["content"],
|
|
"expected": {
|
|
"tool_type": expected.tool_type if expected else None,
|
|
"content": expected.content if expected else None,
|
|
},
|
|
"actual": {
|
|
"tool_type": actual.tool_type if actual else None,
|
|
"content": actual.content if actual else None,
|
|
},
|
|
"tool_ok": tool_ok,
|
|
"exact_ok": exact,
|
|
"parsed_tool_count": len(blocks),
|
|
"error": error,
|
|
"elapsed_seconds": round(time.monotonic() - started, 3),
|
|
"response": text[:1200],
|
|
}
|
|
|
|
|
|
async def _main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--rows", required=True)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:18046/v1")
|
|
parser.add_argument("--model", default="qwen35-9b-tool-router-v4-q4")
|
|
parser.add_argument("--mode", choices=["row_system", "compact_map", "compact_map_v2", "compact_map_v3", "tiny", "zero"], default="row_system")
|
|
parser.add_argument("--per-group", type=int, default=10)
|
|
parser.add_argument("--max-tokens", type=int, default=96)
|
|
parser.add_argument("--temperature", type=float, default=0.0)
|
|
parser.add_argument("--timeout", type=int, default=90)
|
|
parser.add_argument("--out", required=True)
|
|
args = parser.parse_args()
|
|
|
|
rows = _select_rows(Path(args.rows), args.per_group)
|
|
records = []
|
|
for i, row in enumerate(rows, 1):
|
|
record = await _run_one(args, row)
|
|
records.append(record)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"i": i,
|
|
"group": record["group"],
|
|
"tool_ok": record["tool_ok"],
|
|
"exact_ok": record["exact_ok"],
|
|
"elapsed_seconds": record["elapsed_seconds"],
|
|
"actual": record["actual"],
|
|
},
|
|
ensure_ascii=True,
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
by_group = {}
|
|
for record in records:
|
|
group = record["group"]
|
|
bucket = by_group.setdefault(group, {"n": 0, "tool_ok": 0, "exact_ok": 0, "errors": 0})
|
|
bucket["n"] += 1
|
|
bucket["tool_ok"] += int(record["tool_ok"])
|
|
bucket["exact_ok"] += int(record["exact_ok"])
|
|
bucket["errors"] += int(bool(record["error"]))
|
|
|
|
summary = {
|
|
"model": args.model,
|
|
"base_url": args.base_url,
|
|
"mode": args.mode,
|
|
"rows": str(Path(args.rows).resolve()),
|
|
"n": len(records),
|
|
"tool_ok": sum(int(r["tool_ok"]) for r in records),
|
|
"exact_ok": sum(int(r["exact_ok"]) for r in records),
|
|
"errors": sum(int(bool(r["error"])) for r in records),
|
|
"by_group": by_group,
|
|
"records": records,
|
|
}
|
|
out = Path(args.out)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n")
|
|
print("SUMMARY", json.dumps({k: v for k, v in summary.items() if k != "records"}, ensure_ascii=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(_main())
|