mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-13 11:42:20 +02:00
300 lines
8.9 KiB
Python
300 lines
8.9 KiB
Python
from types import SimpleNamespace
|
|
|
|
from scripts.eval_odysseus_tool_use import (
|
|
_hard_turn_timeout,
|
|
_command_contract_ok,
|
|
_reported_model,
|
|
_sse_events,
|
|
_stream_exception_if_empty,
|
|
_summary_exit_code,
|
|
_tool_sequence_matches,
|
|
run_case,
|
|
)
|
|
from scripts.eval_odysseus_crud import build_summary
|
|
from scripts.eval_qwen35_tool_router_extended import cleanup_notes
|
|
|
|
|
|
def test_shared_eval_runner_accepts_extended_evaluator_timeout_name() -> None:
|
|
assert _hard_turn_timeout(SimpleNamespace(hard_case_timeout=30)) == 30.0
|
|
|
|
|
|
def test_shared_eval_runner_defaults_without_timeout_attribute() -> None:
|
|
assert _hard_turn_timeout(SimpleNamespace()) == 0.0
|
|
|
|
|
|
def test_eval_summary_reports_selected_model_over_discovered_model() -> None:
|
|
args = SimpleNamespace(selected_model="router/v12", model="router/v13")
|
|
assert _reported_model(args) == "router/v12"
|
|
|
|
|
|
def test_crud_summary_survives_partial_or_infrastructure_records() -> None:
|
|
summary = build_summary(
|
|
[{
|
|
"workflow": "memory",
|
|
"turns": [{
|
|
"native_call_ok": None,
|
|
"first_action_ok": None,
|
|
"tool_count_ok": None,
|
|
"exact_args_ok": None,
|
|
"execution_ok": None,
|
|
"duplicate_textual_call": None,
|
|
"stream_errors": [{"type": "infra_exception"}],
|
|
}],
|
|
}],
|
|
"router",
|
|
"fixture",
|
|
)
|
|
assert summary["turns"] == 1
|
|
assert summary["native_success"] == 0
|
|
assert summary["stream_errors"] == 1
|
|
|
|
|
|
def test_eval_sse_parser_preserves_standard_event_name() -> None:
|
|
class Response:
|
|
def iter_lines(self):
|
|
return iter((
|
|
"event: error",
|
|
'data: {"error":"upstream unavailable","status":502}',
|
|
"",
|
|
))
|
|
|
|
assert list(_sse_events(Response())) == [
|
|
{
|
|
"type": "error",
|
|
"error": "upstream unavailable",
|
|
"status": 502,
|
|
}
|
|
]
|
|
|
|
|
|
def test_empty_sse_stream_is_not_treated_as_a_valid_turn() -> None:
|
|
assert _stream_exception_if_empty([], [], None) == "empty SSE stream"
|
|
assert _stream_exception_if_empty([], [], "upstream error") == "upstream error"
|
|
|
|
|
|
def test_eval_summary_exit_code_rejects_failed_execution() -> None:
|
|
failed = {"execution_ok": False, "response_quality_ok": True, "duplicate_textual_call": False}
|
|
passed = {"execution_ok": True, "response_quality_ok": True, "duplicate_textual_call": False}
|
|
assert _summary_exit_code([failed]) == 1
|
|
assert _summary_exit_code([passed]) == 0
|
|
|
|
|
|
def test_eval_tool_contract_accepts_ordered_coding_sequence() -> None:
|
|
assert _tool_sequence_matches(
|
|
["read_file", "edit_file"], "read_file->edit_file"
|
|
)
|
|
assert not _tool_sequence_matches(
|
|
["edit_file", "read_file"], "read_file->edit_file"
|
|
)
|
|
|
|
|
|
def test_notes_search_contract_rejects_broad_list_with_ignored_search_key() -> None:
|
|
assert not _command_contract_ok(
|
|
"notes_search",
|
|
[{
|
|
"tool": "manage_notes",
|
|
"command": '{"action":"list","search":"ODY-EVAL-TOOL-NOTES-SEARCH"}',
|
|
}],
|
|
)
|
|
assert _command_contract_ok(
|
|
"notes_search",
|
|
[{
|
|
"tool": "manage_notes",
|
|
"command": '{"action":"search","text":"ODY-EVAL-TOOL-NOTES-SEARCH"}',
|
|
}],
|
|
)
|
|
|
|
|
|
def test_fixture_search_contracts_reject_broad_lists() -> None:
|
|
assert not _command_contract_ok(
|
|
"documents_search_fixture",
|
|
[{"tool": "manage_documents", "command": '{"action":"list"}'}],
|
|
)
|
|
assert not _command_contract_ok(
|
|
"documents_search_fixture",
|
|
[{"tool": "manage_documents", "command": '{"action":"find","query":"ODY-EVAL-TOOL-DOCUMENT-SEARCH"}'}],
|
|
)
|
|
assert _command_contract_ok(
|
|
"documents_search_fixture",
|
|
[
|
|
{"tool": "manage_documents", "command": '{"action":"find","query":"ODY-EVAL-TOOL-DOCUMENT-SEARCH"}'},
|
|
{"tool": "manage_documents", "command": '{"action":"read","document_id":"doc-fixture"}'},
|
|
],
|
|
)
|
|
assert not _command_contract_ok(
|
|
"tasks_search_fixture",
|
|
[{"tool": "manage_tasks", "command": '{"action":"list"}'}],
|
|
)
|
|
assert _command_contract_ok(
|
|
"tasks_search_fixture",
|
|
[{"tool": "manage_tasks", "command": '{"action":"list","name":"ODY-EVAL-TOOL-TASK-SEARCH"}'}],
|
|
)
|
|
assert not _command_contract_ok(
|
|
"calendar_search_fixture",
|
|
[{"tool": "manage_calendar", "command": '{"action":"list_events","query":"ODY-EVAL-TOOL-CALENDAR-SEARCH"}'}],
|
|
)
|
|
assert _command_contract_ok(
|
|
"calendar_search_fixture",
|
|
[{
|
|
"tool": "manage_calendar",
|
|
"command": (
|
|
'{"action":"list_events","query":"ODY-EVAL-TOOL-CALENDAR-SEARCH",'
|
|
'"start":"2026-08-21","end":"2026-08-23"}'
|
|
),
|
|
}],
|
|
)
|
|
|
|
|
|
def test_eval_local_runtime_context_sends_tui_workspace_form_fields(monkeypatch) -> None:
|
|
captured = {}
|
|
|
|
class Response:
|
|
status_code = 200
|
|
|
|
def json(self):
|
|
return {"id": "session-1"}
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def iter_lines(self):
|
|
return iter(("data: [DONE]", ""))
|
|
|
|
class Stream:
|
|
def __enter__(self):
|
|
return Response()
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
class Client:
|
|
def post(self, url, data=None, **_kwargs):
|
|
if url.endswith("/api/session"):
|
|
return Response()
|
|
captured.update(data or {})
|
|
return Response()
|
|
|
|
def stream(self, _method, _url, data=None, **_kwargs):
|
|
captured.update(data or {})
|
|
return Stream()
|
|
|
|
def delete(self, *_args, **_kwargs):
|
|
return Response()
|
|
|
|
args = SimpleNamespace(
|
|
base_url="http://backend",
|
|
endpoint="http://model/v1",
|
|
selected_endpoint_url="http://model/v1/chat/completions",
|
|
endpoint_id="router-1",
|
|
selected_model="router",
|
|
model="router",
|
|
prompt_mode="auto",
|
|
client_runtime_context={
|
|
"surface": "odysseus-tui",
|
|
"session_cwd": "/workspace/project",
|
|
},
|
|
timeout=5,
|
|
auto_approve=False,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"scripts.eval_odysseus_tool_use._cookie",
|
|
lambda _path: "cookie",
|
|
)
|
|
run_case(Client(), args, "local", "inspect this repo", "host_shell")
|
|
|
|
assert captured["cwd"] == "/workspace/project"
|
|
assert captured["workspace"] == "/workspace/project"
|
|
|
|
|
|
def test_eval_local_runtime_context_accepts_cwd_alias(monkeypatch) -> None:
|
|
captured = {}
|
|
|
|
class Response:
|
|
status_code = 200
|
|
|
|
def json(self):
|
|
return {"id": "session-1"}
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def iter_lines(self):
|
|
return iter(("data: [DONE]", ""))
|
|
|
|
class Stream:
|
|
def __enter__(self):
|
|
return Response()
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
class Client:
|
|
def post(self, url, data=None, **_kwargs):
|
|
if url.endswith("/api/session"):
|
|
return Response()
|
|
captured.update(data or {})
|
|
return Response()
|
|
|
|
def stream(self, _method, _url, data=None, **_kwargs):
|
|
captured.update(data or {})
|
|
return Stream()
|
|
|
|
def delete(self, *_args, **_kwargs):
|
|
return Response()
|
|
|
|
args = SimpleNamespace(
|
|
base_url="http://backend",
|
|
endpoint="http://model/v1",
|
|
selected_endpoint_url="http://model/v1/chat/completions",
|
|
endpoint_id="router-1",
|
|
selected_model="router",
|
|
model="router",
|
|
prompt_mode="auto",
|
|
client_runtime_context={
|
|
"surface": "odysseus-tui",
|
|
"cwd": "/workspace/project",
|
|
},
|
|
timeout=5,
|
|
auto_approve=False,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"scripts.eval_odysseus_tool_use._cookie",
|
|
lambda _path: "cookie",
|
|
)
|
|
run_case(Client(), args, "local", "inspect this repo", "host_shell")
|
|
|
|
assert captured["cwd"] == "/workspace/project"
|
|
assert captured["workspace"] == "/workspace/project"
|
|
|
|
|
|
def test_extended_eval_cleanup_only_removes_tagged_fixture_notes() -> None:
|
|
class Response:
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return {
|
|
"notes": [
|
|
{"id": "fixture", "title": "ODY-EVAL-EXT-CREATE"},
|
|
{"id": "user", "title": "Keep this note"},
|
|
{"id": "", "title": "ODY-EVAL-EXT-NO-ID"},
|
|
]
|
|
}
|
|
|
|
class Client:
|
|
def __init__(self):
|
|
self.deleted = []
|
|
|
|
def get(self, url, timeout):
|
|
return Response()
|
|
|
|
def delete(self, url, timeout):
|
|
self.deleted.append(url)
|
|
|
|
client = Client()
|
|
cleanup_notes(client, "http://backend")
|
|
|
|
assert client.deleted == ["http://backend/api/notes/fixture"]
|