fix compound benchmark tool routing

This commit is contained in:
pewdiepie-archdaemon
2026-09-17 23:56:36 +00:00
parent 9c31de087a
commit 687b8aa0e6
3 changed files with 127 additions and 12 deletions
+40
View File
@@ -523,6 +523,28 @@ def test_local_model_discovery_selects_hugging_face_search_and_persistence_switc
) == {"notes"}
def test_unrelated_model_terms_do_not_combine_into_hugging_face_discovery():
prompt = (
"You are in a restricted environment. Use the available tools. Prepare my daily "
"arXiv paper digest. Classify papers under Multimodal / Vision-Language Models. Based "
"on my research interests, highlight papers I might find interesting. If any "
"paper benchmarks against CapRL, extract the comparison results. Save the digest "
"to /tmp_workspace/results/digest.md.\n"
"| CapRL-3B | result |"
)
assert selected_tools_for_request(prompt) != {"search_hf_models"}
assert "search_browser" in requested_capabilities(prompt)
def test_execution_boilerplate_does_not_route_to_background_tasks():
prompt = (
"Solve the task efficiently before the timeout (600s). Use the available tools. "
"Unpack /tmp_workspace/images.tar and classify the images into output folders."
)
capabilities = requested_capabilities(prompt)
assert capabilities == {"shell_files"}
def test_referential_web_source_relationship_keeps_web_tools_warm():
history = [{"role": "assistant", "content": "Official link.", "metadata": {
"tool_events": [{"tool": "web_search", "exit_code": 0}],
@@ -1967,6 +1989,24 @@ def test_explicit_repo_download_selects_tracked_cookbook_download():
assert requested_capabilities(message) == {"cookbook_admin"}
def test_arxiv_source_download_is_not_a_model_download():
message = (
"Download the source package from https://arxiv.org/abs/2501.07888, "
"extract every table, and save each one to /tmp_workspace/results/1.tex."
)
assert selected_tools_for_request(message) != {"download_model"}
assert requested_capabilities(message) == {"search_browser", "shell_files"}
def test_single_web_page_with_workspace_outputs_is_not_sealed_to_fetch_only():
message = (
"Visit https://example.org/catalog and save every item under "
"/tmp_workspace/results/items/ plus a summary.jsonl file."
)
assert selected_tools_for_request(message) is None
assert requested_capabilities(message) == {"search_browser", "shell_files"}
def test_even_have_email_accounts_selects_account_inventory():
message = "do i even have any email accounts connected here?"
assert selected_tools_for_request(message) == {"list_email_accounts"}
@@ -278,6 +278,15 @@ def test_quick_web_lookup_with_official_link_selects_search():
assert requested_capabilities(message) == {"search_browser"}
def test_available_tools_execution_boilerplate_is_not_a_tool_inventory_request():
message = (
"Solve the task efficiently before the timeout. Use the available tools and "
"as many iterative steps as needed. Fetch today's papers, classify every paper "
"into exactly one category, and report which paper is most relevant."
)
assert required_read_operation_for_request(message) is None
@pytest.mark.parametrize(("message", "expected"), [
(
"whats on my agenda today? just the titles, dont change anything",