""" Labelled queries for the Steward routing benchmark. Each fixture carries both `expect` and `forbid`: expect capabilities that must appear. Missing one is under-routing — the Butler answers without a tool it needed. forbid capabilities that must not appear. Over-routing is not cosmetic: a spurious librarian is a real multi-second web call, and a spurious housekeeper can actuate hardware. `forbid` matters more than `expect` here, because over-recommendation is the predicted failure when model thinking is disabled and the Steward has less room to discriminate. The `adversarial` group deserves explanation. Until 2026-08-08 the extractor substring-matched capability *domains* across the Steward's whole response, so ordinary English in its REASON line selected agents: "description" contains the housekeeper domain "script", "acknowledge" contains "knowledge" and "know", "economy" contains the biographer domain "my". Those queries invite exactly that vocabulary. They now serve as an end-to-end regression: routing must depend on what the Steward *decided*, not on the words it happened to use while explaining. Expectations follow the routing rules stated in the Steward prompt itself (src/agents/steward/agent.py), not on what a capability could plausibly cover. """ CORE = "tatlock_core" LIB = "librarian" BIO = "biographer" HOUSE = "housekeeper" ALL = [CORE, LIB, BIO, HOUSE] def _others(*keep: str) -> list[str]: return [c for c in ALL if c not in keep] FIXTURES: list[dict] = [ # --- arithmetic and computation -> tatlock_core -------------------------- {"id": "math_add", "group": "math", "query": "What is 61 plus 12?", "expect": [CORE], "forbid": _others(CORE)}, {"id": "math_percent", "group": "math", "query": "What is 15% of 240?", "expect": [CORE], "forbid": _others(CORE)}, {"id": "math_compound", "group": "math", "query": "If I save 200 a month for 3 years, how much is that?", "expect": [CORE], "forbid": _others(CORE)}, {"id": "math_sqrt", "group": "math", "query": "What is the square root of 1764?", "expect": [CORE], "forbid": _others(CORE)}, # --- date and time -> tatlock_core --------------------------------------- {"id": "time_now", "group": "datetime", "query": "What time is it?", "expect": [CORE], "forbid": _others(CORE)}, {"id": "time_date", "group": "datetime", "query": "What is today's date?", "expect": [CORE], "forbid": _others(CORE)}, {"id": "time_delta", "group": "datetime", "query": "How many days until Christmas?", "expect": [CORE], "forbid": _others(CORE)}, # --- personal memory -> biographer --------------------------------------- {"id": "bio_location", "group": "biographer", "query": "Where do I live?", "expect": [BIO], "forbid": [LIB, HOUSE]}, {"id": "bio_name", "group": "biographer", "query": "What's my name?", "expect": [BIO], "forbid": [LIB, HOUSE]}, {"id": "bio_car", "group": "biographer", "query": "What car do I drive?", "expect": [BIO], "forbid": [LIB, HOUSE]}, {"id": "bio_store", "group": "biographer", "query": "Remember that I prefer my coffee black.", "expect": [BIO], "forbid": [LIB, HOUSE]}, {"id": "bio_list", "group": "biographer", "query": "What do you know about me?", "expect": [BIO], "forbid": [LIB, HOUSE]}, {"id": "bio_forget", "group": "biographer", "query": "Forget my old address.", "expect": [BIO], "forbid": [LIB, HOUSE]}, # --- research and current information -> librarian ------------------------ {"id": "lib_weather", "group": "librarian", "query": "What's the weather in Rotterdam tomorrow?", "expect": [LIB], "forbid": [HOUSE]}, {"id": "lib_news", "group": "librarian", "query": "What's in the news today?", "expect": [LIB], "forbid": [HOUSE, BIO]}, {"id": "lib_url", "group": "librarian", "query": "Read https://example.com/article and summarise it.", "expect": [LIB], "forbid": [HOUSE, BIO]}, {"id": "lib_research", "group": "librarian", "query": "Research how tidal power stations work.", "expect": [LIB], "forbid": [HOUSE, BIO]}, {"id": "lib_wiki_create", "group": "librarian", "query": "Create a wiki page about our network topology.", "expect": [LIB], "forbid": [HOUSE, BIO]}, # --- home automation -> housekeeper -------------------------------------- {"id": "house_lights_on", "group": "housekeeper", "query": "Turn on the kitchen lights.", "expect": [HOUSE], "forbid": [LIB, BIO, CORE]}, {"id": "house_lights_off", "group": "housekeeper", "query": "Switch off all the lights downstairs.", "expect": [HOUSE], "forbid": [LIB, BIO, CORE]}, {"id": "house_thermostat", "group": "housekeeper", "query": "Set the thermostat to 20 degrees.", "expect": [HOUSE], "forbid": [LIB, BIO]}, {"id": "house_blinds", "group": "housekeeper", "query": "Close the blinds in the living room.", "expect": [HOUSE], "forbid": [LIB, BIO, CORE]}, # --- conversational -> nothing at all ------------------------------------- # The expensive failure mode: a greeting that triggers a web search. {"id": "chat_greeting", "group": "conversational", "query": "Hello!", "expect": [], "forbid": ALL}, {"id": "chat_thanks", "group": "conversational", "query": "Thanks, that's helpful.", "expect": [], "forbid": ALL}, {"id": "chat_joke", "group": "conversational", "query": "Tell me a joke.", "expect": [], "forbid": ALL}, {"id": "chat_howareyou", "group": "conversational", "query": "How are you doing today?", "expect": [], "forbid": ALL}, {"id": "chat_prior_turn", "group": "conversational", "query": "What did I just say?", "expect": [], "forbid": ALL}, # --- genuinely multi-capability ------------------------------------------- {"id": "multi_weather_home", "group": "multi", "query": "What's the weather here, and remember that I like it warm?", "expect": [LIB, BIO], "forbid": []}, {"id": "multi_recall_search", "group": "multi", "query": "Look up the best route from my home address to Utrecht.", "expect": [BIO, LIB], "forbid": []}, {"id": "multi_math_memory", "group": "multi", "query": "Remember that my budget is 500 euro, then work out 12% of it.", "expect": [BIO, CORE], "forbid": [LIB, HOUSE]}, # --- adversarial: vocabulary that used to select agents by substring ------ # "temperature" is a housekeeper domain, but this is a unit conversion. {"id": "adv_temperature", "group": "adversarial", "query": "Convert 98.6 Fahrenheit to Celsius.", "expect": [CORE], "forbid": [HOUSE, LIB, BIO]}, # "description" contains "script"; "discover" contains "cover". {"id": "adv_description", "group": "adversarial", "query": "Give me a short description of what 17 times 23 comes to.", "expect": [CORE], "forbid": [HOUSE, LIB]}, # "acknowledge" contains "knowledge" and "know". {"id": "adv_acknowledge", "group": "adversarial", "query": "Just acknowledge this and add 5 and 6 for me.", "expect": [CORE], "forbid": [LIB, BIO]}, # "my" appears inside "economy". {"id": "adv_economy", "group": "adversarial", "query": "How many zeros are in one trillion?", "expect": [CORE], "forbid": [BIO, HOUSE]}, # "fan" inside "fantastic"; also a climate word without a home-control intent. {"id": "adv_fantastic", "group": "adversarial", "query": "That's fantastic. What is 8 squared?", "expect": [CORE], "forbid": [HOUSE, LIB]}, # "home" without any actuation intent. {"id": "adv_home_word", "group": "adversarial", "query": "What time do I usually get home?", "expect": [BIO], "forbid": [HOUSE]}, # "search" as ordinary English, not a web-search request. {"id": "adv_search_word", "group": "adversarial", "query": "No need to search anything, just tell me what 9 times 9 is.", "expect": [CORE], "forbid": [LIB]}, # "create"/"write" are librarian domains but this is conversational. {"id": "adv_write_word", "group": "adversarial", "query": "Can you write that more simply?", "expect": [], "forbid": [LIB, HOUSE]}, # --- mutating intents: routing only, nothing is ever executed ------------- {"id": "mutate_wiki_update", "group": "mutating", "query": "Update the dossier page with today's findings.", "expect": [LIB], "forbid": [HOUSE, CORE]}, {"id": "mutate_scene", "group": "mutating", "query": "Run the movie night scene.", "expect": [HOUSE], "forbid": [LIB, BIO, CORE]}, ] GROUPS = sorted({f["group"] for f in FIXTURES}) assert len({f["id"] for f in FIXTURES}) == len(FIXTURES), "duplicate fixture id"