fix(steward): route on the declared DELEGATE line, not on prose
The prompt tells the Steward to state its choice on a DELEGATE line and to explain itself on REASON, COMPLEXITY and CONTEXT lines. Extraction ignored that structure and substring-matched capability domains across the entire response, so ordinary English in the explanation selected agents: "description" contains the housekeeper domain "script", "discover" contains "cover", "acknowledge" contains "knowledge" and "know", "economy" contains the biographer domain "my". Every one of those was a real delegation. A spurious librarian is a multi-second web call on a query that asked for arithmetic. It also made prose length a routing input, which would have quietly corrupted the thinking benchmark this was found during: anything that shortened the Steward's output reduces accidental substring hits and so reads as improved routing. Resolution is now layered, most explicit first — a DELEGATE line opening with a capability name, then a capability named anywhere on that line, then a domain on that line. With no DELEGATE line at all the response is matched on capability names only, never domains, so the conversational path still answers with no capabilities. Matching is whole-word throughout. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -8,7 +8,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
from src.agents.steward.service import analyze_request, format_steward_note, _build_enriched_query
|
||||
from src.agents.steward.service import (
|
||||
_build_enriched_query,
|
||||
_extract_capabilities,
|
||||
analyze_request,
|
||||
format_steward_note,
|
||||
)
|
||||
from src.core.startup import register_household_members
|
||||
|
||||
|
||||
@@ -283,3 +288,89 @@ class TestBuildEnrichedQuery:
|
||||
result = _build_enriched_query(query, memory_context)
|
||||
|
||||
assert result == query
|
||||
|
||||
|
||||
class TestExtractCapabilities:
|
||||
"""Capability extraction reads the declared DELEGATE line, not free prose.
|
||||
|
||||
The prompt tells the Steward to state its choice on a DELEGATE line and to
|
||||
explain itself on REASON/COMPLEXITY/CONTEXT lines. An earlier version
|
||||
substring-matched capability domains across the entire response, so ordinary
|
||||
English in the explanation routed requests: "description" contains the
|
||||
housekeeper domain "script", "acknowledge" contains "know". These tests pin
|
||||
that the explanation can no longer influence routing.
|
||||
"""
|
||||
|
||||
# (prose, why it used to misroute)
|
||||
SUBSTRING_TRAPS = [
|
||||
("The user wants a description of the algorithm.", "script -> housekeeper"),
|
||||
("I should discover what the answer is.", "cover -> housekeeper"),
|
||||
("That sounds fantastic, let me compute it.", "fan -> housekeeper"),
|
||||
("I acknowledge the request to add two numbers.", "knowledge/know -> librarian, biographer"),
|
||||
("The user asks about the economy myth.", "my -> biographer"),
|
||||
("Convert 98.6 Fahrenheit to Celsius.", "temperature is a housekeeper domain"),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("prose,reason", SUBSTRING_TRAPS)
|
||||
def test_reason_prose_cannot_add_capabilities(self, prose, reason):
|
||||
"""Explanatory prose must not summon agents the Steward did not request."""
|
||||
text = f"DELEGATE: tatlock_core to calculate\nREASON: {prose}\nCOMPLEXITY: simple"
|
||||
|
||||
assert _extract_capabilities(text) == ["tatlock_core"], f"regression: {reason}"
|
||||
|
||||
def test_delegate_line_task_text_does_not_leak(self):
|
||||
"""A domain word inside the task description must not add a capability.
|
||||
|
||||
"home" is a housekeeper domain, but this is plainly a memory recall.
|
||||
"""
|
||||
text = "DELEGATE: biographer to recall the user's home address\nREASON: personal data"
|
||||
|
||||
assert _extract_capabilities(text) == ["biographer"]
|
||||
|
||||
def test_multiple_delegate_lines(self):
|
||||
"""Each DELEGATE line contributes its capability, in order, deduplicated."""
|
||||
text = (
|
||||
"DELEGATE: biographer to recall the user's location\n"
|
||||
"DELEGATE: librarian to search_web for the forecast\n"
|
||||
"DELEGATE: biographer to recall preferences\n"
|
||||
)
|
||||
|
||||
assert _extract_capabilities(text) == ["biographer", "librarian"]
|
||||
|
||||
def test_capability_named_later_on_the_line(self):
|
||||
"""A loosely worded DELEGATE line still resolves by name."""
|
||||
text = "DELEGATE: ask the librarian to search the web"
|
||||
|
||||
assert _extract_capabilities(text) == ["librarian"]
|
||||
|
||||
def test_domain_fallback_within_delegate_line(self):
|
||||
"""With no capability named, domains on the DELEGATE line still resolve."""
|
||||
text = "DELEGATE: turn on the lights in the kitchen"
|
||||
|
||||
assert _extract_capabilities(text) == ["housekeeper"]
|
||||
|
||||
def test_conversational_response_selects_nothing(self):
|
||||
"""No DELEGATE line means no capability, which is the prompt's chat path."""
|
||||
text = "This is a simple greeting. No capabilities are needed. COMPLEXITY: simple"
|
||||
|
||||
assert _extract_capabilities(text) == []
|
||||
|
||||
def test_malformed_response_still_routes_by_name(self):
|
||||
"""If the format is ignored, a named capability is still honoured."""
|
||||
text = "I think the librarian should handle this research request."
|
||||
|
||||
assert _extract_capabilities(text) == ["librarian"]
|
||||
|
||||
def test_malformed_response_does_not_route_on_domains(self):
|
||||
"""...but bare prose must not route on domain words alone."""
|
||||
text = "The user wants a description of home automation, and I acknowledge it."
|
||||
|
||||
assert _extract_capabilities(text) == []
|
||||
|
||||
def test_case_insensitive_delegate_marker(self):
|
||||
text = "delegate: Librarian to search_web"
|
||||
|
||||
assert _extract_capabilities(text) == ["librarian"]
|
||||
|
||||
def test_empty_input(self):
|
||||
assert _extract_capabilities("") == []
|
||||
|
||||
Reference in New Issue
Block a user