fix(steward): route on the declared DELEGATE line, not on prose

The prompt tells the Steward to state its choice on a DELEGATE line and
to explain itself on REASON, COMPLEXITY and CONTEXT lines. Extraction
ignored that structure and substring-matched capability domains across
the entire response, so ordinary English in the explanation selected
agents: "description" contains the housekeeper domain "script",
"discover" contains "cover", "acknowledge" contains "knowledge" and
"know", "economy" contains the biographer domain "my".

Every one of those was a real delegation. A spurious librarian is a
multi-second web call on a query that asked for arithmetic.

It also made prose length a routing input, which would have quietly
corrupted the thinking benchmark this was found during: anything that
shortened the Steward's output reduces accidental substring hits and so
reads as improved routing.

Resolution is now layered, most explicit first — a DELEGATE line opening
with a capability name, then a capability named anywhere on that line,
then a domain on that line. With no DELEGATE line at all the response is
matched on capability names only, never domains, so the conversational
path still answers with no capabilities. Matching is whole-word
throughout.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-08-08 14:37:59 +02:00
co-authored by Claude
parent 19e32cfbd6
commit a90536314e
3 changed files with 168 additions and 15 deletions
+92 -1
View File
@@ -8,7 +8,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
from src.agents.steward.service import analyze_request, format_steward_note, _build_enriched_query
from src.agents.steward.service import (
_build_enriched_query,
_extract_capabilities,
analyze_request,
format_steward_note,
)
from src.core.startup import register_household_members
@@ -283,3 +288,89 @@ class TestBuildEnrichedQuery:
result = _build_enriched_query(query, memory_context)
assert result == query
class TestExtractCapabilities:
"""Capability extraction reads the declared DELEGATE line, not free prose.
The prompt tells the Steward to state its choice on a DELEGATE line and to
explain itself on REASON/COMPLEXITY/CONTEXT lines. An earlier version
substring-matched capability domains across the entire response, so ordinary
English in the explanation routed requests: "description" contains the
housekeeper domain "script", "acknowledge" contains "know". These tests pin
that the explanation can no longer influence routing.
"""
# (prose, why it used to misroute)
SUBSTRING_TRAPS = [
("The user wants a description of the algorithm.", "script -> housekeeper"),
("I should discover what the answer is.", "cover -> housekeeper"),
("That sounds fantastic, let me compute it.", "fan -> housekeeper"),
("I acknowledge the request to add two numbers.", "knowledge/know -> librarian, biographer"),
("The user asks about the economy myth.", "my -> biographer"),
("Convert 98.6 Fahrenheit to Celsius.", "temperature is a housekeeper domain"),
]
@pytest.mark.parametrize("prose,reason", SUBSTRING_TRAPS)
def test_reason_prose_cannot_add_capabilities(self, prose, reason):
"""Explanatory prose must not summon agents the Steward did not request."""
text = f"DELEGATE: tatlock_core to calculate\nREASON: {prose}\nCOMPLEXITY: simple"
assert _extract_capabilities(text) == ["tatlock_core"], f"regression: {reason}"
def test_delegate_line_task_text_does_not_leak(self):
"""A domain word inside the task description must not add a capability.
"home" is a housekeeper domain, but this is plainly a memory recall.
"""
text = "DELEGATE: biographer to recall the user's home address\nREASON: personal data"
assert _extract_capabilities(text) == ["biographer"]
def test_multiple_delegate_lines(self):
"""Each DELEGATE line contributes its capability, in order, deduplicated."""
text = (
"DELEGATE: biographer to recall the user's location\n"
"DELEGATE: librarian to search_web for the forecast\n"
"DELEGATE: biographer to recall preferences\n"
)
assert _extract_capabilities(text) == ["biographer", "librarian"]
def test_capability_named_later_on_the_line(self):
"""A loosely worded DELEGATE line still resolves by name."""
text = "DELEGATE: ask the librarian to search the web"
assert _extract_capabilities(text) == ["librarian"]
def test_domain_fallback_within_delegate_line(self):
"""With no capability named, domains on the DELEGATE line still resolve."""
text = "DELEGATE: turn on the lights in the kitchen"
assert _extract_capabilities(text) == ["housekeeper"]
def test_conversational_response_selects_nothing(self):
"""No DELEGATE line means no capability, which is the prompt's chat path."""
text = "This is a simple greeting. No capabilities are needed. COMPLEXITY: simple"
assert _extract_capabilities(text) == []
def test_malformed_response_still_routes_by_name(self):
"""If the format is ignored, a named capability is still honoured."""
text = "I think the librarian should handle this research request."
assert _extract_capabilities(text) == ["librarian"]
def test_malformed_response_does_not_route_on_domains(self):
"""...but bare prose must not route on domain words alone."""
text = "The user wants a description of home automation, and I acknowledge it."
assert _extract_capabilities(text) == []
def test_case_insensitive_delegate_marker(self):
text = "delegate: Librarian to search_web"
assert _extract_capabilities(text) == ["librarian"]
def test_empty_input(self):
assert _extract_capabilities("") == []