feat(librarian): signal degraded search coverage
- parse source_counts into HybridRAGResponse and additively parse the shared-contract source_status/degraded fields when present (absence tolerated, so deploy order between tatlock and library-desk never matters) - hybrid_search appends a one-line coverage note when a leg reported 'failed' (or degraded is set), falling back to inferring silent legs from source_counts on older library-desk versions Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **Search degradation signaling** - The librarian client parses `source_counts` (plus the additive `source_status`/`degraded` fields when a newer library-desk sends them; absence is tolerated), and `hybrid_search` appends a one-line coverage note when a search is degraded or an enabled source leg contributed nothing, so outages are visible to the model and the user
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Honest expert failures** - `run_librarian`/`run_librarian_stream` now raise a structured `AgentError` instead of returning error text as if it were research output, so delegation correctly reports `success=False` and the streaming error branch is reachable. Failures surface to the user as curated butler-toned sentences; exception detail (including internal URLs) stays in the logs only. Librarian tool errors no longer leak `str(e)` into synthesis
|
||||
|
||||
@@ -75,7 +75,14 @@ class HybridRAGResponse(BaseModel):
|
||||
related_dossiers: list[str] = Field(default_factory=list)
|
||||
formatted_context: str = ""
|
||||
search_id: str | None = None
|
||||
source_counts: dict[str, int] = Field(default_factory=dict)
|
||||
timing: dict[str, float] = Field(default_factory=dict)
|
||||
# Additive degradation contract - only newer library-desk versions
|
||||
# send these; absence means "no status reported", not "healthy".
|
||||
# Maps each leg (vector/graph/web/volatile/documents) to
|
||||
# "ok" | "failed" | "disabled".
|
||||
source_status: dict[str, str] = Field(default_factory=dict)
|
||||
degraded: bool = False
|
||||
|
||||
|
||||
class GraphNode(BaseModel):
|
||||
@@ -327,7 +334,11 @@ class LibraryDeskClient:
|
||||
related_dossiers=related_dossiers,
|
||||
formatted_context=data.get("context", data.get("formatted_context", "")),
|
||||
search_id=data.get("search_id"),
|
||||
source_counts=data.get("source_counts", {}),
|
||||
timing=data.get("timing", {}),
|
||||
# Additive fields - tolerate absence on older library-desk
|
||||
source_status=data.get("source_status") or {},
|
||||
degraded=bool(data.get("degraded", False)),
|
||||
)
|
||||
|
||||
# ========================================================================
|
||||
|
||||
@@ -4,7 +4,7 @@ Librarian tools for PydanticAI agent.
|
||||
These tools wrap the library-desk API and are registered with
|
||||
The Librarian agent for research and knowledge management tasks.
|
||||
"""
|
||||
from src.agents.librarian.client import LibraryDeskClient
|
||||
from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -22,6 +22,62 @@ SOURCE_ICONS = {
|
||||
}
|
||||
|
||||
|
||||
def _coverage_note(
|
||||
response: HybridRAGResponse,
|
||||
include_web: bool,
|
||||
include_documents: bool,
|
||||
include_volatile: bool,
|
||||
) -> str:
|
||||
"""
|
||||
Build a one-line coverage note when the search was degraded or an
|
||||
enabled source leg contributed nothing, so outages stay visible to
|
||||
the model and the user instead of silently narrowing results.
|
||||
|
||||
Prefers the additive source_status/degraded contract when present;
|
||||
falls back to inferring silent legs from source_counts.
|
||||
"""
|
||||
if response.source_status:
|
||||
failed = sorted(
|
||||
leg
|
||||
for leg, status in response.source_status.items()
|
||||
if status == "failed"
|
||||
)
|
||||
if failed:
|
||||
return (
|
||||
"⚠️ *Coverage note: results are partial - "
|
||||
f"these sources failed: {', '.join(failed)}.*"
|
||||
)
|
||||
if response.degraded:
|
||||
return (
|
||||
"⚠️ *Coverage note: results are partial - "
|
||||
"one or more sources failed during this search.*"
|
||||
)
|
||||
return ""
|
||||
|
||||
if not response.source_counts:
|
||||
# Older library-desk without per-source reporting - nothing to infer
|
||||
return ""
|
||||
|
||||
expected = {"vector", "graph"}
|
||||
if include_web:
|
||||
expected.add("web")
|
||||
if include_documents:
|
||||
expected.add("documents")
|
||||
if include_volatile:
|
||||
expected.add("volatile")
|
||||
|
||||
# Normalize count keys to leg names (document/documents, wiki/vector)
|
||||
aliases = {"document": "documents", "wiki": "vector"}
|
||||
reported = {aliases.get(key, key) for key in response.source_counts}
|
||||
missing = sorted(expected - reported)
|
||||
if missing:
|
||||
return (
|
||||
"⚠️ *Coverage note: no results came from: "
|
||||
f"{', '.join(missing)} (source unavailable or nothing found).*"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# HybridRAG Search
|
||||
# ============================================================================
|
||||
@@ -101,10 +157,22 @@ async def hybrid_search(
|
||||
output_parts.append(f" {result.content[:300]}...")
|
||||
output_parts.append("")
|
||||
|
||||
# Surface degraded coverage so outages are visible downstream
|
||||
coverage_note = _coverage_note(
|
||||
response,
|
||||
include_web=include_web,
|
||||
include_documents=include_documents,
|
||||
include_volatile=include_volatile,
|
||||
)
|
||||
if coverage_note:
|
||||
output_parts.append(coverage_note)
|
||||
|
||||
logger.info(
|
||||
"librarian_hybrid_search",
|
||||
query=query,
|
||||
result_count=len(response.results),
|
||||
degraded=response.degraded,
|
||||
source_counts=response.source_counts,
|
||||
)
|
||||
|
||||
return "\n".join(output_parts)
|
||||
|
||||
@@ -17,7 +17,7 @@ import httpx
|
||||
import pytest
|
||||
|
||||
from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient
|
||||
from src.agents.librarian.tools import SOURCE_ICONS, hybrid_search
|
||||
from src.agents.librarian.tools import SOURCE_ICONS, _coverage_note, hybrid_search
|
||||
|
||||
FIXTURE_PATH = Path(__file__).parent / "fixtures" / "hybrid_query_recorded.json"
|
||||
|
||||
@@ -150,6 +150,58 @@ class TestHybridRAGContract:
|
||||
assert config["enable_documents"] is False
|
||||
assert config["enable_volatile"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_counts_and_timing_parsed(
|
||||
self, client_with_recorded_response
|
||||
):
|
||||
"""source_counts and timing map into the response model."""
|
||||
response = await client_with_recorded_response.hybrid_search(
|
||||
"home server infrastructure", user="testuser"
|
||||
)
|
||||
|
||||
assert response.source_counts == {"graph": 3, "web": 2}
|
||||
assert response.timing.get("total_ms", 0) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_status_absent_is_tolerated(
|
||||
self, client_with_recorded_response
|
||||
):
|
||||
"""Recorded response predates source_status/degraded - defaults apply."""
|
||||
response = await client_with_recorded_response.hybrid_search(
|
||||
"home server infrastructure", user="testuser"
|
||||
)
|
||||
|
||||
assert response.source_status == {}
|
||||
assert response.degraded is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_status_parsed_when_present(self, recorded_response):
|
||||
"""Additive source_status/degraded fields parse when the service sends them."""
|
||||
enriched = dict(recorded_response)
|
||||
enriched["source_status"] = {
|
||||
"vector": "ok",
|
||||
"graph": "ok",
|
||||
"web": "failed",
|
||||
"volatile": "disabled",
|
||||
"documents": "ok",
|
||||
}
|
||||
enriched["degraded"] = True
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = enriched
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_httpx = AsyncMock(spec=httpx.AsyncClient)
|
||||
mock_httpx.post.return_value = mock_response
|
||||
|
||||
client = LibraryDeskClient(base_url="http://test:8089", api_key="test-key")
|
||||
client._client = mock_httpx
|
||||
|
||||
response = await client.hybrid_search("home server infrastructure", user="u")
|
||||
|
||||
assert response.degraded is True
|
||||
assert response.source_status["web"] == "failed"
|
||||
assert response.source_status["volatile"] == "disabled"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch):
|
||||
"""The hybrid_search tool renders real sources and non-zero scores."""
|
||||
@@ -173,3 +225,93 @@ class TestHybridRAGContract:
|
||||
assert "unknown" not in output
|
||||
assert "score: 0.00" not in output
|
||||
assert "•" not in output, "every source value should map to an icon"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestCoverageNote:
|
||||
"""Coverage note makes degraded searches visible to model and user."""
|
||||
|
||||
def _response(self, **kwargs) -> HybridRAGResponse:
|
||||
return HybridRAGResponse(**kwargs)
|
||||
|
||||
def test_no_note_when_all_legs_report(self):
|
||||
response = self._response(
|
||||
source_counts={
|
||||
"vector": 2,
|
||||
"graph": 1,
|
||||
"web": 2,
|
||||
"documents": 1,
|
||||
"volatile": 1,
|
||||
},
|
||||
)
|
||||
note = _coverage_note(
|
||||
response, include_web=True, include_documents=True, include_volatile=True
|
||||
)
|
||||
assert note == ""
|
||||
|
||||
def test_note_when_leg_missing_from_counts(self):
|
||||
response = self._response(source_counts={"graph": 3, "web": 2})
|
||||
note = _coverage_note(
|
||||
response, include_web=True, include_documents=True, include_volatile=True
|
||||
)
|
||||
assert "Coverage note" in note
|
||||
assert "vector" in note
|
||||
assert "documents" in note
|
||||
assert "volatile" in note
|
||||
|
||||
def test_disabled_legs_are_not_reported_missing(self):
|
||||
response = self._response(source_counts={"vector": 2, "graph": 1})
|
||||
note = _coverage_note(
|
||||
response,
|
||||
include_web=False,
|
||||
include_documents=False,
|
||||
include_volatile=False,
|
||||
)
|
||||
assert note == ""
|
||||
|
||||
def test_note_prefers_source_status_failures(self):
|
||||
response = self._response(
|
||||
source_counts={"vector": 2, "graph": 1},
|
||||
source_status={
|
||||
"vector": "ok",
|
||||
"graph": "ok",
|
||||
"web": "failed",
|
||||
"volatile": "disabled",
|
||||
"documents": "ok",
|
||||
},
|
||||
degraded=True,
|
||||
)
|
||||
note = _coverage_note(
|
||||
response, include_web=True, include_documents=True, include_volatile=True
|
||||
)
|
||||
assert "failed" in note
|
||||
assert "web" in note
|
||||
# disabled legs are not reported as failures
|
||||
assert "volatile" not in note
|
||||
|
||||
def test_no_note_when_status_all_ok(self):
|
||||
response = self._response(
|
||||
source_counts={"graph": 1},
|
||||
source_status={
|
||||
"vector": "ok",
|
||||
"graph": "ok",
|
||||
"web": "ok",
|
||||
"volatile": "ok",
|
||||
"documents": "ok",
|
||||
},
|
||||
degraded=False,
|
||||
)
|
||||
note = _coverage_note(
|
||||
response, include_web=True, include_documents=True, include_volatile=True
|
||||
)
|
||||
assert note == ""
|
||||
|
||||
def test_degraded_without_named_failures(self):
|
||||
response = self._response(
|
||||
source_status={"vector": "ok", "graph": "ok"},
|
||||
degraded=True,
|
||||
)
|
||||
note = _coverage_note(
|
||||
response, include_web=True, include_documents=True, include_volatile=True
|
||||
)
|
||||
assert "partial" in note
|
||||
|
||||
Reference in New Issue
Block a user