feat(librarian): signal degraded search coverage

- parse source_counts into HybridRAGResponse and additively parse the
  shared-contract source_status/degraded fields when present (absence
  tolerated, so deploy order between tatlock and library-desk never
  matters)
- hybrid_search appends a one-line coverage note when a leg reported
  'failed' (or degraded is set), falling back to inferring silent legs
  from source_counts on older library-desk versions

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-14 10:15:09 +02:00
co-authored by Claude Fable 5
parent f853db8ccc
commit 99e1fe33ca
4 changed files with 227 additions and 2 deletions
+4
View File
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- **Search degradation signaling** - The librarian client parses `source_counts` (plus the additive `source_status`/`degraded` fields when a newer library-desk sends them; absence is tolerated), and `hybrid_search` appends a one-line coverage note when a search is degraded or an enabled source leg contributed nothing, so outages are visible to the model and the user
### Fixed
- **Honest expert failures** - `run_librarian`/`run_librarian_stream` now raise a structured `AgentError` instead of returning error text as if it were research output, so delegation correctly reports `success=False` and the streaming error branch is reachable. Failures surface to the user as curated butler-toned sentences; exception detail (including internal URLs) stays in the logs only. Librarian tool errors no longer leak `str(e)` into synthesis
+11
View File
@@ -75,7 +75,14 @@ class HybridRAGResponse(BaseModel):
related_dossiers: list[str] = Field(default_factory=list)
formatted_context: str = ""
search_id: str | None = None
source_counts: dict[str, int] = Field(default_factory=dict)
timing: dict[str, float] = Field(default_factory=dict)
# Additive degradation contract - only newer library-desk versions
# send these; absence means "no status reported", not "healthy".
# Maps each leg (vector/graph/web/volatile/documents) to
# "ok" | "failed" | "disabled".
source_status: dict[str, str] = Field(default_factory=dict)
degraded: bool = False
class GraphNode(BaseModel):
@@ -327,7 +334,11 @@ class LibraryDeskClient:
related_dossiers=related_dossiers,
formatted_context=data.get("context", data.get("formatted_context", "")),
search_id=data.get("search_id"),
source_counts=data.get("source_counts", {}),
timing=data.get("timing", {}),
# Additive fields - tolerate absence on older library-desk
source_status=data.get("source_status") or {},
degraded=bool(data.get("degraded", False)),
)
# ========================================================================
+69 -1
View File
@@ -4,7 +4,7 @@ Librarian tools for PydanticAI agent.
These tools wrap the library-desk API and are registered with
The Librarian agent for research and knowledge management tasks.
"""
from src.agents.librarian.client import LibraryDeskClient
from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient
from src.core.logging_config import get_logger
logger = get_logger(__name__)
@@ -22,6 +22,62 @@ SOURCE_ICONS = {
}
def _coverage_note(
response: HybridRAGResponse,
include_web: bool,
include_documents: bool,
include_volatile: bool,
) -> str:
"""
Build a one-line coverage note when the search was degraded or an
enabled source leg contributed nothing, so outages stay visible to
the model and the user instead of silently narrowing results.
Prefers the additive source_status/degraded contract when present;
falls back to inferring silent legs from source_counts.
"""
if response.source_status:
failed = sorted(
leg
for leg, status in response.source_status.items()
if status == "failed"
)
if failed:
return (
"⚠️ *Coverage note: results are partial - "
f"these sources failed: {', '.join(failed)}.*"
)
if response.degraded:
return (
"⚠️ *Coverage note: results are partial - "
"one or more sources failed during this search.*"
)
return ""
if not response.source_counts:
# Older library-desk without per-source reporting - nothing to infer
return ""
expected = {"vector", "graph"}
if include_web:
expected.add("web")
if include_documents:
expected.add("documents")
if include_volatile:
expected.add("volatile")
# Normalize count keys to leg names (document/documents, wiki/vector)
aliases = {"document": "documents", "wiki": "vector"}
reported = {aliases.get(key, key) for key in response.source_counts}
missing = sorted(expected - reported)
if missing:
return (
"⚠️ *Coverage note: no results came from: "
f"{', '.join(missing)} (source unavailable or nothing found).*"
)
return ""
# ============================================================================
# HybridRAG Search
# ============================================================================
@@ -101,10 +157,22 @@ async def hybrid_search(
output_parts.append(f" {result.content[:300]}...")
output_parts.append("")
# Surface degraded coverage so outages are visible downstream
coverage_note = _coverage_note(
response,
include_web=include_web,
include_documents=include_documents,
include_volatile=include_volatile,
)
if coverage_note:
output_parts.append(coverage_note)
logger.info(
"librarian_hybrid_search",
query=query,
result_count=len(response.results),
degraded=response.degraded,
source_counts=response.source_counts,
)
return "\n".join(output_parts)
+143 -1
View File
@@ -17,7 +17,7 @@ import httpx
import pytest
from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient
from src.agents.librarian.tools import SOURCE_ICONS, hybrid_search
from src.agents.librarian.tools import SOURCE_ICONS, _coverage_note, hybrid_search
FIXTURE_PATH = Path(__file__).parent / "fixtures" / "hybrid_query_recorded.json"
@@ -150,6 +150,58 @@ class TestHybridRAGContract:
assert config["enable_documents"] is False
assert config["enable_volatile"] is False
@pytest.mark.asyncio
async def test_source_counts_and_timing_parsed(
self, client_with_recorded_response
):
"""source_counts and timing map into the response model."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
assert response.source_counts == {"graph": 3, "web": 2}
assert response.timing.get("total_ms", 0) > 0
@pytest.mark.asyncio
async def test_source_status_absent_is_tolerated(
self, client_with_recorded_response
):
"""Recorded response predates source_status/degraded - defaults apply."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
assert response.source_status == {}
assert response.degraded is False
@pytest.mark.asyncio
async def test_source_status_parsed_when_present(self, recorded_response):
"""Additive source_status/degraded fields parse when the service sends them."""
enriched = dict(recorded_response)
enriched["source_status"] = {
"vector": "ok",
"graph": "ok",
"web": "failed",
"volatile": "disabled",
"documents": "ok",
}
enriched["degraded"] = True
mock_response = MagicMock()
mock_response.json.return_value = enriched
mock_response.raise_for_status = MagicMock()
mock_httpx = AsyncMock(spec=httpx.AsyncClient)
mock_httpx.post.return_value = mock_response
client = LibraryDeskClient(base_url="http://test:8089", api_key="test-key")
client._client = mock_httpx
response = await client.hybrid_search("home server infrastructure", user="u")
assert response.degraded is True
assert response.source_status["web"] == "failed"
assert response.source_status["volatile"] == "disabled"
@pytest.mark.asyncio
async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch):
"""The hybrid_search tool renders real sources and non-zero scores."""
@@ -173,3 +225,93 @@ class TestHybridRAGContract:
assert "unknown" not in output
assert "score: 0.00" not in output
assert "" not in output, "every source value should map to an icon"
@pytest.mark.unit
class TestCoverageNote:
"""Coverage note makes degraded searches visible to model and user."""
def _response(self, **kwargs) -> HybridRAGResponse:
return HybridRAGResponse(**kwargs)
def test_no_note_when_all_legs_report(self):
response = self._response(
source_counts={
"vector": 2,
"graph": 1,
"web": 2,
"documents": 1,
"volatile": 1,
},
)
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
assert note == ""
def test_note_when_leg_missing_from_counts(self):
response = self._response(source_counts={"graph": 3, "web": 2})
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
assert "Coverage note" in note
assert "vector" in note
assert "documents" in note
assert "volatile" in note
def test_disabled_legs_are_not_reported_missing(self):
response = self._response(source_counts={"vector": 2, "graph": 1})
note = _coverage_note(
response,
include_web=False,
include_documents=False,
include_volatile=False,
)
assert note == ""
def test_note_prefers_source_status_failures(self):
response = self._response(
source_counts={"vector": 2, "graph": 1},
source_status={
"vector": "ok",
"graph": "ok",
"web": "failed",
"volatile": "disabled",
"documents": "ok",
},
degraded=True,
)
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
assert "failed" in note
assert "web" in note
# disabled legs are not reported as failures
assert "volatile" not in note
def test_no_note_when_status_all_ok(self):
response = self._response(
source_counts={"graph": 1},
source_status={
"vector": "ok",
"graph": "ok",
"web": "ok",
"volatile": "ok",
"documents": "ok",
},
degraded=False,
)
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
assert note == ""
def test_degraded_without_named_failures(self):
response = self._response(
source_status={"vector": "ok", "graph": "ok"},
degraded=True,
)
note = _coverage_note(
response, include_web=True, include_documents=True, include_volatile=True
)
assert "partial" in note