From 99e1fe33ca6ec97c5b3ef036d95a5705d4a02d14 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 10:15:09 +0200 Subject: [PATCH] feat(librarian): signal degraded search coverage - parse source_counts into HybridRAGResponse and additively parse the shared-contract source_status/degraded fields when present (absence tolerated, so deploy order between tatlock and library-desk never matters) - hybrid_search appends a one-line coverage note when a leg reported 'failed' (or degraded is set), falling back to inferring silent legs from source_counts on older library-desk versions Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 + src/agents/librarian/client.py | 11 ++ src/agents/librarian/tools.py | 70 ++++++++- .../agents/librarian/test_hybrid_contract.py | 144 +++++++++++++++++- 4 files changed, 227 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b5dce..f9c834c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Search degradation signaling** - The librarian client parses `source_counts` (plus the additive `source_status`/`degraded` fields when a newer library-desk sends them; absence is tolerated), and `hybrid_search` appends a one-line coverage note when a search is degraded or an enabled source leg contributed nothing, so outages are visible to the model and the user + ### Fixed - **Honest expert failures** - `run_librarian`/`run_librarian_stream` now raise a structured `AgentError` instead of returning error text as if it were research output, so delegation correctly reports `success=False` and the streaming error branch is reachable. Failures surface to the user as curated butler-toned sentences; exception detail (including internal URLs) stays in the logs only. Librarian tool errors no longer leak `str(e)` into synthesis diff --git a/src/agents/librarian/client.py b/src/agents/librarian/client.py index aa6e9ff..809fd09 100644 --- a/src/agents/librarian/client.py +++ b/src/agents/librarian/client.py @@ -75,7 +75,14 @@ class HybridRAGResponse(BaseModel): related_dossiers: list[str] = Field(default_factory=list) formatted_context: str = "" search_id: str | None = None + source_counts: dict[str, int] = Field(default_factory=dict) timing: dict[str, float] = Field(default_factory=dict) + # Additive degradation contract - only newer library-desk versions + # send these; absence means "no status reported", not "healthy". + # Maps each leg (vector/graph/web/volatile/documents) to + # "ok" | "failed" | "disabled". + source_status: dict[str, str] = Field(default_factory=dict) + degraded: bool = False class GraphNode(BaseModel): @@ -327,7 +334,11 @@ class LibraryDeskClient: related_dossiers=related_dossiers, formatted_context=data.get("context", data.get("formatted_context", "")), search_id=data.get("search_id"), + source_counts=data.get("source_counts", {}), timing=data.get("timing", {}), + # Additive fields - tolerate absence on older library-desk + source_status=data.get("source_status") or {}, + degraded=bool(data.get("degraded", False)), ) # ======================================================================== diff --git a/src/agents/librarian/tools.py b/src/agents/librarian/tools.py index 78573a9..b161983 100644 --- a/src/agents/librarian/tools.py +++ b/src/agents/librarian/tools.py @@ -4,7 +4,7 @@ Librarian tools for PydanticAI agent. These tools wrap the library-desk API and are registered with The Librarian agent for research and knowledge management tasks. """ -from src.agents.librarian.client import LibraryDeskClient +from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient from src.core.logging_config import get_logger logger = get_logger(__name__) @@ -22,6 +22,62 @@ SOURCE_ICONS = { } +def _coverage_note( + response: HybridRAGResponse, + include_web: bool, + include_documents: bool, + include_volatile: bool, +) -> str: + """ + Build a one-line coverage note when the search was degraded or an + enabled source leg contributed nothing, so outages stay visible to + the model and the user instead of silently narrowing results. + + Prefers the additive source_status/degraded contract when present; + falls back to inferring silent legs from source_counts. + """ + if response.source_status: + failed = sorted( + leg + for leg, status in response.source_status.items() + if status == "failed" + ) + if failed: + return ( + "⚠️ *Coverage note: results are partial - " + f"these sources failed: {', '.join(failed)}.*" + ) + if response.degraded: + return ( + "⚠️ *Coverage note: results are partial - " + "one or more sources failed during this search.*" + ) + return "" + + if not response.source_counts: + # Older library-desk without per-source reporting - nothing to infer + return "" + + expected = {"vector", "graph"} + if include_web: + expected.add("web") + if include_documents: + expected.add("documents") + if include_volatile: + expected.add("volatile") + + # Normalize count keys to leg names (document/documents, wiki/vector) + aliases = {"document": "documents", "wiki": "vector"} + reported = {aliases.get(key, key) for key in response.source_counts} + missing = sorted(expected - reported) + if missing: + return ( + "⚠️ *Coverage note: no results came from: " + f"{', '.join(missing)} (source unavailable or nothing found).*" + ) + return "" + + # ============================================================================ # HybridRAG Search # ============================================================================ @@ -101,10 +157,22 @@ async def hybrid_search( output_parts.append(f" {result.content[:300]}...") output_parts.append("") + # Surface degraded coverage so outages are visible downstream + coverage_note = _coverage_note( + response, + include_web=include_web, + include_documents=include_documents, + include_volatile=include_volatile, + ) + if coverage_note: + output_parts.append(coverage_note) + logger.info( "librarian_hybrid_search", query=query, result_count=len(response.results), + degraded=response.degraded, + source_counts=response.source_counts, ) return "\n".join(output_parts) diff --git a/tests/agents/librarian/test_hybrid_contract.py b/tests/agents/librarian/test_hybrid_contract.py index e7b6478..14ccb80 100644 --- a/tests/agents/librarian/test_hybrid_contract.py +++ b/tests/agents/librarian/test_hybrid_contract.py @@ -17,7 +17,7 @@ import httpx import pytest from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient -from src.agents.librarian.tools import SOURCE_ICONS, hybrid_search +from src.agents.librarian.tools import SOURCE_ICONS, _coverage_note, hybrid_search FIXTURE_PATH = Path(__file__).parent / "fixtures" / "hybrid_query_recorded.json" @@ -150,6 +150,58 @@ class TestHybridRAGContract: assert config["enable_documents"] is False assert config["enable_volatile"] is False + @pytest.mark.asyncio + async def test_source_counts_and_timing_parsed( + self, client_with_recorded_response + ): + """source_counts and timing map into the response model.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + assert response.source_counts == {"graph": 3, "web": 2} + assert response.timing.get("total_ms", 0) > 0 + + @pytest.mark.asyncio + async def test_source_status_absent_is_tolerated( + self, client_with_recorded_response + ): + """Recorded response predates source_status/degraded - defaults apply.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + assert response.source_status == {} + assert response.degraded is False + + @pytest.mark.asyncio + async def test_source_status_parsed_when_present(self, recorded_response): + """Additive source_status/degraded fields parse when the service sends them.""" + enriched = dict(recorded_response) + enriched["source_status"] = { + "vector": "ok", + "graph": "ok", + "web": "failed", + "volatile": "disabled", + "documents": "ok", + } + enriched["degraded"] = True + + mock_response = MagicMock() + mock_response.json.return_value = enriched + mock_response.raise_for_status = MagicMock() + mock_httpx = AsyncMock(spec=httpx.AsyncClient) + mock_httpx.post.return_value = mock_response + + client = LibraryDeskClient(base_url="http://test:8089", api_key="test-key") + client._client = mock_httpx + + response = await client.hybrid_search("home server infrastructure", user="u") + + assert response.degraded is True + assert response.source_status["web"] == "failed" + assert response.source_status["volatile"] == "disabled" + @pytest.mark.asyncio async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch): """The hybrid_search tool renders real sources and non-zero scores.""" @@ -173,3 +225,93 @@ class TestHybridRAGContract: assert "unknown" not in output assert "score: 0.00" not in output assert "•" not in output, "every source value should map to an icon" + + +@pytest.mark.unit +class TestCoverageNote: + """Coverage note makes degraded searches visible to model and user.""" + + def _response(self, **kwargs) -> HybridRAGResponse: + return HybridRAGResponse(**kwargs) + + def test_no_note_when_all_legs_report(self): + response = self._response( + source_counts={ + "vector": 2, + "graph": 1, + "web": 2, + "documents": 1, + "volatile": 1, + }, + ) + note = _coverage_note( + response, include_web=True, include_documents=True, include_volatile=True + ) + assert note == "" + + def test_note_when_leg_missing_from_counts(self): + response = self._response(source_counts={"graph": 3, "web": 2}) + note = _coverage_note( + response, include_web=True, include_documents=True, include_volatile=True + ) + assert "Coverage note" in note + assert "vector" in note + assert "documents" in note + assert "volatile" in note + + def test_disabled_legs_are_not_reported_missing(self): + response = self._response(source_counts={"vector": 2, "graph": 1}) + note = _coverage_note( + response, + include_web=False, + include_documents=False, + include_volatile=False, + ) + assert note == "" + + def test_note_prefers_source_status_failures(self): + response = self._response( + source_counts={"vector": 2, "graph": 1}, + source_status={ + "vector": "ok", + "graph": "ok", + "web": "failed", + "volatile": "disabled", + "documents": "ok", + }, + degraded=True, + ) + note = _coverage_note( + response, include_web=True, include_documents=True, include_volatile=True + ) + assert "failed" in note + assert "web" in note + # disabled legs are not reported as failures + assert "volatile" not in note + + def test_no_note_when_status_all_ok(self): + response = self._response( + source_counts={"graph": 1}, + source_status={ + "vector": "ok", + "graph": "ok", + "web": "ok", + "volatile": "ok", + "documents": "ok", + }, + degraded=False, + ) + note = _coverage_note( + response, include_web=True, include_documents=True, include_volatile=True + ) + assert note == "" + + def test_degraded_without_named_failures(self): + response = self._response( + source_status={"vector": "ok", "graph": "ok"}, + degraded=True, + ) + note = _coverage_note( + response, include_web=True, include_documents=True, include_volatile=True + ) + assert "partial" in note