From 59f5b54ac9ab6d6c1df8061e5f790354a0c68a22 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 10:00:10 +0200 Subject: [PATCH] fix(librarian): map live HybridRAG response fields correctly The client parsed field names the live library-desk service never returns, so every result rendered as "unknown (score: 0.00)": - source_type/sources -> source + sources (icons key off sources values) - rrf_score -> score - context -> formatted_context - related_dossiers are per-result; top level aggregates unique titles - synonyms live inside the keywords dict as a {term: [synonyms]} map Also stop sending zero limits (service 422s on limit < 1); disabled legs now rely on the enable_* flags with limits clamped to >= 1. Adds a recorded live response as a fixture plus contract tests that pin the mapping (non-unknown sources, non-zero scores, icon coverage). Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 + src/agents/librarian/client.py | 51 +++- src/agents/librarian/tools.py | 23 +- .../fixtures/hybrid_query_recorded.json | 268 ++++++++++++++++++ .../agents/librarian/test_hybrid_contract.py | 175 ++++++++++++ 5 files changed, 500 insertions(+), 21 deletions(-) create mode 100644 tests/agents/librarian/fixtures/hybrid_query_recorded.json create mode 100644 tests/agents/librarian/test_hybrid_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 499e0c7..2ff7ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **HybridRAG response mapping** - The librarian client now parses the field names library-desk actually returns (`source_type`/`sources`, `rrf_score`, `context`, per-item `related_dossiers`, synonyms nested in the `keywords` dict); previously every result rendered as "unknown (score: 0.00)". Source icons now key off the per-item `sources` list. Requests no longer send zero limits (the service rejects them with 422); legs are disabled via `enable_*` flags. Pinned by a contract test against a recorded live response (`tests/agents/librarian/fixtures/`) + ## [2.3.0] - 2026-07-13 ### Changed diff --git a/src/agents/librarian/client.py b/src/agents/librarian/client.py index c8a8f39..aa6e9ff 100644 --- a/src/agents/librarian/client.py +++ b/src/agents/librarian/client.py @@ -56,12 +56,14 @@ class VectorSearchResult(BaseModel): class HybridSearchResult(BaseModel): """Result from HybridRAG search.""" - source: str # "vector", "graph", "web" + source: str # source_type: "wiki", "web", "volatile", "document" + sources: list[str] = Field(default_factory=list) # legs that found it: "vector", "graph", "web", ... title: str content: str url: str | None = None - score: float + score: float # rrf_score from the live service page_id: int | None = None + related_dossiers: list[dict[str, Any]] = Field(default_factory=list) metadata: dict[str, Any] = Field(default_factory=dict) @@ -250,14 +252,16 @@ class LibraryDeskClient: user = user or get_user() client = self._ensure_client() + # The live service requires all limits >= 1 (422 otherwise); + # legs are disabled via the enable_* flags, not a zero limit. payload = { "query": query, "config": { - "vector_limit": vector_limit, - "graph_limit": graph_limit, - "web_limit": web_limit, - "document_limit": document_limit, - "volatile_limit": volatile_limit, + "vector_limit": max(vector_limit, 1), + "graph_limit": max(graph_limit, 1), + "web_limit": max(web_limit, 1), + "document_limit": max(document_limit, 1), + "volatile_limit": max(volatile_limit, 1), "enable_documents": document_limit > 0, "enable_volatile": volatile_limit > 0, "enable_web": web_limit > 0, @@ -277,32 +281,51 @@ class LibraryDeskClient: data = response.json() - # Parse results + # Parse results (live field names: source_type, sources, rrf_score, + # related_dossiers; older names kept as fallbacks) results = [] for r in data.get("results", []): results.append(HybridSearchResult( - source=r.get("source", "unknown"), + source=r.get("source_type") or r.get("source", "unknown"), + sources=r.get("sources", []), title=r.get("title", ""), content=r.get("content", ""), url=r.get("url"), - score=r.get("score", 0.0), + score=r.get("rrf_score", r.get("score", 0.0)), page_id=r.get("page_id"), + related_dossiers=r.get("related_dossiers", []), metadata=r.get("metadata", {}), )) - # Handle keywords being either a list or a dict with core_keywords + # Handle keywords being either a list or a dict with core_keywords; + # the live service nests synonyms inside the keywords dict as a + # {term: [synonyms]} map. raw_keywords = data.get("keywords", []) + raw_synonyms: Any = data.get("synonyms", []) if isinstance(raw_keywords, dict): keywords = raw_keywords.get("core_keywords", []) + raw_synonyms = raw_keywords.get("synonyms", {}) else: keywords = raw_keywords + if isinstance(raw_synonyms, dict): + synonyms = [s for values in raw_synonyms.values() for s in values] + else: + synonyms = raw_synonyms + + # Aggregate per-result related dossiers into unique top-level titles + related_dossiers: list[str] = [] + for result in results: + for dossier in result.related_dossiers: + title = dossier.get("title", "") + if title and title not in related_dossiers: + related_dossiers.append(title) return HybridRAGResponse( results=results, keywords=keywords, - synonyms=data.get("synonyms", []), - related_dossiers=data.get("related_dossiers", []), - formatted_context=data.get("formatted_context", ""), + synonyms=synonyms, + related_dossiers=related_dossiers, + formatted_context=data.get("context", data.get("formatted_context", "")), search_id=data.get("search_id"), timing=data.get("timing", {}), ) diff --git a/src/agents/librarian/tools.py b/src/agents/librarian/tools.py index 9bc84ba..a51dba7 100644 --- a/src/agents/librarian/tools.py +++ b/src/agents/librarian/tools.py @@ -9,6 +9,18 @@ from src.core.logging_config import get_logger logger = get_logger(__name__) +# Icons keyed by the values library-desk emits in each result's `sources` +# list (search legs) and `source_type` (result origin). +SOURCE_ICONS = { + "vector": "📄", + "graph": "🔗", + "web": "🌐", + "document": "📑", + "documents": "📑", + "volatile": "⚡", + "wiki": "📄", +} + # ============================================================================ # HybridRAG Search @@ -76,13 +88,10 @@ async def hybrid_search( # Add results for i, result in enumerate(response.results, 1): - source_icon = { - "vector": "📄", - "graph": "🔗", - "web": "🌐", - "document": "📑", - "volatile": "⚡", - }.get(result.source, "•") + source_keys = result.sources or [result.source] + source_icon = "".join( + dict.fromkeys(SOURCE_ICONS.get(key, "•") for key in source_keys) + ) output_parts.append( f"{i}. {source_icon} **{result.title}** (score: {result.score:.2f})" diff --git a/tests/agents/librarian/fixtures/hybrid_query_recorded.json b/tests/agents/librarian/fixtures/hybrid_query_recorded.json new file mode 100644 index 0000000..4c964b6 --- /dev/null +++ b/tests/agents/librarian/fixtures/hybrid_query_recorded.json @@ -0,0 +1,268 @@ +{ + "query": "home server infrastructure", + "keywords": { + "core_keywords": [ + "home", + "server", + "infrastructure" + ], + "entities": [], + "synonyms": {}, + "expansions": {} + }, + "results": [ + { + "source_type": "wiki", + "title": "Tower of Joy - AI Butler System", + "content": "", + "url": null, + "page_id": 146, + "page_path": "users/jpmschweitzer/projects/tower-of-joy", + "paperless_id": null, + "rrf_score": 0.01639344262295082, + "final_rank": 1, + "sources": [ + "graph" + ], + "related_dossiers": [ + { + "page_id": 161, + "title": "Library Desk - Knowledge Management", + "path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk", + "tag": "ai", + "shared_entities": 13 + }, + { + "page_id": 161, + "title": "Library Desk - Knowledge Management", + "path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk", + "tag": "dossier:tatlock", + "shared_entities": 13 + }, + { + "page_id": 161, + "title": "Library Desk - Knowledge Management", + "path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk", + "tag": "applications", + "shared_entities": 13 + }, + { + "page_id": 167, + "title": "Qdrant - Vector Database", + "path": "users/jpmschweitzer/projects/tower-of-joy/applications/qdrant", + "tag": "vector", + "shared_entities": 12 + }, + { + "page_id": 167, + "title": "Qdrant - Vector Database", + "path": "users/jpmschweitzer/projects/tower-of-joy/applications/qdrant", + "tag": "dossier:tatlock", + "shared_entities": 12 + } + ], + "metadata": { + "entity_matches": 3, + "matched_entities": [ + "Infrastructure Services", + "Infrastructure Layer\nThe" + ], + "engine": null + } + }, + { + "source_type": "web", + "title": "What do I need to start a home server ? Can I go in almost blind", + "content": "You don't need industrial grade hardware to be a server. You may get better reliability and management options from that, but they can all run\u00a0...", + "url": "https://www.reddit.com/r/HomeServer/comments/1rx7udl/what_do_i_need_to_start_a_home_server_can_i_go_in/", + "page_id": null, + "page_path": null, + "paperless_id": null, + "rrf_score": 0.01639344262295082, + "final_rank": 2, + "sources": [ + "web" + ], + "related_dossiers": [], + "metadata": { + "entity_matches": null, + "matched_entities": null, + "engine": "startpage" + } + }, + { + "source_type": "wiki", + "title": "PostgreSQL Shared - Database Server", + "content": "", + "url": null, + "page_id": 148, + "page_path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/postgres", + "paperless_id": null, + "rrf_score": 0.016129032258064516, + "final_rank": 3, + "sources": [ + "graph" + ], + "related_dossiers": [ + { + "page_id": 149, + "title": "Redis Shared - Cache and Session Store", + "path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis", + "tag": "infrastructure", + "shared_entities": 5 + }, + { + "page_id": 149, + "title": "Redis Shared - Cache and Session Store", + "path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis", + "tag": "dossier:tatlock", + "shared_entities": 5 + }, + { + "page_id": 149, + "title": "Redis Shared - Cache and Session Store", + "path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis", + "tag": "cache", + "shared_entities": 5 + }, + { + "page_id": 105, + "title": "Google Cloud Platform", + "path": "users/jpmschweitzer/technology/cloud-platforms/gcp", + "tag": "technology", + "shared_entities": 4 + }, + { + "page_id": 105, + "title": "Google Cloud Platform", + "path": "users/jpmschweitzer/technology/cloud-platforms/gcp", + "tag": "cloud-platforms", + "shared_entities": 4 + } + ], + "metadata": { + "entity_matches": 1, + "matched_entities": [ + "Database Server" + ], + "engine": null + } + }, + { + "source_type": "web", + "title": "25+ Must-Have Home Server Services for 2025 (Ultimate Guide)", + "content": "I\u2019ve been running a home server setup for years now, and it\u2019s been an incredible journey of discovery, learning, and practical benefits.\nIf you\u2019re considering setting up a home server or looking to expand your existing home lab, you\u2019re in the right place.\nIn this comprehensive guide, I\u2019ll walk you through the essential services that can transform your home server from a simple file storage system into a powerful, versatile hub that enhances your digital life.\nFrom media streaming to home automation, security, productivity, and more \u2013 we\u2019ll cover it all.\nWhether you\u2019re a seasoned self-hosting veteran or just taking your first steps into home server territory, this guide will help you discover new possibilities and build a setup that perfectly suits your needs.\nFoundation Services: The Building Blocks\nBefore diving into specific applications, let\u2019s cover the fundamental services that form the backbone of any robust home server setup.\nThese core components provide the infrastructure for everything else to run smoothly.\nHypervisors & Virtualization Platforms\nHypervisors allow you to run multiple virtual machines on a single physical server, making them crucial for efficient resource utilization.\nProxmox VE: The Homelab Virtualization King\nProxmox is my top recommendation for home server virtualization.\nThis open-source solution combines KVM virtualization with LXC containers, a powerful web interface, and built-in features like clustering, backups, and storage management.\nProxmox gives you the ability to run both full virtual machines and lightweight containers on the same hardware.\nIt\u2019s been extremely reliable in my setup, and its active community provides excellent support.\nI\u2019ve been using Proxmox for about three years, after switching from ESXi.\nThe transition was straightforward, and I\u2019ve found it much more suitable for home lab use.\nYou can create clusters with multiple nodes, run HA setups, and even use distributed storage with Ceph.\nTrueNAS SCALE: Storage with Ad...", + "url": "https://hostbor.com/25-must-have-home-server-services/", + "page_id": null, + "page_path": null, + "paperless_id": null, + "rrf_score": 0.016129032258064516, + "final_rank": 4, + "sources": [ + "web" + ], + "related_dossiers": [], + "metadata": { + "entity_matches": null, + "matched_entities": null, + "engine": "duckduckgo" + } + }, + { + "source_type": "wiki", + "title": "Bazzite", + "content": "", + "url": null, + "page_id": 108, + "page_path": "users/jpmschweitzer/technology/linux_distributions/bazzite", + "paperless_id": null, + "rrf_score": 0.015873015873015872, + "final_rank": 5, + "sources": [ + "graph" + ], + "related_dossiers": [ + { + "page_id": 110, + "title": "Zorin OS", + "path": "users/jpmschweitzer/technology/linux_distributions/zorin_os", + "tag": "technology", + "shared_entities": 22 + }, + { + "page_id": 110, + "title": "Zorin OS", + "path": "users/jpmschweitzer/technology/linux_distributions/zorin_os", + "tag": "linux_distributions", + "shared_entities": 22 + }, + { + "page_id": 110, + "title": "Zorin OS", + "path": "users/jpmschweitzer/technology/linux_distributions/zorin_os", + "tag": "zorin_os", + "shared_entities": 22 + }, + { + "page_id": 109, + "title": "Linux", + "path": "users/jpmschweitzer/technology/linux", + "tag": "technology", + "shared_entities": 21 + }, + { + "page_id": 109, + "title": "Linux", + "path": "users/jpmschweitzer/technology/linux", + "tag": "linux", + "shared_entities": 21 + } + ], + "metadata": { + "entity_matches": 1, + "matched_entities": [ + "Display Server" + ], + "engine": null + } + } + ], + "context": "1. [WIKI] Tower of Joy - AI Butler System\n (no content)...\n Related research: ai, dossier:tatlock, applications\n\n2. [WEB] What do I need to start a home server ? Can I go in almost blind\n You don't need industrial grade hardware to be a server. You may get better reliability and management options from that, but they can all run\u00a0......\n\n3. [WIKI] PostgreSQL Shared - Database Server\n (no content)...\n Related research: infrastructure, dossier:tatlock, cache\n\n4. [WEB] 25+ Must-Have Home Server Services for 2025 (Ultimate Guide)\n I\u2019ve been running a home server setup for years now, and it\u2019s been an incredible journey of discovery, learning, and practical benefits.\nIf you\u2019re considering setting up a home server or looking to expand your existing home lab, you\u2019re in the right place.\nIn this comprehensive guide, I\u2019ll walk you t...\n\n5. [WIKI] Bazzite\n (no content)...\n Related research: technology, linux_distributions, zorin_os", + "source_counts": { + "graph": 3, + "web": 2 + }, + "total_results": 5, + "timing": { + "query_enhancement_ms": 30.002593994140625, + "vector_ms": 105.46708106994629, + "graph_ms": 21.07977867126465, + "web_ms": 1577.7764320373535, + "volatile_ms": 0.0, + "document_ms": 1.8155574798583984, + "fusion_ms": 0.1761913299560547, + "enrichment_ms": 14.33563232421875, + "reranking_ms": 0.0002384185791015625, + "persistence_ms": 33.80393981933594, + "total_ms": 1624.767780303955 + }, + "config_used": { + "vector_limit": 3, + "graph_limit": 3, + "web_limit": 2, + "volatile_limit": 1, + "document_limit": 2, + "enable_vector": true, + "enable_graph": true, + "enable_web": true, + "enable_volatile": false, + "enable_documents": true, + "enable_reranking": false, + "enable_enrichment": true, + "final_result_count": 6, + "rrf_k": 60, + "volatile_threshold": 0.8, + "document_threshold": 0.6 + }, + "search_id": "01062bc7-ca65-4d9a-a210-e1a8f44b93c1" +} diff --git a/tests/agents/librarian/test_hybrid_contract.py b/tests/agents/librarian/test_hybrid_contract.py new file mode 100644 index 0000000..e7b6478 --- /dev/null +++ b/tests/agents/librarian/test_hybrid_contract.py @@ -0,0 +1,175 @@ +""" +Contract tests for HybridRAG parsing against a recorded live response. + +The fixture in fixtures/hybrid_query_recorded.json is a real (recorded) +response from library-desk's POST /query/hybrid. These tests pin the +field mapping (source_type/sources, rrf_score, context, per-item +related_dossiers, keywords dict with nested synonyms) so a drift in +either side shows up as a test failure instead of every result +rendering as "unknown (score: 0.00)". +""" + +import json +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient +from src.agents.librarian.tools import SOURCE_ICONS, hybrid_search + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "hybrid_query_recorded.json" + + +@pytest.fixture +def recorded_response() -> dict: + """Load the recorded /query/hybrid response.""" + return json.loads(FIXTURE_PATH.read_text()) + + +@pytest.fixture +def client_with_recorded_response(recorded_response): + """LibraryDeskClient whose httpx client replays the recorded response.""" + mock_response = MagicMock() + mock_response.json.return_value = recorded_response + mock_response.raise_for_status = MagicMock() + + mock_httpx = AsyncMock(spec=httpx.AsyncClient) + mock_httpx.post.return_value = mock_response + + client = LibraryDeskClient(base_url="http://test:8089", api_key="test-key") + client._client = mock_httpx + return client + + +@pytest.mark.unit +class TestHybridRAGContract: + """Contract tests for parsing the live /query/hybrid response shape.""" + + @pytest.mark.asyncio + async def test_sources_are_not_unknown(self, client_with_recorded_response): + """Every result maps source_type - nothing falls back to 'unknown'.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + assert isinstance(response, HybridRAGResponse) + assert response.results, "recorded fixture must contain results" + for result in response.results: + assert result.source != "unknown" + assert result.source in {"wiki", "web", "volatile", "document"} + + @pytest.mark.asyncio + async def test_scores_are_non_zero(self, client_with_recorded_response): + """rrf_score maps to score - no silent 0.00 fallback.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + for result in response.results: + assert result.score > 0.0 + + @pytest.mark.asyncio + async def test_sources_list_and_icons(self, client_with_recorded_response): + """Per-item sources list is parsed and every value has an icon.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + for result in response.results: + assert result.sources, f"result '{result.title}' has empty sources" + for source in result.sources: + assert source in SOURCE_ICONS, f"no icon for source '{source}'" + + @pytest.mark.asyncio + async def test_context_maps_to_formatted_context( + self, client_with_recorded_response + ): + """Top-level 'context' field maps to formatted_context.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + assert response.formatted_context != "" + + @pytest.mark.asyncio + async def test_keywords_and_synonyms_from_dict( + self, client_with_recorded_response + ): + """keywords is a dict: core_keywords + nested synonyms map.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + assert response.keywords, "core_keywords should be extracted" + assert all(isinstance(k, str) for k in response.keywords) + # synonyms map in the fixture is empty, but must parse to a list + assert isinstance(response.synonyms, list) + + @pytest.mark.asyncio + async def test_per_item_related_dossiers(self, client_with_recorded_response): + """related_dossiers live per result and aggregate to unique titles.""" + response = await client_with_recorded_response.hybrid_search( + "home server infrastructure", user="testuser" + ) + + per_item = [d for r in response.results for d in r.related_dossiers] + assert per_item, "recorded fixture contains per-item related_dossiers" + for dossier in per_item: + assert "title" in dossier + assert "tag" in dossier + + assert response.related_dossiers, "top-level titles are aggregated" + assert len(response.related_dossiers) == len(set(response.related_dossiers)) + + @pytest.mark.asyncio + async def test_payload_never_sends_zero_limits( + self, client_with_recorded_response + ): + """The live service 422s on limits < 1; disabled legs use enable_* flags.""" + await client_with_recorded_response.hybrid_search( + "home server infrastructure", + user="testuser", + web_limit=0, + document_limit=0, + volatile_limit=0, + ) + + payload = client_with_recorded_response._client.post.call_args.kwargs["json"] + config = payload["config"] + for key in ( + "vector_limit", + "graph_limit", + "web_limit", + "document_limit", + "volatile_limit", + ): + assert config[key] >= 1 + assert config["enable_web"] is False + assert config["enable_documents"] is False + assert config["enable_volatile"] is False + + @pytest.mark.asyncio + async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch): + """The hybrid_search tool renders real sources and non-zero scores.""" + + class _Factory: + def __call__(self): + return self + + async def __aenter__(self): + return client_with_recorded_response + + async def __aexit__(self, *args): + return None + + monkeypatch.setattr( + "src.agents.librarian.tools.LibraryDeskClient", _Factory() + ) + + output = await hybrid_search("home server infrastructure") + + assert "unknown" not in output + assert "score: 0.00" not in output + assert "•" not in output, "every source value should map to an icon"