jpmschweitzer/tatlock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

fix(librarian): map live HybridRAG response fields correctly

Browse Source 
The client parsed field names the live library-desk service never
returns, so every result rendered as "unknown (score: 0.00)":

- source_type/sources -> source + sources (icons key off sources values)
- rrf_score -> score
- context -> formatted_context
- related_dossiers are per-result; top level aggregates unique titles
- synonyms live inside the keywords dict as a {term: [synonyms]} map

Also stop sending zero limits (service 422s on limit < 1); disabled
legs now rely on the enable_* flags with limits clamped to >= 1.

Adds a recorded live response as a fixture plus contract tests that
pin the mapping (non-unknown sources, non-zero scores, icon coverage).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jpmschweitzer
2026-07-14 10:00:10 +02:00
co-authored by Claude Fable 5
parent 6ec77091b6
commit 59f5b54ac9
5 changed files with 500 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+4
View File
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- **HybridRAG response mapping** - The librarian client now parses the field names library-desk actually returns (`source_type`/`sources`, `rrf_score`, `context`, per-item `related_dossiers`, synonyms nested in the `keywords` dict); previously every result rendered as "unknown (score: 0.00)". Source icons now key off the per-item `sources` list. Requests no longer send zero limits (the service rejects them with 422); legs are disabled via `enable_*` flags. Pinned by a contract test against a recorded live response (`tests/agents/librarian/fixtures/`)
## [2.3.0] - 2026-07-13
### Changed
src/agents/librarian/client.py
+37 -14
View File
@@ -56,12 +56,14 @@ class VectorSearchResult(BaseModel):
class HybridSearchResult(BaseModel):
"""Result from HybridRAG search."""
source: str # "vector", "graph", "web"
source: str # source_type: "wiki", "web", "volatile", "document"
sources: list[str] = Field(default_factory=list) # legs that found it: "vector", "graph", "web", ...
title: str
content: str
url: str | None = None
score: float
score: float # rrf_score from the live service
page_id: int | None = None
related_dossiers: list[dict[str, Any]] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
@@ -250,14 +252,16 @@ class LibraryDeskClient:
user = user or get_user()
client = self._ensure_client()
# The live service requires all limits >= 1 (422 otherwise);
# legs are disabled via the enable_* flags, not a zero limit.
payload = {
"query": query,
"config": {
"vector_limit": vector_limit,
"graph_limit": graph_limit,
"web_limit": web_limit,
"document_limit": document_limit,
"volatile_limit": volatile_limit,
"vector_limit": max(vector_limit, 1),
"graph_limit": max(graph_limit, 1),
"web_limit": max(web_limit, 1),
"document_limit": max(document_limit, 1),
"volatile_limit": max(volatile_limit, 1),
"enable_documents": document_limit > 0,
"enable_volatile": volatile_limit > 0,
"enable_web": web_limit > 0,
@@ -277,32 +281,51 @@ class LibraryDeskClient:
data = response.json()
# Parse results
# Parse results (live field names: source_type, sources, rrf_score,
# related_dossiers; older names kept as fallbacks)
results = []
for r in data.get("results", []):
results.append(HybridSearchResult(
source=r.get("source", "unknown"),
source=r.get("source_type") or r.get("source", "unknown"),
sources=r.get("sources", []),
title=r.get("title", ""),
content=r.get("content", ""),
url=r.get("url"),
score=r.get("score", 0.0),
score=r.get("rrf_score", r.get("score", 0.0)),
page_id=r.get("page_id"),
related_dossiers=r.get("related_dossiers", []),
metadata=r.get("metadata", {}),
))
# Handle keywords being either a list or a dict with core_keywords
# Handle keywords being either a list or a dict with core_keywords;
# the live service nests synonyms inside the keywords dict as a
# {term: [synonyms]} map.
raw_keywords = data.get("keywords", [])
raw_synonyms: Any = data.get("synonyms", [])
if isinstance(raw_keywords, dict):
keywords = raw_keywords.get("core_keywords", [])
raw_synonyms = raw_keywords.get("synonyms", {})
else:
keywords = raw_keywords
if isinstance(raw_synonyms, dict):
synonyms = [s for values in raw_synonyms.values() for s in values]
else:
synonyms = raw_synonyms
# Aggregate per-result related dossiers into unique top-level titles
related_dossiers: list[str] = []
for result in results:
for dossier in result.related_dossiers:
title = dossier.get("title", "")
if title and title not in related_dossiers:
related_dossiers.append(title)
return HybridRAGResponse(
results=results,
keywords=keywords,
synonyms=data.get("synonyms", []),
related_dossiers=data.get("related_dossiers", []),
formatted_context=data.get("formatted_context", ""),
synonyms=synonyms,
related_dossiers=related_dossiers,
formatted_context=data.get("context", data.get("formatted_context", "")),
search_id=data.get("search_id"),
timing=data.get("timing", {}),
)
src/agents/librarian/tools.py
+16 -7
View File
@@ -9,6 +9,18 @@ from src.core.logging_config import get_logger
logger = get_logger(__name__)
# Icons keyed by the values library-desk emits in each result's `sources`
# list (search legs) and `source_type` (result origin).
SOURCE_ICONS = {
"vector": "📄",
"graph": "🔗",
"web": "🌐",
"document": "📑",
"documents": "📑",
"volatile": "⚡",
"wiki": "📄",
}
# ============================================================================
# HybridRAG Search
@@ -76,13 +88,10 @@ async def hybrid_search(
# Add results
for i, result in enumerate(response.results, 1):
source_icon = {
"vector": "📄",
"graph": "🔗",
"web": "🌐",
"document": "📑",
"volatile": "⚡",
}.get(result.source, "•")
source_keys = result.sources or [result.source]
source_icon = "".join(
dict.fromkeys(SOURCE_ICONS.get(key, "•") for key in source_keys)
)
output_parts.append(
f"{i}. {source_icon} **{result.title}** (score: {result.score:.2f})"
tests/agents/librarian/fixtures/hybrid_query_recorded.json
+268
View File
@@ -0,0 +1,268 @@
{
"query": "home server infrastructure",
"keywords": {
"core_keywords": [
"home",
"server",
"infrastructure"
],
"entities": [],
"synonyms": {},
"expansions": {}
},
"results": [
{
"source_type": "wiki",
"title": "Tower of Joy - AI Butler System",
"content": "",
"url": null,
"page_id": 146,
"page_path": "users/jpmschweitzer/projects/tower-of-joy",
"paperless_id": null,
"rrf_score": 0.01639344262295082,
"final_rank": 1,
"sources": [
"graph"
],
"related_dossiers": [
{
"page_id": 161,
"title": "Library Desk - Knowledge Management",
"path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk",
"tag": "ai",
"shared_entities": 13
},
{
"page_id": 161,
"title": "Library Desk - Knowledge Management",
"path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk",
"tag": "dossier:tatlock",
"shared_entities": 13
},
{
"page_id": 161,
"title": "Library Desk - Knowledge Management",
"path": "users/jpmschweitzer/projects/tower-of-joy/applications/library-desk",
"tag": "applications",
"shared_entities": 13
},
{
"page_id": 167,
"title": "Qdrant - Vector Database",
"path": "users/jpmschweitzer/projects/tower-of-joy/applications/qdrant",
"tag": "vector",
"shared_entities": 12
},
{
"page_id": 167,
"title": "Qdrant - Vector Database",
"path": "users/jpmschweitzer/projects/tower-of-joy/applications/qdrant",
"tag": "dossier:tatlock",
"shared_entities": 12
}
],
"metadata": {
"entity_matches": 3,
"matched_entities": [
"Infrastructure Services",
"Infrastructure Layer\nThe"
],
"engine": null
}
},
{
"source_type": "web",
"title": "What do I need to start a home server ? Can I go in almost blind",
"content": "You don't need industrial grade hardware to be a server. You may get better reliability and management options from that, but they can all run\u00a0...",
"url": "https://www.reddit.com/r/HomeServer/comments/1rx7udl/what_do_i_need_to_start_a_home_server_can_i_go_in/",
"page_id": null,
"page_path": null,
"paperless_id": null,
"rrf_score": 0.01639344262295082,
"final_rank": 2,
"sources": [
"web"
],
"related_dossiers": [],
"metadata": {
"entity_matches": null,
"matched_entities": null,
"engine": "startpage"
}
},
{
"source_type": "wiki",
"title": "PostgreSQL Shared - Database Server",
"content": "",
"url": null,
"page_id": 148,
"page_path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/postgres",
"paperless_id": null,
"rrf_score": 0.016129032258064516,
"final_rank": 3,
"sources": [
"graph"
],
"related_dossiers": [
{
"page_id": 149,
"title": "Redis Shared - Cache and Session Store",
"path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis",
"tag": "infrastructure",
"shared_entities": 5
},
{
"page_id": 149,
"title": "Redis Shared - Cache and Session Store",
"path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis",
"tag": "dossier:tatlock",
"shared_entities": 5
},
{
"page_id": 149,
"title": "Redis Shared - Cache and Session Store",
"path": "users/jpmschweitzer/projects/tower-of-joy/infrastructure/redis",
"tag": "cache",
"shared_entities": 5
},
{
"page_id": 105,
"title": "Google Cloud Platform",
"path": "users/jpmschweitzer/technology/cloud-platforms/gcp",
"tag": "technology",
"shared_entities": 4
},
{
"page_id": 105,
"title": "Google Cloud Platform",
"path": "users/jpmschweitzer/technology/cloud-platforms/gcp",
"tag": "cloud-platforms",
"shared_entities": 4
}
],
"metadata": {
"entity_matches": 1,
"matched_entities": [
"Database Server"
],
"engine": null
}
},
{
"source_type": "web",
"title": "25+ Must-Have Home Server Services for 2025 (Ultimate Guide)",
"content": "I\u2019ve been running a home server setup for years now, and it\u2019s been an incredible journey of discovery, learning, and practical benefits.\nIf you\u2019re considering setting up a home server or looking to expand your existing home lab, you\u2019re in the right place.\nIn this comprehensive guide, I\u2019ll walk you through the essential services that can transform your home server from a simple file storage system into a powerful, versatile hub that enhances your digital life.\nFrom media streaming to home automation, security, productivity, and more \u2013 we\u2019ll cover it all.\nWhether you\u2019re a seasoned self-hosting veteran or just taking your first steps into home server territory, this guide will help you discover new possibilities and build a setup that perfectly suits your needs.\nFoundation Services: The Building Blocks\nBefore diving into specific applications, let\u2019s cover the fundamental services that form the backbone of any robust home server setup.\nThese core components provide the infrastructure for everything else to run smoothly.\nHypervisors & Virtualization Platforms\nHypervisors allow you to run multiple virtual machines on a single physical server, making them crucial for efficient resource utilization.\nProxmox VE: The Homelab Virtualization King\nProxmox is my top recommendation for home server virtualization.\nThis open-source solution combines KVM virtualization with LXC containers, a powerful web interface, and built-in features like clustering, backups, and storage management.\nProxmox gives you the ability to run both full virtual machines and lightweight containers on the same hardware.\nIt\u2019s been extremely reliable in my setup, and its active community provides excellent support.\nI\u2019ve been using Proxmox for about three years, after switching from ESXi.\nThe transition was straightforward, and I\u2019ve found it much more suitable for home lab use.\nYou can create clusters with multiple nodes, run HA setups, and even use distributed storage with Ceph.\nTrueNAS SCALE: Storage with Ad...",
"url": "https://hostbor.com/25-must-have-home-server-services/",
"page_id": null,
"page_path": null,
"paperless_id": null,
"rrf_score": 0.016129032258064516,
"final_rank": 4,
"sources": [
"web"
],
"related_dossiers": [],
"metadata": {
"entity_matches": null,
"matched_entities": null,
"engine": "duckduckgo"
}
},
{
"source_type": "wiki",
"title": "Bazzite",
"content": "",
"url": null,
"page_id": 108,
"page_path": "users/jpmschweitzer/technology/linux_distributions/bazzite",
"paperless_id": null,
"rrf_score": 0.015873015873015872,
"final_rank": 5,
"sources": [
"graph"
],
"related_dossiers": [
{
"page_id": 110,
"title": "Zorin OS",
"path": "users/jpmschweitzer/technology/linux_distributions/zorin_os",
"tag": "technology",
"shared_entities": 22
},
{
"page_id": 110,
"title": "Zorin OS",
"path": "users/jpmschweitzer/technology/linux_distributions/zorin_os",
"tag": "linux_distributions",
"shared_entities": 22
},
{
"page_id": 110,
"title": "Zorin OS",
"path": "users/jpmschweitzer/technology/linux_distributions/zorin_os",
"tag": "zorin_os",
"shared_entities": 22
},
{
"page_id": 109,
"title": "Linux",
"path": "users/jpmschweitzer/technology/linux",
"tag": "technology",
"shared_entities": 21
},
{
"page_id": 109,
"title": "Linux",
"path": "users/jpmschweitzer/technology/linux",
"tag": "linux",
"shared_entities": 21
}
],
"metadata": {
"entity_matches": 1,
"matched_entities": [
"Display Server"
],
"engine": null
}
}
],
"context": "1. [WIKI] Tower of Joy - AI Butler System\n (no content)...\n Related research: ai, dossier:tatlock, applications\n\n2. [WEB] What do I need to start a home server ? Can I go in almost blind\n You don't need industrial grade hardware to be a server. You may get better reliability and management options from that, but they can all run\u00a0......\n\n3. [WIKI] PostgreSQL Shared - Database Server\n (no content)...\n Related research: infrastructure, dossier:tatlock, cache\n\n4. [WEB] 25+ Must-Have Home Server Services for 2025 (Ultimate Guide)\n I\u2019ve been running a home server setup for years now, and it\u2019s been an incredible journey of discovery, learning, and practical benefits.\nIf you\u2019re considering setting up a home server or looking to expand your existing home lab, you\u2019re in the right place.\nIn this comprehensive guide, I\u2019ll walk you t...\n\n5. [WIKI] Bazzite\n (no content)...\n Related research: technology, linux_distributions, zorin_os",
"source_counts": {
"graph": 3,
"web": 2
},
"total_results": 5,
"timing": {
"query_enhancement_ms": 30.002593994140625,
"vector_ms": 105.46708106994629,
"graph_ms": 21.07977867126465,
"web_ms": 1577.7764320373535,
"volatile_ms": 0.0,
"document_ms": 1.8155574798583984,
"fusion_ms": 0.1761913299560547,
"enrichment_ms": 14.33563232421875,
"reranking_ms": 0.0002384185791015625,
"persistence_ms": 33.80393981933594,
"total_ms": 1624.767780303955
},
"config_used": {
"vector_limit": 3,
"graph_limit": 3,
"web_limit": 2,
"volatile_limit": 1,
"document_limit": 2,
"enable_vector": true,
"enable_graph": true,
"enable_web": true,
"enable_volatile": false,
"enable_documents": true,
"enable_reranking": false,
"enable_enrichment": true,
"final_result_count": 6,
"rrf_k": 60,
"volatile_threshold": 0.8,
"document_threshold": 0.6
},
"search_id": "01062bc7-ca65-4d9a-a210-e1a8f44b93c1"
}
tests/agents/librarian/test_hybrid_contract.py
+175
View File
@@ -0,0 +1,175 @@
"""
Contract tests for HybridRAG parsing against a recorded live response.
The fixture in fixtures/hybrid_query_recorded.json is a real (recorded)
response from library-desk's POST /query/hybrid. These tests pin the
field mapping (source_type/sources, rrf_score, context, per-item
related_dossiers, keywords dict with nested synonyms) so a drift in
either side shows up as a test failure instead of every result
rendering as "unknown (score: 0.00)".
"""
import json
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient
from src.agents.librarian.tools import SOURCE_ICONS, hybrid_search
FIXTURE_PATH = Path(__file__).parent / "fixtures" / "hybrid_query_recorded.json"
@pytest.fixture
def recorded_response() -> dict:
"""Load the recorded /query/hybrid response."""
return json.loads(FIXTURE_PATH.read_text())
@pytest.fixture
def client_with_recorded_response(recorded_response):
"""LibraryDeskClient whose httpx client replays the recorded response."""
mock_response = MagicMock()
mock_response.json.return_value = recorded_response
mock_response.raise_for_status = MagicMock()
mock_httpx = AsyncMock(spec=httpx.AsyncClient)
mock_httpx.post.return_value = mock_response
client = LibraryDeskClient(base_url="http://test:8089", api_key="test-key")
client._client = mock_httpx
return client
@pytest.mark.unit
class TestHybridRAGContract:
"""Contract tests for parsing the live /query/hybrid response shape."""
@pytest.mark.asyncio
async def test_sources_are_not_unknown(self, client_with_recorded_response):
"""Every result maps source_type - nothing falls back to 'unknown'."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
assert isinstance(response, HybridRAGResponse)
assert response.results, "recorded fixture must contain results"
for result in response.results:
assert result.source != "unknown"
assert result.source in {"wiki", "web", "volatile", "document"}
@pytest.mark.asyncio
async def test_scores_are_non_zero(self, client_with_recorded_response):
"""rrf_score maps to score - no silent 0.00 fallback."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
for result in response.results:
assert result.score > 0.0
@pytest.mark.asyncio
async def test_sources_list_and_icons(self, client_with_recorded_response):
"""Per-item sources list is parsed and every value has an icon."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
for result in response.results:
assert result.sources, f"result '{result.title}' has empty sources"
for source in result.sources:
assert source in SOURCE_ICONS, f"no icon for source '{source}'"
@pytest.mark.asyncio
async def test_context_maps_to_formatted_context(
self, client_with_recorded_response
):
"""Top-level 'context' field maps to formatted_context."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
assert response.formatted_context != ""
@pytest.mark.asyncio
async def test_keywords_and_synonyms_from_dict(
self, client_with_recorded_response
):
"""keywords is a dict: core_keywords + nested synonyms map."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
assert response.keywords, "core_keywords should be extracted"
assert all(isinstance(k, str) for k in response.keywords)
# synonyms map in the fixture is empty, but must parse to a list
assert isinstance(response.synonyms, list)
@pytest.mark.asyncio
async def test_per_item_related_dossiers(self, client_with_recorded_response):
"""related_dossiers live per result and aggregate to unique titles."""
response = await client_with_recorded_response.hybrid_search(
"home server infrastructure", user="testuser"
)
per_item = [d for r in response.results for d in r.related_dossiers]
assert per_item, "recorded fixture contains per-item related_dossiers"
for dossier in per_item:
assert "title" in dossier
assert "tag" in dossier
assert response.related_dossiers, "top-level titles are aggregated"
assert len(response.related_dossiers) == len(set(response.related_dossiers))
@pytest.mark.asyncio
async def test_payload_never_sends_zero_limits(
self, client_with_recorded_response
):
"""The live service 422s on limits < 1; disabled legs use enable_* flags."""
await client_with_recorded_response.hybrid_search(
"home server infrastructure",
user="testuser",
web_limit=0,
document_limit=0,
volatile_limit=0,
)
payload = client_with_recorded_response._client.post.call_args.kwargs["json"]
config = payload["config"]
for key in (
"vector_limit",
"graph_limit",
"web_limit",
"document_limit",
"volatile_limit",
):
assert config[key] >= 1
assert config["enable_web"] is False
assert config["enable_documents"] is False
assert config["enable_volatile"] is False
@pytest.mark.asyncio
async def test_tool_renders_no_unknown_results(self, client_with_recorded_response, monkeypatch):
"""The hybrid_search tool renders real sources and non-zero scores."""
class _Factory:
def __call__(self):
return self
async def __aenter__(self):
return client_with_recorded_response
async def __aexit__(self, *args):
return None
monkeypatch.setattr(
"src.agents.librarian.tools.LibraryDeskClient", _Factory()
)
output = await hybrid_search("home server infrastructure")
assert "unknown" not in output
assert "score: 0.00" not in output
assert "•" not in output, "every source value should map to an icon"