test: add offline unit tests for Phase A reliability fixes
Mock-based tests (no live services) covering:
- ollama_llm_model resolution under the OLLAMA_MODEL env collision
- per-leg retrieval failure -> source_status/degraded signaling
- Phase 0/Phase 4 LLM timeout fallbacks
- /stats wiki page count using the users/{user} path prefix
- Wiki.js list_all_pages limit-growth pagination and
list_pages limit-after-filter behavior
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Unit tests for application settings (offline).
|
||||
|
||||
Covers the OLLAMA_MODEL env collision: the deployed container sets
|
||||
OLLAMA_MODEL=nomic-embed-text for embeddings, which must NOT shadow the
|
||||
generation model setting (ollama_llm_model / OLLAMA_LLM_MODEL).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.config import Settings
|
||||
|
||||
# Required fields so Settings can be constructed without a .env file
|
||||
REQUIRED = {
|
||||
"library_api_key": "test-key",
|
||||
"neo4j_password": "test-pass",
|
||||
"wikijs_db_password": "test-pass",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestOllamaModelResolution:
|
||||
"""Generation model resolution must be immune to the OLLAMA_MODEL env var."""
|
||||
|
||||
def test_ollama_model_env_does_not_shadow_generation_model(self, monkeypatch):
|
||||
"""The container env OLLAMA_MODEL (embedding model) must not leak into ollama_llm_model."""
|
||||
monkeypatch.setenv("OLLAMA_MODEL", "nomic-embed-text")
|
||||
|
||||
settings = Settings(_env_file=None, **REQUIRED)
|
||||
|
||||
assert settings.ollama_llm_model == "gemma4:e2b"
|
||||
assert settings.ollama_llm_model != "nomic-embed-text"
|
||||
|
||||
def test_generation_model_default(self, monkeypatch):
|
||||
monkeypatch.delenv("OLLAMA_LLM_MODEL", raising=False)
|
||||
|
||||
settings = Settings(_env_file=None, **REQUIRED)
|
||||
|
||||
assert settings.ollama_llm_model == "gemma4:e2b"
|
||||
|
||||
def test_generation_model_from_dedicated_env_var(self, monkeypatch):
|
||||
"""OLLAMA_LLM_MODEL is the dedicated env var for the generation model."""
|
||||
monkeypatch.setenv("OLLAMA_MODEL", "nomic-embed-text")
|
||||
monkeypatch.setenv("OLLAMA_LLM_MODEL", "mistral-nemo:latest")
|
||||
|
||||
settings = Settings(_env_file=None, **REQUIRED)
|
||||
|
||||
assert settings.ollama_llm_model == "mistral-nemo:latest"
|
||||
|
||||
def test_embedding_model_setting_untouched(self, monkeypatch):
|
||||
"""The embedding model keeps its own setting and env var."""
|
||||
monkeypatch.setenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text")
|
||||
|
||||
settings = Settings(_env_file=None, **REQUIRED)
|
||||
|
||||
assert settings.ollama_embedding_model == "nomic-embed-text"
|
||||
|
||||
def test_legacy_setting_name_removed(self):
|
||||
"""The old ollama_model attribute must be gone so nothing binds to OLLAMA_MODEL."""
|
||||
settings = Settings(_env_file=None, **REQUIRED)
|
||||
|
||||
assert not hasattr(settings, "ollama_model")
|
||||
@@ -0,0 +1,231 @@
|
||||
"""
|
||||
Unit tests for HybridRAG degradation signaling (offline, all clients mocked).
|
||||
|
||||
Covers:
|
||||
- Per-leg failure -> source_status reports 'failed', degraded=True
|
||||
- Disabled legs -> 'disabled', do not trigger degraded
|
||||
- Healthy legs -> 'ok', degraded=False
|
||||
- Reranker model resolution from settings.ollama_llm_model
|
||||
- Phase 0 keyword-extraction timeout falls back gracefully
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import src.services.hybrid_rag_service as hybrid_rag_module
|
||||
from src.services.hybrid_rag_service import HybridRAGService
|
||||
from src.models.hybrid_rag import HybridRAGConfig
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def settings():
|
||||
settings = MagicMock()
|
||||
settings.ollama_llm_model = "gemma4:e2b"
|
||||
settings.vector_similarity_threshold = 0.7
|
||||
return settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vector_service():
|
||||
"""Vector service returning a single wiki hit."""
|
||||
vector = MagicMock()
|
||||
hit = MagicMock()
|
||||
hit.page_id = 1
|
||||
hit.page_title = "Test Page"
|
||||
hit.content = "Test content"
|
||||
hit.page_path = "users/jp/test"
|
||||
hit.score = 0.9
|
||||
response = MagicMock()
|
||||
response.results = [hit]
|
||||
vector.search = AsyncMock(return_value=response)
|
||||
return vector
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def graph_service():
|
||||
graph = MagicMock()
|
||||
graph.search_documents = AsyncMock(return_value=[])
|
||||
graph.get_related_documents = AsyncMock(return_value=[])
|
||||
graph.neo4j.execute_query = AsyncMock(return_value=[{"id": "search-1"}])
|
||||
return graph
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def searxng_client():
|
||||
searxng = MagicMock()
|
||||
searxng.search_general = AsyncMock(return_value=[])
|
||||
return searxng
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ollama_client():
|
||||
ollama = MagicMock()
|
||||
ollama.generate_text = AsyncMock(
|
||||
return_value='{"core_keywords": ["test"], "synonyms": {}}'
|
||||
)
|
||||
return ollama
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def content_extractor():
|
||||
extractor = MagicMock()
|
||||
extractor.extract_batch = AsyncMock(return_value=[])
|
||||
return extractor
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service(settings, vector_service, graph_service, searxng_client, ollama_client, content_extractor):
|
||||
return HybridRAGService(
|
||||
vector_service=vector_service,
|
||||
graph_service=graph_service,
|
||||
searxng_client=searxng_client,
|
||||
ollama_client=ollama_client,
|
||||
content_extractor=content_extractor,
|
||||
settings=settings,
|
||||
volatile_service=None,
|
||||
)
|
||||
|
||||
|
||||
def make_config(**overrides):
|
||||
"""Config with documents disabled (leg needs a real Qdrant client) and fast phases."""
|
||||
defaults = {
|
||||
"enable_documents": False,
|
||||
"enable_volatile": False,
|
||||
"enable_reranking": False,
|
||||
"enable_enrichment": False,
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return HybridRAGConfig(**defaults)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSourceStatus:
|
||||
"""source_status must report every leg as ok/failed/disabled."""
|
||||
|
||||
async def test_all_enabled_legs_ok(self, service):
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
|
||||
assert response.source_status == {
|
||||
"vector": "ok",
|
||||
"graph": "ok",
|
||||
"web": "ok",
|
||||
"volatile": "disabled",
|
||||
"documents": "disabled",
|
||||
}
|
||||
assert response.degraded is False
|
||||
|
||||
async def test_failed_leg_reported_and_degraded(self, service, graph_service):
|
||||
graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down"))
|
||||
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
|
||||
assert response.source_status["graph"] == "failed"
|
||||
assert response.source_status["vector"] == "ok"
|
||||
assert response.degraded is True
|
||||
|
||||
async def test_failed_leg_still_contributes_no_results(self, service, graph_service):
|
||||
"""Existing behavior preserved: failure -> empty leg, other legs still work."""
|
||||
graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down"))
|
||||
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
|
||||
# The vector hit still comes through
|
||||
assert response.total_results == 1
|
||||
assert response.results[0].page_id == 1
|
||||
|
||||
async def test_multiple_failures(self, service, graph_service, searxng_client):
|
||||
graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down"))
|
||||
searxng_client.search_general = AsyncMock(side_effect=OSError("searxng unreachable"))
|
||||
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
|
||||
assert response.source_status["graph"] == "failed"
|
||||
assert response.source_status["web"] == "failed"
|
||||
assert response.degraded is True
|
||||
|
||||
async def test_disabled_legs_do_not_degrade(self, service):
|
||||
config = make_config(enable_graph=False, enable_web=False)
|
||||
|
||||
response = await service.search("test query", "jp", config)
|
||||
|
||||
assert response.source_status["graph"] == "disabled"
|
||||
assert response.source_status["web"] == "disabled"
|
||||
assert response.degraded is False
|
||||
|
||||
async def test_volatile_without_service_is_disabled(self, service):
|
||||
"""enable_volatile=True but no volatile service wired -> disabled, not failed."""
|
||||
response = await service.search("test query", "jp", make_config(enable_volatile=True))
|
||||
|
||||
assert response.source_status["volatile"] == "disabled"
|
||||
assert response.degraded is False
|
||||
|
||||
async def test_status_fields_serialized(self, service, graph_service):
|
||||
"""Contract: fields present in the serialized response for tatlock to parse."""
|
||||
graph_service.search_documents = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
payload = response.model_dump()
|
||||
|
||||
assert set(payload["source_status"].keys()) == {"vector", "graph", "web", "volatile", "documents"}
|
||||
assert payload["degraded"] is True
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestModelResolution:
|
||||
"""The reranker/keyword model must come from settings.ollama_llm_model."""
|
||||
|
||||
def test_reranker_model_from_llm_setting(self, service):
|
||||
assert service.reranker_model == "gemma4:e2b"
|
||||
|
||||
async def test_generation_calls_use_llm_model(self, service, ollama_client):
|
||||
await service.search("test query", "jp", make_config())
|
||||
|
||||
# Phase 0 keyword extraction ran with the generation model
|
||||
assert ollama_client.generate_text.await_count >= 1
|
||||
for call in ollama_client.generate_text.await_args_list:
|
||||
assert call.kwargs["model"] == "gemma4:e2b"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestLLMTimeout:
|
||||
"""A hung LLM call must not gate retrieval: 12s wait_for with fallback."""
|
||||
|
||||
async def test_keyword_extraction_timeout_falls_back(self, service, ollama_client, monkeypatch):
|
||||
monkeypatch.setattr(hybrid_rag_module, "LLM_CALL_TIMEOUT_SECONDS", 0.05)
|
||||
|
||||
async def hang(*args, **kwargs):
|
||||
await asyncio.sleep(5)
|
||||
|
||||
ollama_client.generate_text = AsyncMock(side_effect=hang)
|
||||
|
||||
response = await service.search("test query", "jp", make_config())
|
||||
|
||||
# Fallback: raw query words as keywords, retrieval still ran
|
||||
assert response.keywords.core_keywords == ["test", "query"]
|
||||
assert response.source_status["vector"] == "ok"
|
||||
|
||||
async def test_rerank_timeout_keeps_rrf_order(self, service, ollama_client, monkeypatch, searxng_client, content_extractor):
|
||||
monkeypatch.setattr(hybrid_rag_module, "LLM_CALL_TIMEOUT_SECONDS", 0.05)
|
||||
|
||||
# Two web results so re-ranking actually runs (needs > 1 result)
|
||||
searxng_client.search_general = AsyncMock(return_value=[
|
||||
{"url": "http://a.test", "title": "A", "content": "a"},
|
||||
{"url": "http://b.test", "title": "B", "content": "b"},
|
||||
])
|
||||
|
||||
keyword_json = '{"core_keywords": ["test"], "synonyms": {}}'
|
||||
|
||||
async def generate(prompt, **kwargs):
|
||||
if "Rank these documents" in prompt:
|
||||
await asyncio.sleep(5) # Hang only the re-rank call
|
||||
return keyword_json
|
||||
|
||||
ollama_client.generate_text = AsyncMock(side_effect=generate)
|
||||
|
||||
response = await service.search("test query", "jp", make_config(enable_reranking=True))
|
||||
|
||||
# RRF order preserved despite the hung re-rank call
|
||||
assert response.total_results >= 2
|
||||
assert response.degraded is False
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
Unit tests for the /stats endpoint (offline, all clients mocked).
|
||||
|
||||
Covers the wiki page count fix: the endpoint must count pages under the
|
||||
user namespace ("users/{user}"), not pass the bare user name as prefix
|
||||
(which matched nothing and always reported 0 pages).
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.main import stats
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def neo4j_client():
|
||||
neo4j = MagicMock()
|
||||
neo4j.execute_query = AsyncMock(return_value=[{"count": 5}])
|
||||
return neo4j
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qdrant_client():
|
||||
qdrant = MagicMock()
|
||||
qdrant.list_collections = AsyncMock(return_value=[
|
||||
{"name": "user_jp", "vectors_count": 42}
|
||||
])
|
||||
return qdrant
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wikijs_client():
|
||||
wikijs = MagicMock()
|
||||
wikijs.list_all_pages = AsyncMock(return_value=[
|
||||
{"id": i, "path": f"users/jpmschweitzer/p{i}"} for i in range(138)
|
||||
])
|
||||
return wikijs
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def paperless_client():
|
||||
paperless = MagicMock()
|
||||
paperless.list_documents = AsyncMock(return_value={"count": 3})
|
||||
paperless.list_tags = AsyncMock(return_value=[])
|
||||
paperless.list_correspondents = AsyncMock(return_value=[])
|
||||
paperless.list_document_types = AsyncMock(return_value=[])
|
||||
return paperless
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestStatsWikiPageCount:
|
||||
"""/stats must scope the wiki page count to the user namespace."""
|
||||
|
||||
async def test_uses_user_namespace_path_prefix(
|
||||
self, neo4j_client, qdrant_client, wikijs_client, paperless_client
|
||||
):
|
||||
await stats(
|
||||
user="jpmschweitzer",
|
||||
neo4j=neo4j_client,
|
||||
qdrant=qdrant_client,
|
||||
wikijs=wikijs_client,
|
||||
paperless=paperless_client,
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
wikijs_client.list_all_pages.assert_awaited_once_with(
|
||||
path_prefix="users/jpmschweitzer"
|
||||
)
|
||||
|
||||
async def test_reports_page_count(
|
||||
self, neo4j_client, qdrant_client, wikijs_client, paperless_client
|
||||
):
|
||||
response = await stats(
|
||||
user="jpmschweitzer",
|
||||
neo4j=neo4j_client,
|
||||
qdrant=qdrant_client,
|
||||
wikijs=wikijs_client,
|
||||
paperless=paperless_client,
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
assert response.wiki_pages == 138
|
||||
|
||||
async def test_wiki_failure_degrades_to_zero(
|
||||
self, neo4j_client, qdrant_client, wikijs_client, paperless_client
|
||||
):
|
||||
wikijs_client.list_all_pages = AsyncMock(side_effect=RuntimeError("wiki down"))
|
||||
|
||||
response = await stats(
|
||||
user="jpmschweitzer",
|
||||
neo4j=neo4j_client,
|
||||
qdrant=qdrant_client,
|
||||
wikijs=wikijs_client,
|
||||
paperless=paperless_client,
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
assert response.wiki_pages == 0
|
||||
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
Unit tests for WikiJSClient page listing and pagination (offline).
|
||||
|
||||
Wiki.js 2.x `pages.list` supports only a `limit` argument (no offset), so
|
||||
exhaustive listing works by growing the limit until the API returns fewer
|
||||
pages than requested. These tests mock the GraphQL layer.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
|
||||
|
||||
def make_pages(count_other: int, count_user: int):
|
||||
"""Build a fake TITLE-ordered page listing: 'other/' pages sort before 'users/jp/' pages."""
|
||||
pages = [
|
||||
{"id": i, "path": f"other/p{i:03d}", "title": f"A{i:03d}", "tags": []}
|
||||
for i in range(count_other)
|
||||
]
|
||||
pages += [
|
||||
{"id": count_other + i, "path": f"users/jp/p{i:03d}", "title": f"Z{i:03d}", "tags": ["projects"]}
|
||||
for i in range(count_user)
|
||||
]
|
||||
return pages
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""WikiJSClient with a mocked GraphQL layer serving 138 pages (100 other + 38 user)."""
|
||||
client = WikiJSClient("http://wiki.test", "")
|
||||
client._all_pages = make_pages(100, 38)
|
||||
client._requested_limits = []
|
||||
|
||||
async def fake_execute(query, variables=None):
|
||||
limit = variables["limit"]
|
||||
client._requested_limits.append(limit)
|
||||
return {"pages": {"list": client._all_pages[:limit]}}
|
||||
|
||||
client._execute_query = fake_execute
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestListAllPagesPagination:
|
||||
"""list_all_pages must exhaust the listing via a limit-growth loop."""
|
||||
|
||||
async def test_grows_limit_until_exhausted(self, client):
|
||||
pages = await client.list_all_pages(batch_size=50)
|
||||
|
||||
# 50 -> full batch, 100 -> full batch, 200 -> 138 < 200 done
|
||||
assert client._requested_limits == [50, 100, 200]
|
||||
assert len(pages) == 138
|
||||
|
||||
async def test_single_call_when_first_batch_not_full(self, client):
|
||||
pages = await client.list_all_pages(batch_size=500)
|
||||
|
||||
assert client._requested_limits == [500]
|
||||
assert len(pages) == 138
|
||||
|
||||
async def test_path_prefix_filter_after_exhaustion(self, client):
|
||||
"""All 38 user pages are returned even though they sort last (beyond batch_size)."""
|
||||
pages = await client.list_all_pages(path_prefix="users/jp", batch_size=50)
|
||||
|
||||
assert len(pages) == 38
|
||||
assert all(p["path"].startswith("users/jp") for p in pages)
|
||||
|
||||
async def test_empty_wiki(self, client):
|
||||
client._all_pages = []
|
||||
|
||||
pages = await client.list_all_pages(batch_size=100)
|
||||
|
||||
assert pages == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestListPagesLimitAfterFilter:
|
||||
"""list_pages must apply `limit` AFTER client-side filters, not before."""
|
||||
|
||||
async def test_prefix_filter_with_small_limit(self, client):
|
||||
"""Old defect: API limit=5 returned 5 'other/' pages, filter dropped all -> 0 results."""
|
||||
pages = await client.list_pages(path_prefix="users/jp", limit=5)
|
||||
|
||||
assert len(pages) == 5
|
||||
assert all(p["path"].startswith("users/jp") for p in pages)
|
||||
|
||||
async def test_tag_filter_with_small_limit(self, client):
|
||||
pages = await client.list_pages(tags=["projects"], limit=10)
|
||||
|
||||
assert len(pages) == 10
|
||||
assert all("projects" in p["tags"] for p in pages)
|
||||
|
||||
async def test_unfiltered_passes_limit_to_api(self, client):
|
||||
pages = await client.list_pages(limit=7)
|
||||
|
||||
assert client._requested_limits == [7]
|
||||
assert len(pages) == 7
|
||||
|
||||
async def test_tags_normalized_to_list(self, client):
|
||||
client._all_pages = [{"id": 1, "path": "home", "title": "Home", "tags": None}]
|
||||
|
||||
pages = await client.list_pages(limit=10)
|
||||
|
||||
assert pages[0]["tags"] == []
|
||||
Reference in New Issue
Block a user