diff --git a/CHANGELOG.md b/CHANGELOG.md index eb65115..7dbb067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Degradation signaling** - `HybridRAGResponse` now includes `source_status` (per-leg `'ok'`/`'failed'`/`'disabled'` for vector, graph, web, volatile, documents) and `degraded` (true when any enabled leg failed). Retrieval legs report errors instead of silently swallowing them; failed legs are logged at WARNING. Both fields are additive and optional, so clients that ignore them are unaffected. +- **Offline unit tests** - New mock-based tests (no live services) for model-name resolution under the env collision, per-leg failure signaling, the `/stats` page-count prefix, LLM-call timeouts, and Wiki.js listing pagination. ### Fixed diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..681e983 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,62 @@ +""" +Unit tests for application settings (offline). + +Covers the OLLAMA_MODEL env collision: the deployed container sets +OLLAMA_MODEL=nomic-embed-text for embeddings, which must NOT shadow the +generation model setting (ollama_llm_model / OLLAMA_LLM_MODEL). +""" + +import pytest + +from src.config import Settings + +# Required fields so Settings can be constructed without a .env file +REQUIRED = { + "library_api_key": "test-key", + "neo4j_password": "test-pass", + "wikijs_db_password": "test-pass", +} + + +@pytest.mark.unit +class TestOllamaModelResolution: + """Generation model resolution must be immune to the OLLAMA_MODEL env var.""" + + def test_ollama_model_env_does_not_shadow_generation_model(self, monkeypatch): + """The container env OLLAMA_MODEL (embedding model) must not leak into ollama_llm_model.""" + monkeypatch.setenv("OLLAMA_MODEL", "nomic-embed-text") + + settings = Settings(_env_file=None, **REQUIRED) + + assert settings.ollama_llm_model == "gemma4:e2b" + assert settings.ollama_llm_model != "nomic-embed-text" + + def test_generation_model_default(self, monkeypatch): + monkeypatch.delenv("OLLAMA_LLM_MODEL", raising=False) + + settings = Settings(_env_file=None, **REQUIRED) + + assert settings.ollama_llm_model == "gemma4:e2b" + + def test_generation_model_from_dedicated_env_var(self, monkeypatch): + """OLLAMA_LLM_MODEL is the dedicated env var for the generation model.""" + monkeypatch.setenv("OLLAMA_MODEL", "nomic-embed-text") + monkeypatch.setenv("OLLAMA_LLM_MODEL", "mistral-nemo:latest") + + settings = Settings(_env_file=None, **REQUIRED) + + assert settings.ollama_llm_model == "mistral-nemo:latest" + + def test_embedding_model_setting_untouched(self, monkeypatch): + """The embedding model keeps its own setting and env var.""" + monkeypatch.setenv("OLLAMA_EMBEDDING_MODEL", "nomic-embed-text") + + settings = Settings(_env_file=None, **REQUIRED) + + assert settings.ollama_embedding_model == "nomic-embed-text" + + def test_legacy_setting_name_removed(self): + """The old ollama_model attribute must be gone so nothing binds to OLLAMA_MODEL.""" + settings = Settings(_env_file=None, **REQUIRED) + + assert not hasattr(settings, "ollama_model") diff --git a/tests/test_hybrid_rag_degradation.py b/tests/test_hybrid_rag_degradation.py new file mode 100644 index 0000000..9de1c29 --- /dev/null +++ b/tests/test_hybrid_rag_degradation.py @@ -0,0 +1,231 @@ +""" +Unit tests for HybridRAG degradation signaling (offline, all clients mocked). + +Covers: +- Per-leg failure -> source_status reports 'failed', degraded=True +- Disabled legs -> 'disabled', do not trigger degraded +- Healthy legs -> 'ok', degraded=False +- Reranker model resolution from settings.ollama_llm_model +- Phase 0 keyword-extraction timeout falls back gracefully +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import src.services.hybrid_rag_service as hybrid_rag_module +from src.services.hybrid_rag_service import HybridRAGService +from src.models.hybrid_rag import HybridRAGConfig + + +@pytest.fixture +def settings(): + settings = MagicMock() + settings.ollama_llm_model = "gemma4:e2b" + settings.vector_similarity_threshold = 0.7 + return settings + + +@pytest.fixture +def vector_service(): + """Vector service returning a single wiki hit.""" + vector = MagicMock() + hit = MagicMock() + hit.page_id = 1 + hit.page_title = "Test Page" + hit.content = "Test content" + hit.page_path = "users/jp/test" + hit.score = 0.9 + response = MagicMock() + response.results = [hit] + vector.search = AsyncMock(return_value=response) + return vector + + +@pytest.fixture +def graph_service(): + graph = MagicMock() + graph.search_documents = AsyncMock(return_value=[]) + graph.get_related_documents = AsyncMock(return_value=[]) + graph.neo4j.execute_query = AsyncMock(return_value=[{"id": "search-1"}]) + return graph + + +@pytest.fixture +def searxng_client(): + searxng = MagicMock() + searxng.search_general = AsyncMock(return_value=[]) + return searxng + + +@pytest.fixture +def ollama_client(): + ollama = MagicMock() + ollama.generate_text = AsyncMock( + return_value='{"core_keywords": ["test"], "synonyms": {}}' + ) + return ollama + + +@pytest.fixture +def content_extractor(): + extractor = MagicMock() + extractor.extract_batch = AsyncMock(return_value=[]) + return extractor + + +@pytest.fixture +def service(settings, vector_service, graph_service, searxng_client, ollama_client, content_extractor): + return HybridRAGService( + vector_service=vector_service, + graph_service=graph_service, + searxng_client=searxng_client, + ollama_client=ollama_client, + content_extractor=content_extractor, + settings=settings, + volatile_service=None, + ) + + +def make_config(**overrides): + """Config with documents disabled (leg needs a real Qdrant client) and fast phases.""" + defaults = { + "enable_documents": False, + "enable_volatile": False, + "enable_reranking": False, + "enable_enrichment": False, + } + defaults.update(overrides) + return HybridRAGConfig(**defaults) + + +@pytest.mark.unit +class TestSourceStatus: + """source_status must report every leg as ok/failed/disabled.""" + + async def test_all_enabled_legs_ok(self, service): + response = await service.search("test query", "jp", make_config()) + + assert response.source_status == { + "vector": "ok", + "graph": "ok", + "web": "ok", + "volatile": "disabled", + "documents": "disabled", + } + assert response.degraded is False + + async def test_failed_leg_reported_and_degraded(self, service, graph_service): + graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down")) + + response = await service.search("test query", "jp", make_config()) + + assert response.source_status["graph"] == "failed" + assert response.source_status["vector"] == "ok" + assert response.degraded is True + + async def test_failed_leg_still_contributes_no_results(self, service, graph_service): + """Existing behavior preserved: failure -> empty leg, other legs still work.""" + graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down")) + + response = await service.search("test query", "jp", make_config()) + + # The vector hit still comes through + assert response.total_results == 1 + assert response.results[0].page_id == 1 + + async def test_multiple_failures(self, service, graph_service, searxng_client): + graph_service.search_documents = AsyncMock(side_effect=RuntimeError("neo4j down")) + searxng_client.search_general = AsyncMock(side_effect=OSError("searxng unreachable")) + + response = await service.search("test query", "jp", make_config()) + + assert response.source_status["graph"] == "failed" + assert response.source_status["web"] == "failed" + assert response.degraded is True + + async def test_disabled_legs_do_not_degrade(self, service): + config = make_config(enable_graph=False, enable_web=False) + + response = await service.search("test query", "jp", config) + + assert response.source_status["graph"] == "disabled" + assert response.source_status["web"] == "disabled" + assert response.degraded is False + + async def test_volatile_without_service_is_disabled(self, service): + """enable_volatile=True but no volatile service wired -> disabled, not failed.""" + response = await service.search("test query", "jp", make_config(enable_volatile=True)) + + assert response.source_status["volatile"] == "disabled" + assert response.degraded is False + + async def test_status_fields_serialized(self, service, graph_service): + """Contract: fields present in the serialized response for tatlock to parse.""" + graph_service.search_documents = AsyncMock(side_effect=RuntimeError("boom")) + + response = await service.search("test query", "jp", make_config()) + payload = response.model_dump() + + assert set(payload["source_status"].keys()) == {"vector", "graph", "web", "volatile", "documents"} + assert payload["degraded"] is True + + +@pytest.mark.unit +class TestModelResolution: + """The reranker/keyword model must come from settings.ollama_llm_model.""" + + def test_reranker_model_from_llm_setting(self, service): + assert service.reranker_model == "gemma4:e2b" + + async def test_generation_calls_use_llm_model(self, service, ollama_client): + await service.search("test query", "jp", make_config()) + + # Phase 0 keyword extraction ran with the generation model + assert ollama_client.generate_text.await_count >= 1 + for call in ollama_client.generate_text.await_args_list: + assert call.kwargs["model"] == "gemma4:e2b" + + +@pytest.mark.unit +class TestLLMTimeout: + """A hung LLM call must not gate retrieval: 12s wait_for with fallback.""" + + async def test_keyword_extraction_timeout_falls_back(self, service, ollama_client, monkeypatch): + monkeypatch.setattr(hybrid_rag_module, "LLM_CALL_TIMEOUT_SECONDS", 0.05) + + async def hang(*args, **kwargs): + await asyncio.sleep(5) + + ollama_client.generate_text = AsyncMock(side_effect=hang) + + response = await service.search("test query", "jp", make_config()) + + # Fallback: raw query words as keywords, retrieval still ran + assert response.keywords.core_keywords == ["test", "query"] + assert response.source_status["vector"] == "ok" + + async def test_rerank_timeout_keeps_rrf_order(self, service, ollama_client, monkeypatch, searxng_client, content_extractor): + monkeypatch.setattr(hybrid_rag_module, "LLM_CALL_TIMEOUT_SECONDS", 0.05) + + # Two web results so re-ranking actually runs (needs > 1 result) + searxng_client.search_general = AsyncMock(return_value=[ + {"url": "http://a.test", "title": "A", "content": "a"}, + {"url": "http://b.test", "title": "B", "content": "b"}, + ]) + + keyword_json = '{"core_keywords": ["test"], "synonyms": {}}' + + async def generate(prompt, **kwargs): + if "Rank these documents" in prompt: + await asyncio.sleep(5) # Hang only the re-rank call + return keyword_json + + ollama_client.generate_text = AsyncMock(side_effect=generate) + + response = await service.search("test query", "jp", make_config(enable_reranking=True)) + + # RRF order preserved despite the hung re-rank call + assert response.total_results >= 2 + assert response.degraded is False diff --git a/tests/test_stats.py b/tests/test_stats.py new file mode 100644 index 0000000..75c0c71 --- /dev/null +++ b/tests/test_stats.py @@ -0,0 +1,99 @@ +""" +Unit tests for the /stats endpoint (offline, all clients mocked). + +Covers the wiki page count fix: the endpoint must count pages under the +user namespace ("users/{user}"), not pass the bare user name as prefix +(which matched nothing and always reported 0 pages). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from src.main import stats + + +@pytest.fixture +def neo4j_client(): + neo4j = MagicMock() + neo4j.execute_query = AsyncMock(return_value=[{"count": 5}]) + return neo4j + + +@pytest.fixture +def qdrant_client(): + qdrant = MagicMock() + qdrant.list_collections = AsyncMock(return_value=[ + {"name": "user_jp", "vectors_count": 42} + ]) + return qdrant + + +@pytest.fixture +def wikijs_client(): + wikijs = MagicMock() + wikijs.list_all_pages = AsyncMock(return_value=[ + {"id": i, "path": f"users/jpmschweitzer/p{i}"} for i in range(138) + ]) + return wikijs + + +@pytest.fixture +def paperless_client(): + paperless = MagicMock() + paperless.list_documents = AsyncMock(return_value={"count": 3}) + paperless.list_tags = AsyncMock(return_value=[]) + paperless.list_correspondents = AsyncMock(return_value=[]) + paperless.list_document_types = AsyncMock(return_value=[]) + return paperless + + +@pytest.mark.unit +class TestStatsWikiPageCount: + """/stats must scope the wiki page count to the user namespace.""" + + async def test_uses_user_namespace_path_prefix( + self, neo4j_client, qdrant_client, wikijs_client, paperless_client + ): + await stats( + user="jpmschweitzer", + neo4j=neo4j_client, + qdrant=qdrant_client, + wikijs=wikijs_client, + paperless=paperless_client, + api_key="test-key", + ) + + wikijs_client.list_all_pages.assert_awaited_once_with( + path_prefix="users/jpmschweitzer" + ) + + async def test_reports_page_count( + self, neo4j_client, qdrant_client, wikijs_client, paperless_client + ): + response = await stats( + user="jpmschweitzer", + neo4j=neo4j_client, + qdrant=qdrant_client, + wikijs=wikijs_client, + paperless=paperless_client, + api_key="test-key", + ) + + assert response.wiki_pages == 138 + + async def test_wiki_failure_degrades_to_zero( + self, neo4j_client, qdrant_client, wikijs_client, paperless_client + ): + wikijs_client.list_all_pages = AsyncMock(side_effect=RuntimeError("wiki down")) + + response = await stats( + user="jpmschweitzer", + neo4j=neo4j_client, + qdrant=qdrant_client, + wikijs=wikijs_client, + paperless=paperless_client, + api_key="test-key", + ) + + assert response.wiki_pages == 0 diff --git a/tests/test_wikijs_pagination.py b/tests/test_wikijs_pagination.py new file mode 100644 index 0000000..30fd412 --- /dev/null +++ b/tests/test_wikijs_pagination.py @@ -0,0 +1,103 @@ +""" +Unit tests for WikiJSClient page listing and pagination (offline). + +Wiki.js 2.x `pages.list` supports only a `limit` argument (no offset), so +exhaustive listing works by growing the limit until the API returns fewer +pages than requested. These tests mock the GraphQL layer. +""" + +import pytest + +from src.clients.wikijs_client import WikiJSClient + + +def make_pages(count_other: int, count_user: int): + """Build a fake TITLE-ordered page listing: 'other/' pages sort before 'users/jp/' pages.""" + pages = [ + {"id": i, "path": f"other/p{i:03d}", "title": f"A{i:03d}", "tags": []} + for i in range(count_other) + ] + pages += [ + {"id": count_other + i, "path": f"users/jp/p{i:03d}", "title": f"Z{i:03d}", "tags": ["projects"]} + for i in range(count_user) + ] + return pages + + +@pytest.fixture +def client(): + """WikiJSClient with a mocked GraphQL layer serving 138 pages (100 other + 38 user).""" + client = WikiJSClient("http://wiki.test", "") + client._all_pages = make_pages(100, 38) + client._requested_limits = [] + + async def fake_execute(query, variables=None): + limit = variables["limit"] + client._requested_limits.append(limit) + return {"pages": {"list": client._all_pages[:limit]}} + + client._execute_query = fake_execute + return client + + +@pytest.mark.unit +class TestListAllPagesPagination: + """list_all_pages must exhaust the listing via a limit-growth loop.""" + + async def test_grows_limit_until_exhausted(self, client): + pages = await client.list_all_pages(batch_size=50) + + # 50 -> full batch, 100 -> full batch, 200 -> 138 < 200 done + assert client._requested_limits == [50, 100, 200] + assert len(pages) == 138 + + async def test_single_call_when_first_batch_not_full(self, client): + pages = await client.list_all_pages(batch_size=500) + + assert client._requested_limits == [500] + assert len(pages) == 138 + + async def test_path_prefix_filter_after_exhaustion(self, client): + """All 38 user pages are returned even though they sort last (beyond batch_size).""" + pages = await client.list_all_pages(path_prefix="users/jp", batch_size=50) + + assert len(pages) == 38 + assert all(p["path"].startswith("users/jp") for p in pages) + + async def test_empty_wiki(self, client): + client._all_pages = [] + + pages = await client.list_all_pages(batch_size=100) + + assert pages == [] + + +@pytest.mark.unit +class TestListPagesLimitAfterFilter: + """list_pages must apply `limit` AFTER client-side filters, not before.""" + + async def test_prefix_filter_with_small_limit(self, client): + """Old defect: API limit=5 returned 5 'other/' pages, filter dropped all -> 0 results.""" + pages = await client.list_pages(path_prefix="users/jp", limit=5) + + assert len(pages) == 5 + assert all(p["path"].startswith("users/jp") for p in pages) + + async def test_tag_filter_with_small_limit(self, client): + pages = await client.list_pages(tags=["projects"], limit=10) + + assert len(pages) == 10 + assert all("projects" in p["tags"] for p in pages) + + async def test_unfiltered_passes_limit_to_api(self, client): + pages = await client.list_pages(limit=7) + + assert client._requested_limits == [7] + assert len(pages) == 7 + + async def test_tags_normalized_to_list(self, client): + client._all_pages = [{"id": 1, "path": "home", "title": "Home", "tags": None}] + + pages = await client.list_pages(limit=10) + + assert pages[0]["tags"] == []