jpmschweitzerandClaude Fable 5
406143e7ae
Every vector call ran on the sync QdrantClient inside async wrapper methods, blocking the FastAPI event loop per Qdrant round-trip. The wrapper now holds an AsyncQdrantClient (timeout via QDRANT_TIMEOUT, default 30s) and awaits all client calls; the wrapper API is unchanged. Call sites off the wrapper were fixed too: the HybridRAG document leg now uses the async search_vectors wrapper instead of the deprecated raw client.search (also fixing its call to the nonexistent ollama.embed_text which made the leg permanently report 'failed'), the health check awaits get_collections, and document_sync's raw delete/upsert calls are awaited (routed through wrappers in the next commit). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
352 lines
11 KiB
Python
352 lines
11 KiB
Python
"""Integration tests for Library Desk service clients.
|
|
|
|
These tests require actual service connectivity:
|
|
- Neo4j running at bolt://neo4j:7687
|
|
- Qdrant running at http://qdrant:6333
|
|
- Wiki.js running at http://wiki:3000
|
|
- SearXNG running at http://searxng:8080
|
|
- Ollama running at http://ollama:11434
|
|
|
|
These tests run against the SHARED production services under the reserved
|
|
test tenant only. They are gated behind the session tenant guard and the
|
|
RUN_INTEGRATION_TESTS=1 environment flag (see tests/conftest.py) and are
|
|
skipped otherwise.
|
|
|
|
Run with:
|
|
RUN_INTEGRATION_TESTS=1 TEST_HOST=<shared-host> \
|
|
.venv/bin/python -m pytest tests/test_integration.py -v
|
|
"""
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from typing import AsyncGenerator
|
|
|
|
from tests.conftest import TEST_TENANT, assert_safe_test_tenant
|
|
|
|
from src.clients.neo4j_client import Neo4jClient
|
|
from src.clients.qdrant_client import QdrantClientWrapper
|
|
from src.clients.wikijs_client import WikiJSClient
|
|
from src.clients.searxng_client import SearXNGClient
|
|
from src.clients.ollama_client import OllamaClient
|
|
from src.jobs.job_manager import JobManager, JobType, JobStatus
|
|
from src.config import get_settings
|
|
|
|
# Only run when the tenant guard passes and integration mode is enabled.
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def settings():
|
|
"""Get application settings."""
|
|
return get_settings()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def neo4j_client(settings, neo4j_test_uri) -> AsyncGenerator[Neo4jClient, None]:
|
|
"""Get connected Neo4j client."""
|
|
client = Neo4jClient(
|
|
uri=neo4j_test_uri,
|
|
user=settings.neo4j_user,
|
|
password=settings.neo4j_password
|
|
)
|
|
await client.connect()
|
|
yield client
|
|
await client.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def qdrant_client(settings, qdrant_test_url) -> QdrantClientWrapper:
|
|
"""Get Qdrant client."""
|
|
return QdrantClientWrapper(url=qdrant_test_url)
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def wikijs_client(wikijs_test_config) -> AsyncGenerator[WikiJSClient, None]:
|
|
"""Get Wiki.js client."""
|
|
client = WikiJSClient(
|
|
base_url=wikijs_test_config["base_url"],
|
|
api_token=wikijs_test_config["api_token"]
|
|
)
|
|
yield client
|
|
await client.close()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def searxng_client(searxng_test_url) -> AsyncGenerator[SearXNGClient, None]:
|
|
"""Get SearXNG client."""
|
|
client = SearXNGClient(base_url=searxng_test_url)
|
|
yield client
|
|
await client.close()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def ollama_client(ollama_test_config) -> AsyncGenerator[OllamaClient, None]:
|
|
"""Get Ollama client for embeddings."""
|
|
client = OllamaClient(
|
|
base_url=ollama_test_config["base_url"],
|
|
model=ollama_test_config["model"]
|
|
)
|
|
yield client
|
|
await client.close()
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def job_manager(redis_test_url) -> AsyncGenerator[JobManager, None]:
|
|
"""Get job manager."""
|
|
manager = JobManager(redis_url=redis_test_url)
|
|
await manager.connect()
|
|
yield manager
|
|
await manager.close()
|
|
|
|
|
|
class TestNeo4jIntegration:
|
|
"""Test Neo4j connectivity and basic operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_connection(self, neo4j_client):
|
|
"""Test Neo4j connection."""
|
|
result = await neo4j_client.execute_query("RETURN 1 as test", {})
|
|
assert result
|
|
assert result[0]["test"] == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_and_get_document(self, neo4j_client, test_user):
|
|
"""Test creating and retrieving a document node."""
|
|
assert_safe_test_tenant(test_user)
|
|
doc_id = "test_doc_integration"
|
|
|
|
# Create document
|
|
doc = await neo4j_client.create_document_node(
|
|
user=test_user,
|
|
doc_id=doc_id,
|
|
properties={
|
|
"title": "Integration Test Document",
|
|
"source": "test"
|
|
}
|
|
)
|
|
assert doc is not None
|
|
assert doc["id"] == doc_id
|
|
|
|
# Retrieve document
|
|
retrieved = await neo4j_client.get_document_node(test_user, doc_id)
|
|
assert retrieved is not None
|
|
assert retrieved["id"] == doc_id
|
|
assert retrieved["title"] == "Integration Test Document"
|
|
|
|
# Cleanup
|
|
deleted = await neo4j_client.delete_document_node(test_user, doc_id)
|
|
assert deleted is True
|
|
|
|
|
|
class TestQdrantIntegration:
|
|
"""Test Qdrant connectivity and basic operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collection_creation(self, qdrant_client, test_user):
|
|
"""Test creating a collection."""
|
|
assert_safe_test_tenant(test_user)
|
|
collection_name = qdrant_client.get_collection_name(test_user)
|
|
await qdrant_client.ensure_collection(collection_name)
|
|
|
|
collections = await qdrant_client.client.get_collections()
|
|
collection_names = [c.name for c in collections.collections]
|
|
|
|
assert collection_name in collection_names
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_and_search(self, qdrant_client, test_user):
|
|
"""Test upserting and searching chunks."""
|
|
assert_safe_test_tenant(test_user)
|
|
await qdrant_client.ensure_collection(
|
|
qdrant_client.get_collection_name(test_user)
|
|
)
|
|
|
|
# Create test chunks with 768-dimensional embeddings
|
|
chunks = [
|
|
{"content": "Test chunk 1", "metadata": {}},
|
|
{"content": "Test chunk 2", "metadata": {}}
|
|
]
|
|
embeddings = [
|
|
[0.1] * 768,
|
|
[0.2] * 768
|
|
]
|
|
|
|
# Upsert
|
|
count = await qdrant_client.upsert_document_chunks(
|
|
user=test_user,
|
|
doc_id="test_doc_qdrant",
|
|
chunks=chunks,
|
|
embeddings=embeddings
|
|
)
|
|
assert count == 2
|
|
|
|
# Search
|
|
results = await qdrant_client.search(
|
|
user=test_user,
|
|
query_vector=[0.1] * 768,
|
|
limit=5,
|
|
score_threshold=0.0
|
|
)
|
|
assert len(results) > 0
|
|
|
|
# Cleanup
|
|
deleted = await qdrant_client.delete_document(test_user, "test_doc_qdrant")
|
|
assert deleted is True
|
|
|
|
|
|
class TestWikiJSIntegration:
|
|
"""Test Wiki.js connectivity and basic operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_pages(self, wikijs_client):
|
|
"""Test listing pages."""
|
|
pages = await wikijs_client.list_pages(limit=10)
|
|
assert isinstance(pages, list)
|
|
# May be empty if wiki is new
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_all_pages(self, wikijs_client):
|
|
"""Test listing all pages with pagination support."""
|
|
pages = await wikijs_client.list_all_pages(path_prefix=f"users/{TEST_TENANT}")
|
|
assert isinstance(pages, list)
|
|
# Verify each page has expected fields
|
|
for page in pages[:5]: # Check first 5
|
|
assert "id" in page
|
|
assert "path" in page
|
|
assert "title" in page
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_taxonomy_structure(self, wikijs_client):
|
|
"""Test getting taxonomy structure for a user."""
|
|
taxonomy = await wikijs_client.get_taxonomy_structure(f"users/{TEST_TENANT}")
|
|
assert isinstance(taxonomy, dict)
|
|
# Each key should be a category, value should be list of subcategories
|
|
for category, subcategories in taxonomy.items():
|
|
assert isinstance(category, str)
|
|
assert isinstance(subcategories, list)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_pages(self, wikijs_client):
|
|
"""Test searching pages."""
|
|
results = await wikijs_client.search_pages("test")
|
|
assert isinstance(results, list)
|
|
|
|
|
|
class TestSearXNGIntegration:
|
|
"""Test SearXNG connectivity and search."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_general_search(self, searxng_client):
|
|
"""Test general web search."""
|
|
results = await searxng_client.search_general("python programming", limit=5)
|
|
assert isinstance(results, list)
|
|
if results:
|
|
assert "title" in results[0]
|
|
assert "url" in results[0]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_with_suggestions(self, searxng_client):
|
|
"""Test getting search suggestions."""
|
|
suggestions = await searxng_client.get_suggestions("pytho")
|
|
assert isinstance(suggestions, list)
|
|
|
|
|
|
class TestOllamaIntegration:
|
|
"""Test Ollama connectivity and embeddings."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_check(self, ollama_client):
|
|
"""Test Ollama health check."""
|
|
is_healthy = await ollama_client.health_check()
|
|
# May be False if model not pulled
|
|
assert isinstance(is_healthy, bool)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_models(self, ollama_client):
|
|
"""Test listing available models."""
|
|
models = await ollama_client.list_models()
|
|
assert isinstance(models, list)
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.skipif(
|
|
True, # Skip by default as embeddings can be slow
|
|
reason="Embedding generation is slow - enable manually if needed"
|
|
)
|
|
async def test_generate_embedding(self, ollama_client):
|
|
"""Test generating a single embedding."""
|
|
embedding = await ollama_client.embed("test text")
|
|
if embedding:
|
|
assert isinstance(embedding, list)
|
|
assert len(embedding) > 0
|
|
|
|
|
|
class TestJobManagerIntegration:
|
|
"""Test Job Manager with Redis."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_and_get_job(self, job_manager, test_user):
|
|
"""Test creating and retrieving a job."""
|
|
# Create job
|
|
job_id = await job_manager.create_job(
|
|
job_type=JobType.DOCUMENT_INGESTION,
|
|
user=test_user,
|
|
parameters={"doc_url": "https://example.com/doc.pdf"}
|
|
)
|
|
assert job_id
|
|
|
|
# Get job
|
|
job = await job_manager.get_job(job_id)
|
|
assert job is not None
|
|
assert job["job_id"] == job_id
|
|
assert job["user"] == test_user
|
|
assert job["status"] == JobStatus.QUEUED.value
|
|
|
|
# Update job
|
|
await job_manager.update_job_status(
|
|
job_id,
|
|
JobStatus.PROCESSING,
|
|
progress=50
|
|
)
|
|
|
|
updated_job = await job_manager.get_job(job_id)
|
|
assert updated_job["status"] == JobStatus.PROCESSING.value
|
|
assert updated_job["progress"] == 50
|
|
|
|
# Cleanup
|
|
deleted = await job_manager.delete_job(job_id, test_user)
|
|
assert deleted is True
|
|
|
|
|
|
class TestDependencyInjection:
|
|
"""Test dependency injection and lifecycle management."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_startup_clients(self):
|
|
"""Test client startup."""
|
|
from src.core.dependencies import startup_clients
|
|
|
|
# Should not raise exceptions
|
|
await startup_clients()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_service_health(self):
|
|
"""Test health check for all services."""
|
|
from src.core.dependencies import check_service_health
|
|
|
|
health = await check_service_health()
|
|
|
|
assert isinstance(health, dict)
|
|
assert "neo4j" in health
|
|
assert "qdrant" in health
|
|
assert "wikijs" in health
|
|
assert "searxng" in health
|
|
assert "ollama" in health
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_shutdown_clients(self):
|
|
"""Test client shutdown."""
|
|
from src.core.dependencies import shutdown_clients
|
|
|
|
# Should not raise exceptions
|
|
await shutdown_clients()
|