Initial commit: library-desk service extraction from portainer-core

tests/test_integration.py

@@ -0,0 +1,334 @@
+"""Integration tests for Library Desk service clients.
+
+These tests require actual service connectivity:
+- Neo4j running at bolt://neo4j:7687
+- Qdrant running at http://qdrant:6333
+- Wiki.js running at http://wiki:3000
+- SearXNG running at http://searxng:8080
+- Ollama running at http://ollama:11434
+
+Run with: pytest tests/test_integration.py -v
+"""
+
+import pytest
+import pytest_asyncio
+from typing import AsyncGenerator
+
+from src.clients.neo4j_client import Neo4jClient
+from src.clients.qdrant_client import QdrantClientWrapper
+from src.clients.wikijs_client import WikiJSClient
+from src.clients.searxng_client import SearXNGClient
+from src.clients.ollama_client import OllamaClient
+from src.jobs.job_manager import JobManager, JobType, JobStatus
+from src.config import get_settings
+
+
+@pytest.fixture
+def settings():
+    """Get application settings."""
+    return get_settings()
+
+
+@pytest_asyncio.fixture
+async def neo4j_client(settings) -> AsyncGenerator[Neo4jClient, None]:
+    """Get connected Neo4j client."""
+    client = Neo4jClient(
+        uri=settings.neo4j_uri,
+        user=settings.neo4j_user,
+        password=settings.neo4j_password
+    )
+    await client.connect()
+    yield client
+    await client.close()
+
+
+@pytest.fixture
+def qdrant_client(settings) -> QdrantClientWrapper:
+    """Get Qdrant client."""
+    return QdrantClientWrapper(url=settings.qdrant_url)
+
+
+@pytest_asyncio.fixture
+async def wikijs_client(settings) -> AsyncGenerator[WikiJSClient, None]:
+    """Get Wiki.js client."""
+    client = WikiJSClient(
+        base_url=settings.wikijs_url,
+        api_key=settings.wikijs_api_key
+    )
+    yield client
+    await client.close()
+
+
+@pytest_asyncio.fixture
+async def searxng_client(settings) -> AsyncGenerator[SearXNGClient, None]:
+    """Get SearXNG client."""
+    client = SearXNGClient(base_url=settings.searxng_url)
+    yield client
+    await client.close()
+
+
+@pytest_asyncio.fixture
+async def ollama_client(settings) -> AsyncGenerator[OllamaClient, None]:
+    """Get Ollama client."""
+    client = OllamaClient(
+        base_url=settings.ollama_url,
+        model=settings.ollama_model
+    )
+    yield client
+    await client.close()
+
+
+@pytest_asyncio.fixture
+async def job_manager(settings) -> AsyncGenerator[JobManager, None]:
+    """Get job manager."""
+    manager = JobManager(redis_url=settings.redis_url)
+    await manager.connect()
+    yield manager
+    await manager.close()
+
+
+class TestNeo4jIntegration:
+    """Test Neo4j connectivity and basic operations."""
+
+    @pytest.mark.asyncio
+    async def test_connection(self, neo4j_client):
+        """Test Neo4j connection."""
+        result = await neo4j_client.execute_query("RETURN 1 as test", {})
+        assert result
+        assert result[0]["test"] == 1
+
+    @pytest.mark.asyncio
+    async def test_create_and_get_document(self, neo4j_client, test_user):
+        """Test creating and retrieving a document node."""
+        doc_id = "test_doc_integration"
+
+        # Create document
+        doc = await neo4j_client.create_document_node(
+            user=test_user,
+            doc_id=doc_id,
+            properties={
+                "title": "Integration Test Document",
+                "source": "test"
+            }
+        )
+        assert doc is not None
+        assert doc["id"] == doc_id
+
+        # Retrieve document
+        retrieved = await neo4j_client.get_document_node(test_user, doc_id)
+        assert retrieved is not None
+        assert retrieved["id"] == doc_id
+        assert retrieved["title"] == "Integration Test Document"
+
+        # Cleanup
+        deleted = await neo4j_client.delete_document_node(test_user, doc_id)
+        assert deleted is True
+
+
+class TestQdrantIntegration:
+    """Test Qdrant connectivity and basic operations."""
+
+    @pytest.mark.asyncio
+    async def test_collection_creation(self, qdrant_client, test_user):
+        """Test creating a collection."""
+        await qdrant_client.ensure_collection(test_user)
+
+        collection_name = qdrant_client.get_collection_name(test_user)
+        collections = qdrant_client.client.get_collections()
+        collection_names = [c.name for c in collections.collections]
+
+        assert collection_name in collection_names
+
+    @pytest.mark.asyncio
+    async def test_upsert_and_search(self, qdrant_client, test_user):
+        """Test upserting and searching chunks."""
+        await qdrant_client.ensure_collection(test_user)
+
+        # Create test chunks with 768-dimensional embeddings
+        chunks = [
+            {"content": "Test chunk 1", "metadata": {}},
+            {"content": "Test chunk 2", "metadata": {}}
+        ]
+        embeddings = [
+            [0.1] * 768,
+            [0.2] * 768
+        ]
+
+        # Upsert
+        count = await qdrant_client.upsert_document_chunks(
+            user=test_user,
+            doc_id="test_doc_qdrant",
+            chunks=chunks,
+            embeddings=embeddings
+        )
+        assert count == 2
+
+        # Search
+        results = await qdrant_client.search(
+            user=test_user,
+            query_vector=[0.1] * 768,
+            limit=5,
+            score_threshold=0.0
+        )
+        assert len(results) > 0
+
+        # Cleanup
+        deleted = await qdrant_client.delete_document(test_user, "test_doc_qdrant")
+        assert deleted is True
+
+
+class TestWikiJSIntegration:
+    """Test Wiki.js connectivity and basic operations."""
+
+    @pytest.mark.asyncio
+    async def test_list_pages(self, wikijs_client):
+        """Test listing pages."""
+        pages = await wikijs_client.list_pages(limit=10)
+        assert isinstance(pages, list)
+        # May be empty if wiki is new
+
+    @pytest.mark.asyncio
+    async def test_list_all_pages(self, wikijs_client):
+        """Test listing all pages with pagination support."""
+        pages = await wikijs_client.list_all_pages(path_prefix="users/")
+        assert isinstance(pages, list)
+        # Verify each page has expected fields
+        for page in pages[:5]:  # Check first 5
+            assert "id" in page
+            assert "path" in page
+            assert "title" in page
+
+    @pytest.mark.asyncio
+    async def test_get_taxonomy_structure(self, wikijs_client):
+        """Test getting taxonomy structure for a user."""
+        taxonomy = await wikijs_client.get_taxonomy_structure("users/jpmschweitzer")
+        assert isinstance(taxonomy, dict)
+        # Each key should be a category, value should be list of subcategories
+        for category, subcategories in taxonomy.items():
+            assert isinstance(category, str)
+            assert isinstance(subcategories, list)
+
+    @pytest.mark.asyncio
+    async def test_search_pages(self, wikijs_client):
+        """Test searching pages."""
+        results = await wikijs_client.search_pages("test")
+        assert isinstance(results, list)
+
+
+class TestSearXNGIntegration:
+    """Test SearXNG connectivity and search."""
+
+    @pytest.mark.asyncio
+    async def test_general_search(self, searxng_client):
+        """Test general web search."""
+        results = await searxng_client.search_general("python programming", limit=5)
+        assert isinstance(results, list)
+        if results:
+            assert "title" in results[0]
+            assert "url" in results[0]
+
+    @pytest.mark.asyncio
+    async def test_search_with_suggestions(self, searxng_client):
+        """Test getting search suggestions."""
+        suggestions = await searxng_client.get_suggestions("pytho")
+        assert isinstance(suggestions, list)
+
+
+class TestOllamaIntegration:
+    """Test Ollama connectivity and embeddings."""
+
+    @pytest.mark.asyncio
+    async def test_health_check(self, ollama_client):
+        """Test Ollama health check."""
+        is_healthy = await ollama_client.health_check()
+        # May be False if model not pulled
+        assert isinstance(is_healthy, bool)
+
+    @pytest.mark.asyncio
+    async def test_list_models(self, ollama_client):
+        """Test listing available models."""
+        models = await ollama_client.list_models()
+        assert isinstance(models, list)
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        True,  # Skip by default as embeddings can be slow
+        reason="Embedding generation is slow - enable manually if needed"
+    )
+    async def test_generate_embedding(self, ollama_client):
+        """Test generating a single embedding."""
+        embedding = await ollama_client.embed("test text")
+        if embedding:
+            assert isinstance(embedding, list)
+            assert len(embedding) > 0
+
+
+class TestJobManagerIntegration:
+    """Test Job Manager with Redis."""
+
+    @pytest.mark.asyncio
+    async def test_create_and_get_job(self, job_manager, test_user):
+        """Test creating and retrieving a job."""
+        # Create job
+        job_id = await job_manager.create_job(
+            job_type=JobType.DOCUMENT_INGESTION,
+            user=test_user,
+            parameters={"doc_url": "https://example.com/doc.pdf"}
+        )
+        assert job_id
+
+        # Get job
+        job = await job_manager.get_job(job_id)
+        assert job is not None
+        assert job["job_id"] == job_id
+        assert job["user"] == test_user
+        assert job["status"] == JobStatus.QUEUED.value
+
+        # Update job
+        await job_manager.update_job_status(
+            job_id,
+            JobStatus.PROCESSING,
+            progress=50
+        )
+
+        updated_job = await job_manager.get_job(job_id)
+        assert updated_job["status"] == JobStatus.PROCESSING.value
+        assert updated_job["progress"] == 50
+
+        # Cleanup
+        deleted = await job_manager.delete_job(job_id, test_user)
+        assert deleted is True
+
+
+class TestDependencyInjection:
+    """Test dependency injection and lifecycle management."""
+
+    @pytest.mark.asyncio
+    async def test_startup_clients(self):
+        """Test client startup."""
+        from src.core.dependencies import startup_clients
+
+        # Should not raise exceptions
+        await startup_clients()
+
+    @pytest.mark.asyncio
+    async def test_check_service_health(self):
+        """Test health check for all services."""
+        from src.core.dependencies import check_service_health
+
+        health = await check_service_health()
+
+        assert isinstance(health, dict)
+        assert "neo4j" in health
+        assert "qdrant" in health
+        assert "wikijs" in health
+        assert "searxng" in health
+        assert "ollama" in health
+
+    @pytest.mark.asyncio
+    async def test_shutdown_clients(self):
+        """Test client shutdown."""
+        from src.core.dependencies import shutdown_clients
+
+        # Should not raise exceptions
+        await shutdown_clients()