Initial commit: library-desk service extraction from portainer-core

src/clients/neo4j_client.py

@@ -0,0 +1,413 @@
+"""
+Neo4j async client for Library Desk.
+
+Provides async Neo4j operations with:
+- Connection pooling via AsyncGraphDatabase
+- Session management with context managers
+- Multi-tenancy support via user labels
+- Automatic retry on transient failures
+"""
+
+from neo4j import AsyncGraphDatabase, AsyncDriver, AsyncSession
+from typing import Optional, List, Dict, Any
+import logging
+
+from src.core.multi_tenancy import get_neo4j_user_label
+
+logger = logging.getLogger(__name__)
+
+
+class Neo4jClient:
+    """
+    Async Neo4j client with connection pooling.
+
+    Features:
+    - Singleton driver instance (expensive to create)
+    - Session-per-request pattern (lightweight)
+    - Automatic transaction retry
+    - Multi-tenancy via user-specific labels
+    """
+
+    def __init__(self, uri: str, user: str, password: str):
+        """
+        Initialize Neo4j client.
+
+        Args:
+            uri: Neo4j Bolt URI (e.g., "bolt://neo4j:7687")
+            user: Neo4j username
+            password: Neo4j password
+        """
+        self.uri = uri
+        self._driver: Optional[AsyncDriver] = None
+        self._auth = (user, password)
+
+    async def connect(self):
+        """
+        Initialize connection pool.
+
+        Should be called once at app startup.
+        Driver handles connection pooling internally.
+        """
+        if not self._driver:
+            self._driver = AsyncGraphDatabase.driver(
+                self.uri,
+                auth=self._auth,
+                max_connection_pool_size=50,
+                connection_timeout=30.0,
+                max_transaction_retry_time=30.0
+            )
+            # Verify connectivity
+            await self._driver.verify_connectivity()
+            logger.info(f"Connected to Neo4j at {self.uri}")
+
+    async def close(self):
+        """
+        Close connection pool.
+
+        Should be called once at app shutdown.
+        """
+        if self._driver:
+            await self._driver.close()
+            self._driver = None
+            logger.info("Closed Neo4j connection")
+
+    async def execute_query(
+        self,
+        cypher: str,
+        parameters: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute Cypher query and return results.
+
+        Args:
+            cypher: Cypher query string
+            parameters: Query parameters
+
+        Returns:
+            List of result records as dictionaries
+
+        Raises:
+            Exception: If driver not initialized or query fails
+        """
+        if not self._driver:
+            await self.connect()
+
+        async with self._driver.session() as session:
+            result = await session.run(cypher, parameters or {})
+            records = await result.data()
+            return records
+
+    async def execute_write(
+        self,
+        cypher: str,
+        parameters: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute write transaction with automatic retry.
+
+        Args:
+            cypher: Cypher query string
+            parameters: Query parameters
+
+        Returns:
+            List of result records as dictionaries
+        """
+        if not self._driver:
+            await self.connect()
+
+        async def write_tx(tx):
+            result = await tx.run(cypher, parameters or {})
+            return await result.data()
+
+        async with self._driver.session() as session:
+            return await session.execute_write(write_tx)
+
+    # Multi-tenancy helpers
+
+    def get_user_label(self, user: str) -> str:
+        """
+        Get Neo4j label for user's documents.
+
+        Args:
+            user: User identifier
+
+        Returns:
+            Neo4j label string (e.g., "User_Jpmschweitzer_Document")
+        """
+        return get_neo4j_user_label(user)
+
+    # Document node operations
+
+    async def create_document_node(
+        self,
+        user: str,
+        doc_id: str,
+        properties: Dict[str, Any]
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Create document node with user label.
+
+        Node structure:
+        (doc:Document:User_{user}_Document {
+            id: "doc_123",
+            source: "github",
+            repository: "anthropic-cookbook",
+            path: "skills/citation/guide.md",
+            title: "Citation Guide",
+            created_at: timestamp(),
+            updated_at: timestamp(),
+            content_hash: "sha256:..."
+        })
+
+        Args:
+            user: User identifier
+            doc_id: Unique document ID
+            properties: Document properties
+
+        Returns:
+            Created node properties or None on failure
+        """
+        user_label = self.get_user_label(user)
+
+        # Ensure required properties
+        properties["id"] = doc_id
+        if "created_at" not in properties:
+            properties["created_at"] = "timestamp()"
+
+        cypher = f"""
+        CREATE (doc:Document:{user_label})
+        SET doc = $properties
+        SET doc.created_at = timestamp()
+        SET doc.updated_at = timestamp()
+        RETURN doc
+        """
+
+        try:
+            result = await self.execute_write(cypher, {"properties": properties})
+            return result[0]["doc"] if result else None
+        except Exception as e:
+            logger.error(f"Failed to create document node: {e}", exc_info=True)
+            return None
+
+    async def get_document_node(
+        self,
+        user: str,
+        doc_id: str
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Get document node by ID.
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+
+        Returns:
+            Document node properties or None if not found
+        """
+        user_label = self.get_user_label(user)
+
+        cypher = f"""
+        MATCH (doc:Document:{user_label} {{id: $doc_id}})
+        RETURN doc
+        """
+
+        try:
+            result = await self.execute_query(cypher, {"doc_id": doc_id})
+            return result[0]["doc"] if result else None
+        except Exception as e:
+            logger.error(f"Failed to get document node: {e}", exc_info=True)
+            return None
+
+    async def delete_document_node(
+        self,
+        user: str,
+        doc_id: str
+    ) -> bool:
+        """
+        Delete document node and all its relationships.
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+
+        Returns:
+            True if deleted, False otherwise
+        """
+        user_label = self.get_user_label(user)
+
+        cypher = f"""
+        MATCH (doc:Document:{user_label} {{id: $doc_id}})
+        DETACH DELETE doc
+        RETURN count(doc) as deleted
+        """
+
+        try:
+            result = await self.execute_write(cypher, {"doc_id": doc_id})
+            return result[0]["deleted"] > 0 if result else False
+        except Exception as e:
+            logger.error(f"Failed to delete document node: {e}", exc_info=True)
+            return False
+
+    async def find_similar_documents(
+        self,
+        user: str,
+        doc_ids: List[str],
+        max_depth: int = 2,
+        limit: int = 20
+    ) -> List[Dict[str, Any]]:
+        """
+        Find documents similar to given docs via graph traversal.
+
+        Uses: Shared concepts, shared entities, citation links.
+
+        Args:
+            user: User identifier
+            doc_ids: List of source document IDs
+            max_depth: Maximum traversal depth
+            limit: Maximum results to return
+
+        Returns:
+            List of similar documents with connection strength
+        """
+        user_label = self.get_user_label(user)
+
+        cypher = f"""
+        MATCH (source:Document:{user_label})
+        WHERE source.id IN $doc_ids
+        MATCH (source)-[*1..{max_depth}]-(related:Document:{user_label})
+        WHERE related.id <> source.id AND NOT related.id IN $doc_ids
+        WITH related, count(*) as connection_strength
+        ORDER BY connection_strength DESC
+        LIMIT $limit
+        RETURN related, connection_strength
+        """
+
+        try:
+            result = await self.execute_query(
+                cypher,
+                {"doc_ids": doc_ids, "limit": limit}
+            )
+            return result
+        except Exception as e:
+            logger.error(f"Failed to find similar documents: {e}", exc_info=True)
+            return []
+
+    async def list_user_documents(
+        self,
+        user: str,
+        limit: int = 100,
+        offset: int = 0
+    ) -> List[Dict[str, Any]]:
+        """
+        List all documents for a user.
+
+        Args:
+            user: User identifier
+            limit: Maximum results to return
+            offset: Number of results to skip
+
+        Returns:
+            List of document nodes
+        """
+        user_label = self.get_user_label(user)
+
+        cypher = f"""
+        MATCH (doc:Document:{user_label})
+        RETURN doc
+        ORDER BY doc.created_at DESC
+        SKIP $offset
+        LIMIT $limit
+        """
+
+        try:
+            result = await self.execute_query(
+                cypher,
+                {"offset": offset, "limit": limit}
+            )
+            return [r["doc"] for r in result]
+        except Exception as e:
+            logger.error(f"Failed to list documents: {e}", exc_info=True)
+            return []
+
+    # Concept/entity operations
+
+    async def create_concept_node(
+        self,
+        concept_name: str,
+        concept_type: str,
+        properties: Optional[Dict[str, Any]] = None
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Create or update concept node.
+
+        Args:
+            concept_name: Concept name
+            concept_type: Concept type (Technique, Tool, Pattern, etc.)
+            properties: Additional properties
+
+        Returns:
+            Concept node properties
+        """
+        cypher = """
+        MERGE (concept:Concept {name: $name})
+        ON CREATE SET
+            concept.type = $type,
+            concept.first_seen = timestamp(),
+            concept.mention_count = 1
+        ON MATCH SET
+            concept.mention_count = concept.mention_count + 1
+        SET concept += $properties
+        RETURN concept
+        """
+
+        try:
+            result = await self.execute_write(
+                cypher,
+                {
+                    "name": concept_name,
+                    "type": concept_type,
+                    "properties": properties or {}
+                }
+            )
+            return result[0]["concept"] if result else None
+        except Exception as e:
+            logger.error(f"Failed to create concept node: {e}", exc_info=True)
+            return None
+
+    async def link_document_to_concept(
+        self,
+        user: str,
+        doc_id: str,
+        concept_name: str
+    ) -> bool:
+        """
+        Create MENTIONS relationship between document and concept.
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+            concept_name: Concept name
+
+        Returns:
+            True if link created
+        """
+        user_label = self.get_user_label(user)
+
+        cypher = f"""
+        MATCH (doc:Document:{user_label} {{id: $doc_id}})
+        MATCH (concept:Concept {{name: $concept_name}})
+        MERGE (doc)-[r:MENTIONS]->(concept)
+        ON CREATE SET r.count = 1
+        ON MATCH SET r.count = r.count + 1
+        RETURN r
+        """
+
+        try:
+            result = await self.execute_write(
+                cypher,
+                {"doc_id": doc_id, "concept_name": concept_name}
+            )
+            return len(result) > 0
+        except Exception as e:
+            logger.error(f"Failed to link document to concept: {e}", exc_info=True)
+            return False

src/clients/ollama_client.py

@@ -0,0 +1,386 @@
+"""
+Ollama client for embeddings generation.
+
+Provides async embedding operations via Ollama API:
+- Text embedding generation
+- Batch embedding support
+- Model management
+"""
+
+import httpx
+from typing import List, Dict, Any, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class OllamaClient:
+    """
+    Ollama API client for embeddings.
+
+    Documentation: https://github.com/ollama/ollama/blob/main/docs/api.md
+    Default model: nomic-embed-text (768-dimensional embeddings)
+    """
+
+    def __init__(self, base_url: str, model: str = "nomic-embed-text"):
+        """
+        Initialize Ollama client.
+
+        Args:
+            base_url: Ollama server URL (e.g., "http://ollama:11434")
+            model: Embedding model name (default: "nomic-embed-text")
+        """
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.embeddings_url = f"{self.base_url}/api/embeddings"
+        self.generate_url = f"{self.base_url}/api/generate"
+        self.tags_url = f"{self.base_url}/api/tags"
+        self.client = httpx.AsyncClient(timeout=120.0)  # Embeddings can be slow
+        logger.info(f"Initialized Ollama client: {base_url}, model: {model}")
+
+    async def close(self):
+        """Close HTTP client"""
+        await self.client.aclose()
+
+    async def embed(self, text: str) -> Optional[List[float]]:
+        """
+        Generate embedding for single text.
+
+        Args:
+            text: Text to embed
+
+        Returns:
+            Embedding vector (768-dimensional for nomic-embed-text) or None on failure
+
+        Example:
+            >>> embedding = await client.embed("Hello world")
+            >>> len(embedding)
+            768
+        """
+        try:
+            payload = {
+                "model": self.model,
+                "prompt": text
+            }
+
+            response = await self.client.post(
+                self.embeddings_url,
+                json=payload
+            )
+            response.raise_for_status()
+            data = response.json()
+
+            embedding = data.get("embedding")
+            if not embedding:
+                logger.error(f"No embedding in response: {data}")
+                return None
+
+            return embedding
+
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error: {e.response.status_code} - {e.response.text}")
+            return None
+        except Exception as e:
+            logger.error(f"Embedding failed: {e}", exc_info=True)
+            return None
+
+    async def embed_batch(
+        self,
+        texts: List[str],
+        show_progress: bool = False
+    ) -> List[Optional[List[float]]]:
+        """
+        Generate embeddings for multiple texts.
+
+        Args:
+            texts: List of texts to embed
+            show_progress: Log progress for large batches
+
+        Returns:
+            List of embedding vectors (same order as input)
+            None entries for texts that failed to embed
+
+        Example:
+            >>> texts = ["Hello", "World", "Test"]
+            >>> embeddings = await client.embed_batch(texts)
+            >>> len(embeddings)
+            3
+        """
+        embeddings = []
+
+        for i, text in enumerate(texts):
+            if show_progress and i % 10 == 0:
+                logger.info(f"Embedding progress: {i}/{len(texts)}")
+
+            embedding = await self.embed(text)
+            embeddings.append(embedding)
+
+        if show_progress:
+            logger.info(f"Embedding complete: {len(embeddings)}/{len(texts)}")
+
+        return embeddings
+
+    async def embed_batch_filtered(
+        self,
+        texts: List[str],
+        show_progress: bool = False
+    ) -> List[List[float]]:
+        """
+        Generate embeddings for multiple texts, filtering out failures.
+
+        Args:
+            texts: List of texts to embed
+            show_progress: Log progress for large batches
+
+        Returns:
+            List of successful embedding vectors (may be shorter than input)
+
+        Example:
+            >>> texts = ["Hello", "World", "Test"]
+            >>> embeddings = await client.embed_batch_filtered(texts)
+            >>> all(e is not None for e in embeddings)
+            True
+        """
+        all_embeddings = await self.embed_batch(texts, show_progress)
+        return [e for e in all_embeddings if e is not None]
+
+    async def embed_documents(
+        self,
+        documents: List[Dict[str, Any]],
+        content_field: str = "content",
+        show_progress: bool = False
+    ) -> List[Dict[str, Any]]:
+        """
+        Embed documents with metadata preservation.
+
+        Args:
+            documents: List of document dictionaries
+            content_field: Field name containing text to embed
+            show_progress: Log progress for large batches
+
+        Returns:
+            List of documents with added "embedding" field
+
+        Example:
+            >>> docs = [
+            ...     {"content": "Hello world", "id": 1},
+            ...     {"content": "Test doc", "id": 2}
+            ... ]
+            >>> embedded = await client.embed_documents(docs)
+            >>> "embedding" in embedded[0]
+            True
+        """
+        texts = [doc.get(content_field, "") for doc in documents]
+        embeddings = await self.embed_batch(texts, show_progress)
+
+        results = []
+        for doc, embedding in zip(documents, embeddings):
+            doc_copy = doc.copy()
+            doc_copy["embedding"] = embedding
+            results.append(doc_copy)
+
+        return results
+
+    async def get_embedding_dimension(self) -> Optional[int]:
+        """
+        Get embedding dimension for current model.
+
+        Returns:
+            Embedding dimension (e.g., 768 for nomic-embed-text) or None on failure
+
+        Example:
+            >>> dim = await client.get_embedding_dimension()
+            >>> dim
+            768
+        """
+        # Generate a test embedding to determine dimension
+        test_embedding = await self.embed("test")
+        if test_embedding:
+            return len(test_embedding)
+        return None
+
+    async def list_models(self) -> List[Dict[str, Any]]:
+        """
+        List available Ollama models.
+
+        Returns:
+            List of model information dictionaries
+
+        Example:
+            >>> models = await client.list_models()
+            >>> any(m["name"] == "nomic-embed-text" for m in models)
+            True
+        """
+        try:
+            response = await self.client.get(self.tags_url)
+            response.raise_for_status()
+            data = response.json()
+            return data.get("models", [])
+        except Exception as e:
+            logger.error(f"Failed to list models: {e}")
+            return []
+
+    async def check_model_available(self, model_name: Optional[str] = None) -> bool:
+        """
+        Check if a model is available.
+
+        Args:
+            model_name: Model name to check (defaults to self.model)
+
+        Returns:
+            True if model is available, False otherwise
+        """
+        check_model = model_name or self.model
+        models = await self.list_models()
+
+        # Check for exact match or match with :latest tag
+        for m in models:
+            name = m.get("name", "")
+            # Exact match
+            if name == check_model:
+                return True
+            # Match without tag (e.g., "nomic-embed-text" matches "nomic-embed-text:latest")
+            if name.startswith(f"{check_model}:"):
+                return True
+
+        return False
+
+    async def generate_text(
+        self,
+        prompt: str,
+        model: Optional[str] = None,
+        stream: bool = False
+    ) -> Optional[str]:
+        """
+        Generate text completion (for non-embedding use cases).
+
+        Args:
+            prompt: Input prompt
+            model: Model name (defaults to self.model)
+            stream: Enable streaming response
+
+        Returns:
+            Generated text or None on failure
+
+        Note: This is primarily for debugging/testing. Use specialized
+        LLM services for production text generation.
+        """
+        try:
+            payload = {
+                "model": model or self.model,
+                "prompt": prompt,
+                "stream": stream
+            }
+
+            response = await self.client.post(
+                self.generate_url,
+                json=payload
+            )
+            response.raise_for_status()
+
+            if stream:
+                # For streaming, return first chunk
+                # Full streaming implementation would need async generator
+                return response.text
+            else:
+                data = response.json()
+                return data.get("response")
+
+        except Exception as e:
+            logger.error(f"Text generation failed: {e}")
+            return None
+
+    async def health_check(self) -> bool:
+        """
+        Check if Ollama server is reachable and model is available.
+
+        Returns:
+            True if healthy, False otherwise
+        """
+        try:
+            # Check server is up and get models in one call
+            response = await self.client.get(self.tags_url, timeout=5.0)
+            response.raise_for_status()
+            data = response.json()
+            models = data.get("models", [])
+
+            # Check model is available
+            check_model = self.model
+            model_found = False
+            for m in models:
+                name = m.get("name", "")
+                if name == check_model or name.startswith(f"{check_model}:"):
+                    model_found = True
+                    break
+
+            if not model_found:
+                logger.warning(f"Model '{self.model}' not found in Ollama")
+                return False
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Health check failed: {e}")
+            return False
+
+    def estimate_tokens(self, text: str) -> int:
+        """
+        Rough estimate of token count for text.
+
+        Uses simple heuristic: ~4 characters per token.
+
+        Args:
+            text: Input text
+
+        Returns:
+            Estimated token count
+        """
+        return len(text) // 4
+
+    def chunk_text_for_embedding(
+        self,
+        text: str,
+        max_tokens: int = 512,
+        overlap: int = 50
+    ) -> List[str]:
+        """
+        Chunk text into segments suitable for embedding.
+
+        Args:
+            text: Input text
+            max_tokens: Maximum tokens per chunk
+            overlap: Token overlap between chunks
+
+        Returns:
+            List of text chunks
+
+        Example:
+            >>> chunks = client.chunk_text_for_embedding(long_text, max_tokens=512)
+            >>> all(client.estimate_tokens(c) <= 512 for c in chunks)
+            True
+        """
+        # Convert tokens to approximate character count
+        max_chars = max_tokens * 4
+        overlap_chars = overlap * 4
+
+        if len(text) <= max_chars:
+            return [text]
+
+        chunks = []
+        start = 0
+
+        while start < len(text):
+            end = start + max_chars
+            chunk = text[start:end]
+
+            # Try to break at sentence boundary
+            if end < len(text):
+                last_period = chunk.rfind(". ")
+                if last_period > max_chars * 0.5:  # Only break if > 50% through chunk
+                    end = start + last_period + 1
+                    chunk = text[start:end]
+
+            chunks.append(chunk.strip())
+            start = end - overlap_chars
+
+        return chunks

src/clients/qdrant_client.py

@@ -0,0 +1,588 @@
+"""
+Qdrant client wrapper for Library Desk.
+
+Provides async vector operations with:
+- Collection-per-user multi-tenancy
+- Document chunk storage with embeddings
+- Semantic search
+- Similarity queries
+"""
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance, VectorParams, PointStruct,
+    Filter, FieldCondition, MatchValue
+)
+from typing import List, Dict, Any, Optional
+import uuid
+import logging
+
+from src.core.multi_tenancy import get_qdrant_collection_name
+
+logger = logging.getLogger(__name__)
+
+
+class QdrantClientWrapper:
+    """
+    Qdrant client wrapper with multi-tenancy support.
+
+    Pattern: Collection per user (from qdrant_memory.py)
+    Each user has isolated vector collection for their documents.
+    """
+
+    def __init__(self, url: str, embedding_dim: int = 768):
+        """
+        Initialize Qdrant client.
+
+        Args:
+            url: Qdrant server URL (e.g., "http://qdrant:6333")
+            embedding_dim: Vector embedding dimension (default 768 for nomic-embed-text)
+        """
+        self.client = QdrantClient(url=url)
+        self.embedding_dim = embedding_dim
+        logger.info(f"Initialized Qdrant client: {url}")
+
+    def get_collection_name(self, user: str) -> str:
+        """
+        Get collection name for user.
+
+        Args:
+            user: User identifier
+
+        Returns:
+            Collection name (e.g., "library_desk_jpmschweitzer")
+        """
+        return get_qdrant_collection_name(user)
+
+    async def ensure_collection(self, collection_name: str):
+        """
+        Create collection if it doesn't exist.
+
+        Args:
+            collection_name: Collection name
+        """
+        try:
+            collections = self.client.get_collections()
+            existing = [c.name for c in collections.collections]
+
+            if collection_name not in existing:
+                logger.info(f"Creating Qdrant collection: {collection_name}")
+                self.client.create_collection(
+                    collection_name=collection_name,
+                    vectors_config=VectorParams(
+                        size=self.embedding_dim,
+                        distance=Distance.COSINE
+                    )
+                )
+                logger.info(f"Created collection: {collection_name}")
+        except Exception as e:
+            logger.error(f"Error ensuring collection: {e}", exc_info=True)
+            raise
+
+    async def collection_exists(self, collection_name: str) -> bool:
+        """
+        Check if collection exists.
+
+        Args:
+            collection_name: Collection name
+
+        Returns:
+            True if collection exists
+        """
+        try:
+            collections = self.client.get_collections()
+            existing = [c.name for c in collections.collections]
+            return collection_name in existing
+        except Exception as e:
+            logger.error(f"Error checking collection: {e}", exc_info=True)
+            return False
+
+    async def upsert_document_chunks(
+        self,
+        user: str,
+        doc_id: str,
+        chunks: List[Dict[str, Any]],
+        embeddings: List[List[float]]
+    ) -> int:
+        """
+        Upsert document chunks with embeddings.
+
+        Point structure:
+        {
+            id: "doc_123_chunk_0",
+            vector: [...],
+            payload: {
+                doc_id: "doc_123",
+                chunk_index: 0,
+                content: "text content",
+                metadata: {...}
+            }
+        }
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+            chunks: List of chunk dictionaries with content
+            embeddings: List of embedding vectors
+
+        Returns:
+            Number of chunks upserted
+
+        Raises:
+            ValueError: If chunks and embeddings length mismatch
+        """
+        if len(chunks) != len(embeddings):
+            raise ValueError(
+                f"Chunks ({len(chunks)}) and embeddings ({len(embeddings)}) length mismatch"
+            )
+
+        collection_name = self.get_collection_name(user)
+        await self.ensure_collection(user)
+
+        points = []
+        for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
+            # Generate deterministic point ID
+            point_id = str(uuid.uuid5(
+                uuid.NAMESPACE_DNS,
+                f"{doc_id}_chunk_{i}"
+            ))
+
+            points.append(PointStruct(
+                id=point_id,
+                vector=embedding,
+                payload={
+                    "doc_id": doc_id,
+                    "chunk_index": i,
+                    "content": chunk.get("content", ""),
+                    "metadata": chunk.get("metadata", {})
+                }
+            ))
+
+        try:
+            self.client.upsert(
+                collection_name=collection_name,
+                points=points
+            )
+            logger.info(f"Upserted {len(points)} chunks for {doc_id}")
+            return len(points)
+        except Exception as e:
+            logger.error(f"Failed to upsert chunks: {e}", exc_info=True)
+            raise
+
+    async def search(
+        self,
+        user: str,
+        query_vector: List[float],
+        limit: int = 10,
+        score_threshold: float = 0.7,
+        filter_dict: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Semantic search in user's collection.
+
+        Args:
+            user: User identifier
+            query_vector: Query embedding vector
+            limit: Maximum results to return
+            score_threshold: Minimum similarity score (0.0-1.0)
+            filter_dict: Optional payload filters
+
+        Returns:
+            List of matches with scores and payloads
+
+        Example:
+            results = await client.search(
+                user="jpmschweitzer",
+                query_vector=[0.1, 0.2, ...],
+                limit=5,
+                filter_dict={"doc_id": "doc_123"}
+            )
+        """
+        collection_name = self.get_collection_name(user)
+
+        # Build filter if provided
+        query_filter = None
+        if filter_dict:
+            conditions = []
+            for key, value in filter_dict.items():
+                conditions.append(
+                    FieldCondition(key=key, match=MatchValue(value=value))
+                )
+            query_filter = Filter(must=conditions)
+
+        try:
+            response = self.client.query_points(
+                collection_name=collection_name,
+                query=query_vector,
+                limit=limit,
+                score_threshold=score_threshold,
+                query_filter=query_filter,
+                with_payload=True
+            )
+
+            return [
+                {
+                    "id": str(point.id),
+                    "score": point.score,
+                    "doc_id": point.payload["doc_id"],
+                    "chunk_index": point.payload["chunk_index"],
+                    "content": point.payload["content"],
+                    "metadata": point.payload.get("metadata", {})
+                }
+                for point in response.points
+            ]
+        except Exception as e:
+            logger.error(f"Search failed: {e}", exc_info=True)
+            return []
+
+    async def delete_document(self, user: str, doc_id: str) -> bool:
+        """
+        Delete all chunks for a document.
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+
+        Returns:
+            True if deleted successfully
+        """
+        collection_name = self.get_collection_name(user)
+
+        try:
+            self.client.delete(
+                collection_name=collection_name,
+                points_selector=Filter(
+                    must=[
+                        FieldCondition(
+                            key="doc_id",
+                            match=MatchValue(value=doc_id)
+                        )
+                    ]
+                )
+            )
+            logger.info(f"Deleted chunks for {doc_id}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete document: {e}", exc_info=True)
+            return False
+
+    async def get_document_chunks(
+        self,
+        user: str,
+        doc_id: str
+    ) -> List[Dict[str, Any]]:
+        """
+        Get all chunks for a document.
+
+        Args:
+            user: User identifier
+            doc_id: Document ID
+
+        Returns:
+            List of chunks with content and metadata
+        """
+        collection_name = self.get_collection_name(user)
+
+        try:
+            # Scroll through points with doc_id filter
+            points, _ = self.client.scroll(
+                collection_name=collection_name,
+                scroll_filter=Filter(
+                    must=[
+                        FieldCondition(
+                            key="doc_id",
+                            match=MatchValue(value=doc_id)
+                        )
+                    ]
+                ),
+                limit=1000,
+                with_payload=True,
+                with_vectors=False
+            )
+
+            return [
+                {
+                    "id": str(point.id),
+                    "chunk_index": point.payload["chunk_index"],
+                    "content": point.payload["content"],
+                    "metadata": point.payload.get("metadata", {})
+                }
+                for point in points
+            ]
+        except Exception as e:
+            logger.error(f"Failed to get document chunks: {e}", exc_info=True)
+            return []
+
+    async def count_documents(self, user: str) -> int:
+        """
+        Count total number of unique documents in user's collection.
+
+        Args:
+            user: User identifier
+
+        Returns:
+            Number of unique documents
+        """
+        collection_name = self.get_collection_name(user)
+
+        try:
+            # Get collection info
+            collection_info = self.client.get_collection(collection_name)
+            # This gives total points, not unique docs
+            # For unique docs, would need to aggregate by doc_id
+            return collection_info.points_count
+        except Exception as e:
+            logger.error(f"Failed to count documents: {e}", exc_info=True)
+            return 0
+
+    async def delete_collection(self, user: str) -> bool:
+        """
+        Delete user's entire collection.
+
+        Warning: This removes all data for the user!
+
+        Args:
+            user: User identifier
+
+        Returns:
+            True if deleted successfully
+        """
+        collection_name = self.get_collection_name(user)
+
+        try:
+            self.client.delete_collection(collection_name)
+            logger.warning(f"Deleted collection: {collection_name}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to delete collection: {e}", exc_info=True)
+            return False
+
+    async def find_similar_chunks(
+        self,
+        user: str,
+        doc_id: str,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Find chunks similar to those in a given document.
+
+        Strategy: Get all chunks from doc, use their vectors to find similar chunks.
+
+        Args:
+            user: User identifier
+            doc_id: Source document ID
+            limit: Maximum results per chunk
+
+        Returns:
+            List of similar chunks from other documents
+        """
+        collection_name = self.get_collection_name(user)
+
+        try:
+            # Get source document chunks with vectors
+            source_points, _ = self.client.scroll(
+                collection_name=collection_name,
+                scroll_filter=Filter(
+                    must=[
+                        FieldCondition(
+                            key="doc_id",
+                            match=MatchValue(value=doc_id)
+                        )
+                    ]
+                ),
+                limit=10,  # Sample first 10 chunks
+                with_payload=True,
+                with_vectors=True
+            )
+
+            if not source_points:
+                return []
+
+            # Search using first chunk's vector
+            # (could aggregate multiple chunks for better results)
+            first_vector = source_points[0].vector
+
+            response = self.client.query_points(
+                collection_name=collection_name,
+                query=first_vector,
+                limit=limit * 2,  # Get more to filter out same doc
+                with_payload=True
+            )
+
+            # Filter out chunks from same document
+            similar = [
+                {
+                    "id": str(point.id),
+                    "score": point.score,
+                    "doc_id": point.payload["doc_id"],
+                    "content": point.payload["content"]
+                }
+                for point in response.points
+                if point.payload["doc_id"] != doc_id
+            ]
+
+            return similar[:limit]
+
+        except Exception as e:
+            logger.error(f"Failed to find similar chunks: {e}", exc_info=True)
+            return []
+
+    async def upsert_vector(
+        self,
+        collection_name: str,
+        vector_id: str,
+        vector: List[float],
+        payload: Dict[str, Any]
+    ) -> bool:
+        """
+        Upsert a single vector point.
+
+        Args:
+            collection_name: Collection name
+            vector_id: Point ID
+            vector: Embedding vector
+            payload: Point payload/metadata
+
+        Returns:
+            True if successful
+        """
+        try:
+            self.client.upsert(
+                collection_name=collection_name,
+                points=[PointStruct(
+                    id=vector_id,
+                    vector=vector,
+                    payload=payload
+                )]
+            )
+            return True
+        except Exception as e:
+            logger.error(f"Failed to upsert vector: {e}", exc_info=True)
+            return False
+
+    async def delete_by_filter(
+        self,
+        collection_name: str,
+        filter_conditions: Dict[str, Any]
+    ) -> int:
+        """
+        Delete points matching filter conditions.
+
+        Args:
+            collection_name: Collection name
+            filter_conditions: Filter conditions (e.g., {"page_id": 5})
+
+        Returns:
+            Number of points deleted (approximation)
+        """
+        try:
+            # Build filter
+            conditions = []
+            for key, value in filter_conditions.items():
+                conditions.append(
+                    FieldCondition(key=key, match=MatchValue(value=value))
+                )
+            query_filter = Filter(must=conditions)
+
+            # Delete points
+            result = self.client.delete(
+                collection_name=collection_name,
+                points_selector=query_filter
+            )
+
+            # Return operation status (Qdrant doesn't return count directly)
+            logger.info(f"Deleted points with filter {filter_conditions}")
+            return 1  # Placeholder, actual count not available from API
+
+        except Exception as e:
+            logger.error(f"Failed to delete by filter: {e}", exc_info=True)
+            return 0
+
+    async def search_vectors(
+        self,
+        collection_name: str,
+        query_vector: List[float],
+        limit: int = 10,
+        score_threshold: float = 0.5,
+        filter_conditions: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Search vectors in collection.
+
+        Args:
+            collection_name: Collection name
+            query_vector: Query embedding vector
+            limit: Maximum results
+            score_threshold: Minimum similarity score
+            filter_conditions: Optional filter conditions
+
+        Returns:
+            List of search results with scores and payloads
+        """
+        # Build filter if provided
+        query_filter = None
+        if filter_conditions:
+            conditions = []
+            for key, value in filter_conditions.items():
+                conditions.append(
+                    FieldCondition(key=key, match=MatchValue(value=value))
+                )
+            query_filter = Filter(must=conditions)
+
+        try:
+            response = self.client.query_points(
+                collection_name=collection_name,
+                query=query_vector,
+                limit=limit,
+                score_threshold=score_threshold,
+                query_filter=query_filter,
+                with_payload=True
+            )
+
+            return [
+                {
+                    "id": str(point.id),
+                    "score": point.score,
+                    "payload": dict(point.payload)
+                }
+                for point in response.points
+            ]
+        except Exception as e:
+            logger.error(f"Search failed: {e}", exc_info=True)
+            return []
+
+    async def list_collections(self) -> List[Dict[str, Any]]:
+        """
+        List all collections with stats.
+
+        Returns:
+            List of collection info dictionaries
+        """
+        try:
+            collections = self.client.get_collections()
+            result = []
+
+            for coll in collections.collections:
+                # Get detailed collection info
+                try:
+                    info = self.client.get_collection(coll.name)
+                    result.append({
+                        "name": coll.name,
+                        "vectors_count": info.vectors_count or 0,
+                        "points_count": info.points_count or 0,
+                        "segments_count": info.segments_count or 0
+                    })
+                except Exception as e:
+                    logger.warning(f"Failed to get info for collection {coll.name}: {e}")
+                    result.append({
+                        "name": coll.name,
+                        "vectors_count": 0,
+                        "points_count": 0,
+                        "segments_count": 0
+                    })
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to list collections: {e}", exc_info=True)
+            return []

src/clients/searxng_client.py

@@ -0,0 +1,348 @@
+"""
+SearXNG search client for Library Desk.
+
+Provides async web search via SearXNG instance:
+- General web search
+- Category-specific search
+- Language filtering
+- Result formatting
+"""
+
+import httpx
+from typing import Optional, List, Dict, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SearXNGClient:
+    """
+    SearXNG search API client.
+
+    Documentation: https://docs.searxng.org/dev/search_api.html
+    API Format: JSON (?format=json)
+    """
+
+    def __init__(self, base_url: str):
+        """
+        Initialize SearXNG client.
+
+        Args:
+            base_url: SearXNG base URL (e.g., "http://searxng:8080")
+        """
+        self.base_url = base_url.rstrip("/")
+        self.search_url = f"{self.base_url}/search"
+        self.client = httpx.AsyncClient(timeout=30.0)
+        logger.info(f"Initialized SearXNG client: {base_url}")
+
+    async def close(self):
+        """Close HTTP client"""
+        await self.client.aclose()
+
+    async def search(
+        self,
+        query: str,
+        categories: Optional[List[str]] = None,
+        language: str = "en",
+        time_range: Optional[str] = None,
+        safesearch: int = 0,
+        pageno: int = 1,
+        limit: int = 10
+    ) -> Dict[str, Any]:
+        """
+        Search via SearXNG.
+
+        Args:
+            query: Search query string
+            categories: List of categories (e.g., ["general", "images", "news"])
+            language: Language code (e.g., "en", "fr", "auto")
+            time_range: Time range filter ("day", "week", "month", "year", None for all)
+            safesearch: Safe search level (0=off, 1=moderate, 2=strict)
+            pageno: Page number (starts at 1)
+            limit: Maximum results to return
+
+        Returns:
+            Search results dictionary with:
+            - query: Original query
+            - results: List of result items
+            - number_of_results: Total results found
+            - suggestions: Query suggestions
+
+        Raises:
+            Exception: If search fails
+        """
+        params = {
+            "q": query,
+            "format": "json",
+            "language": language,
+            "safesearch": safesearch,
+            "pageno": pageno
+        }
+
+        # Add optional parameters
+        if categories:
+            params["categories"] = ",".join(categories)
+        if time_range:
+            params["time_range"] = time_range
+
+        try:
+            response = await self.client.get(self.search_url, params=params)
+            response.raise_for_status()
+            data = response.json()
+
+            # Limit results if requested
+            results = data.get("results", [])
+            if limit:
+                results = results[:limit]
+
+            return {
+                "query": data.get("query", query),
+                "results": results,
+                "number_of_results": data.get("number_of_results", 0),
+                "suggestions": data.get("suggestions", []),
+                "answers": data.get("answers", []),
+                "infoboxes": data.get("infoboxes", [])
+            }
+
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error: {e.response.status_code} - {e.response.text}")
+            raise Exception(f"SearXNG search failed: {e.response.status_code}")
+        except Exception as e:
+            logger.error(f"Search failed: {e}", exc_info=True)
+            raise
+
+    async def search_general(
+        self,
+        query: str,
+        language: str = "en",
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Simplified general web search.
+
+        Args:
+            query: Search query
+            language: Language code
+            limit: Maximum results
+
+        Returns:
+            List of search results with:
+            - title: Page title
+            - url: Page URL
+            - content: Page description/snippet
+            - engine: Search engine that provided result
+            - score: Relevance score (if available)
+        """
+        try:
+            result = await self.search(
+                query=query,
+                categories=["general"],
+                language=language,
+                limit=limit
+            )
+            return result.get("results", [])
+        except Exception as e:
+            logger.error(f"General search failed: {e}")
+            return []
+
+    async def search_with_context(
+        self,
+        query: str,
+        context: Optional[str] = None,
+        language: str = "en",
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search with additional context appended to query.
+
+        Useful for RAG queries where you want to enhance search with context.
+
+        Args:
+            query: Primary search query
+            context: Additional context to append
+            language: Language code
+            limit: Maximum results
+
+        Returns:
+            List of search results
+        """
+        enhanced_query = f"{query} {context}" if context else query
+        return await self.search_general(enhanced_query, language, limit)
+
+    async def search_documentation(
+        self,
+        query: str,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for technical documentation.
+
+        Enhances query with "documentation" and filters for technical content.
+
+        Args:
+            query: Search query (e.g., "FastAPI")
+            limit: Maximum results
+
+        Returns:
+            List of documentation search results
+        """
+        enhanced_query = f"{query} documentation"
+        return await self.search_general(enhanced_query, limit=limit)
+
+    async def search_recent(
+        self,
+        query: str,
+        time_range: str = "month",
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for recent content only.
+
+        Args:
+            query: Search query
+            time_range: Time range ("day", "week", "month", "year")
+            limit: Maximum results
+
+        Returns:
+            List of recent search results
+        """
+        try:
+            result = await self.search(
+                query=query,
+                categories=["general"],
+                time_range=time_range,
+                limit=limit
+            )
+            return result.get("results", [])
+        except Exception as e:
+            logger.error(f"Recent search failed: {e}")
+            return []
+
+    async def get_suggestions(self, query: str) -> List[str]:
+        """
+        Get search query suggestions.
+
+        Args:
+            query: Partial query
+
+        Returns:
+            List of suggested queries
+        """
+        try:
+            result = await self.search(query=query, limit=1)
+            return result.get("suggestions", [])
+        except Exception as e:
+            logger.error(f"Failed to get suggestions: {e}")
+            return []
+
+    async def search_images(
+        self,
+        query: str,
+        safesearch: int = 1,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for images.
+
+        Args:
+            query: Search query
+            safesearch: Safe search level (0=off, 1=moderate, 2=strict)
+            limit: Maximum results
+
+        Returns:
+            List of image results with:
+            - title: Image title
+            - url: Image URL
+            - thumbnail_src: Thumbnail URL
+            - img_src: Full image URL
+            - content: Description
+        """
+        try:
+            result = await self.search(
+                query=query,
+                categories=["images"],
+                safesearch=safesearch,
+                limit=limit
+            )
+            return result.get("results", [])
+        except Exception as e:
+            logger.error(f"Image search failed: {e}")
+            return []
+
+    async def search_news(
+        self,
+        query: str,
+        time_range: str = "week",
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for news articles.
+
+        Args:
+            query: Search query
+            time_range: Time range ("day", "week", "month")
+            limit: Maximum results
+
+        Returns:
+            List of news results
+        """
+        try:
+            result = await self.search(
+                query=query,
+                categories=["news"],
+                time_range=time_range,
+                limit=limit
+            )
+            return result.get("results", [])
+        except Exception as e:
+            logger.error(f"News search failed: {e}")
+            return []
+
+    async def format_results_for_rag(
+        self,
+        results: List[Dict[str, Any]],
+        max_snippet_length: int = 500
+    ) -> List[Dict[str, str]]:
+        """
+        Format search results for RAG context.
+
+        Extracts relevant fields and truncates content.
+
+        Args:
+            results: Raw search results
+            max_snippet_length: Maximum length of content snippet
+
+        Returns:
+            Formatted results with title, url, snippet
+        """
+        formatted = []
+        for result in results:
+            content = result.get("content", "")
+            if len(content) > max_snippet_length:
+                content = content[:max_snippet_length] + "..."
+
+            formatted.append({
+                "title": result.get("title", ""),
+                "url": result.get("url", ""),
+                "snippet": content,
+                "engine": result.get("engine", ""),
+                "score": result.get("score", 0.0)
+            })
+
+        return formatted
+
+    async def health_check(self) -> bool:
+        """
+        Simple health check - verify SearXNG is responding.
+
+        Returns:
+            True if service is reachable, False otherwise
+        """
+        try:
+            # Just hit the base URL to check if service is up
+            response = await self.client.get(self.base_url, timeout=5.0)
+            # Accept any 2xx or 3xx status (redirects are ok)
+            return response.status_code < 400
+        except Exception as e:
+            logger.error(f"SearXNG health check failed: {e}")
+            return False

src/clients/wikijs_client.py

@@ -0,0 +1,697 @@
+"""
+Wiki.js GraphQL client for Library Desk.
+
+Provides async Wiki.js operations via GraphQL API:
+- Page CRUD (create, read, update, delete)
+- Search and listing
+- Tag management
+- Multi-tenancy via path namespaces
+"""
+
+import httpx
+from typing import Optional, List, Dict, Any
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class WikiJSClient:
+    """
+    Wiki.js GraphQL API client.
+
+    Documentation: https://docs.requarks.io/dev/api
+    Authentication: Username/password login to get user-specific JWT token
+    """
+
+    def __init__(self, base_url: str, username: str, password: str):
+        """
+        Initialize Wiki.js client.
+
+        Args:
+            base_url: Wiki.js base URL (e.g., "http://wiki:3000")
+            username: Wiki.js username (e.g., "librarian@schweitz.net")
+            password: Wiki.js password
+        """
+        self.base_url = base_url.rstrip("/")
+        self.graphql_url = f"{self.base_url}/graphql"
+        self.username = username
+        self.password = password
+        self.jwt_token: Optional[str] = None
+        self.client = httpx.AsyncClient(timeout=30.0)
+        logger.info(f"Initialized Wiki.js client: {base_url} (user: {username})")
+
+    async def close(self):
+        """Close HTTP client"""
+        await self.client.aclose()
+
+    async def login(self) -> bool:
+        """
+        Authenticate with Wiki.js using username/password.
+
+        Returns:
+            True if login successful, False otherwise
+        """
+        login_mutation = """
+        mutation Login($username: String!, $password: String!, $strategy: String!) {
+            authentication {
+                login(username: $username, password: $password, strategy: $strategy) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        message
+                    }
+                    jwt
+                }
+            }
+        }
+        """
+
+        variables = {
+            "username": self.username,
+            "password": self.password,
+            "strategy": "local"
+        }
+
+        try:
+            response = await self.client.post(
+                self.graphql_url,
+                headers={"Content-Type": "application/json"},
+                json={"query": login_mutation, "variables": variables}
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if "errors" in result:
+                logger.error(f"Login failed: {result['errors']}")
+                return False
+
+            login_result = result.get("data", {}).get("authentication", {}).get("login", {})
+            response_result = login_result.get("responseResult", {})
+
+            if not response_result.get("succeeded"):
+                logger.error(f"Login failed: {response_result.get('message')}")
+                return False
+
+            self.jwt_token = login_result.get("jwt")
+            if not self.jwt_token:
+                logger.error("Login succeeded but no JWT token received")
+                return False
+
+            logger.info(f"Successfully authenticated as {self.username}")
+            return True
+
+        except Exception as e:
+            logger.error(f"Login failed: {e}", exc_info=True)
+            return False
+
+    async def _ensure_authenticated(self):
+        """Ensure we have a valid JWT token, login if needed."""
+        if not self.jwt_token:
+            success = await self.login()
+            if not success:
+                raise Exception("Failed to authenticate with Wiki.js")
+
+    async def _execute_query(
+        self,
+        query: str,
+        variables: Optional[Dict] = None
+    ) -> Dict[str, Any]:
+        """
+        Execute GraphQL query.
+
+        Args:
+            query: GraphQL query string
+            variables: Query variables
+
+        Returns:
+            Query result data
+
+        Raises:
+            Exception: If query fails or returns errors
+        """
+        # Ensure we're authenticated before making requests
+        await self._ensure_authenticated()
+
+        payload = {
+            "query": query,
+            "variables": variables or {}
+        }
+
+        headers = {
+            "Authorization": f"Bearer {self.jwt_token}",
+            "Content-Type": "application/json"
+        }
+
+        try:
+            response = await self.client.post(
+                self.graphql_url,
+                headers=headers,
+                json=payload
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if "errors" in result:
+                logger.error(f"GraphQL errors: {result['errors']}")
+                raise Exception(f"GraphQL errors: {result['errors']}")
+
+            return result.get("data", {})
+
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error: {e.response.status_code} - {e.response.text}")
+            raise
+        except Exception as e:
+            logger.error(f"GraphQL query failed: {e}", exc_info=True)
+            raise
+
+    async def list_pages(
+        self,
+        path_prefix: str = "",
+        tags: Optional[List[str]] = None,
+        limit: int = 50
+    ) -> List[Dict[str, Any]]:
+        """
+        List pages with optional filtering.
+
+        Multi-tenancy: Use path_prefix to filter by user namespace.
+
+        Args:
+            path_prefix: Filter by path prefix (e.g., "/users/jpmschweitzer")
+            tags: Filter by tags (e.g., ["projects"])
+            limit: Maximum results
+
+        Returns:
+            List of page objects
+        """
+        query = """
+        query ListPages($limit: Int, $orderBy: PageOrderBy) {
+            pages {
+                list(limit: $limit, orderBy: $orderBy) {
+                    id
+                    path
+                    title
+                    description
+                    tags
+                    createdAt
+                    updatedAt
+                    isPublished
+                }
+            }
+        }
+        """
+
+        variables = {
+            "limit": limit,
+            "orderBy": "TITLE"
+        }
+
+        data = await self._execute_query(query, variables)
+        pages = data.get("pages", {}).get("list", [])
+
+        # Ensure tags is always a list
+        for page in pages:
+            if "tags" not in page or page["tags"] is None:
+                page["tags"] = []
+
+        # Filter by path prefix (client-side if API doesn't support)
+        if path_prefix:
+            # Normalize paths to have leading slash for consistent comparison
+            normalized_prefix = "/" + path_prefix.lstrip("/")
+            pages = [
+                p for p in pages
+                if ("/" + p["path"].lstrip("/")).startswith(normalized_prefix)
+            ]
+
+        # Filter by tags (dossiers)
+        if tags:
+            pages = [
+                p for p in pages
+                if any(tag in (p.get("tags") or []) for tag in tags)
+            ]
+
+        return pages
+
+    async def list_all_pages(
+        self,
+        path_prefix: str = "",
+        tags: Optional[List[str]] = None,
+        batch_size: int = 100
+    ) -> List[Dict[str, Any]]:
+        """
+        List ALL pages with pagination support.
+
+        Fetches pages in batches until all are retrieved.
+
+        Args:
+            path_prefix: Filter by path prefix (e.g., "users/jpmschweitzer")
+            tags: Filter by tags
+            batch_size: Number of pages per batch (max 100)
+
+        Returns:
+            Complete list of page objects
+        """
+        all_pages = []
+        offset = 0
+
+        while True:
+            # Wiki.js list doesn't support offset, but limit is enough
+            # since we filter client-side by path_prefix
+            # Just fetch a large batch
+            pages = await self.list_pages(
+                path_prefix=path_prefix,
+                tags=tags,
+                limit=1000  # Fetch up to 1000 at once
+            )
+
+            if not pages:
+                break
+
+            all_pages = pages
+            break  # Wiki.js list doesn't paginate, so one call is enough
+
+        logger.info(f"list_all_pages: found {len(all_pages)} pages (prefix: {path_prefix or 'all'})")
+        return all_pages
+
+    async def get_taxonomy_structure(
+        self,
+        user_namespace: str,
+        limit: int = 500
+    ) -> Dict[str, List[str]]:
+        """
+        Get the existing taxonomy structure for a user namespace.
+
+        Returns a dict mapping top-level categories to their subcategories.
+        This helps the LLM choose existing paths rather than creating new ones.
+
+        Args:
+            user_namespace: User namespace (e.g., "users/jpmschweitzer")
+            limit: Maximum pages to fetch
+
+        Returns:
+            Dict like {"reference": ["political-entities", "tech"], "places": ["the-netherlands"]}
+        """
+        from collections import defaultdict
+
+        pages = await self.list_pages(path_prefix=user_namespace, limit=limit)
+
+        # Extract structure: category -> set of subcategories
+        structure: Dict[str, set] = defaultdict(set)
+        prefix = user_namespace.strip("/") + "/"
+
+        for page in pages:
+            path = page.get("path", "")
+            if not path.startswith(prefix):
+                continue
+
+            # Get relative path within user namespace
+            rel_path = path[len(prefix):]
+            parts = rel_path.split("/")
+
+            if len(parts) >= 1:
+                category = parts[0]
+                if len(parts) >= 2:
+                    # Has subcategory (e.g., reference/political-entities/nato)
+                    structure[category].add(parts[1])
+                else:
+                    # Just ensure category exists in structure
+                    structure[category]  # Access to ensure key exists
+
+        # Convert sets to sorted lists
+        return {cat: sorted(list(subs)) for cat, subs in sorted(structure.items())}
+
+    async def get_page(self, page_id: int) -> Optional[Dict[str, Any]]:
+        """
+        Get single page by ID.
+
+        Args:
+            page_id: Page ID
+
+        Returns:
+            Page object or None if not found
+        """
+        query = """
+        query GetPage($id: Int!) {
+            pages {
+                single(id: $id) {
+                    id
+                    path
+                    title
+                    description
+                    content
+                    tags {
+                        tag
+                    }
+                    createdAt
+                    updatedAt
+                    isPublished
+                    editor
+                }
+            }
+        }
+        """
+
+        try:
+            # Ensure page_id is int (GraphQL requires Int, not String)
+            data = await self._execute_query(query, {"id": int(page_id)})
+            page = data.get("pages", {}).get("single")
+
+            # Extract tag strings from tag objects
+            if page and "tags" in page and page["tags"]:
+                page["tags"] = [t["tag"] for t in page["tags"]]
+            elif page:
+                page["tags"] = []
+
+            return page
+        except Exception as e:
+            logger.error(f"Failed to get page {page_id}: {e}")
+            return None
+
+    async def create_page(
+        self,
+        path: str,
+        title: str,
+        content: str,
+        description: str = "",
+        tags: Optional[List[str]] = None,
+        is_published: bool = True,
+        is_private: bool = False,
+        editor: str = "markdown"
+    ) -> Dict[str, Any]:
+        """
+        Create new page.
+
+        Multi-tenancy: Ensure path starts with user namespace.
+
+        Args:
+            path: Page path (e.g., "/users/jpmschweitzer/projects/library-desk")
+            title: Page title
+            content: Page content (markdown)
+            description: Short description
+            tags: List of tags (for dossier organization)
+            is_published: Whether page is published
+            is_private: Whether page is private
+            editor: Editor type (markdown, wysiwyg, etc.)
+
+        Returns:
+            Created page object
+
+        Raises:
+            Exception: If creation fails
+        """
+        mutation = """
+        mutation CreatePage(
+            $content: String!,
+            $description: String!,
+            $editor: String!,
+            $isPublished: Boolean!,
+            $isPrivate: Boolean!,
+            $locale: String!,
+            $path: String!,
+            $tags: [String]!,
+            $title: String!
+        ) {
+            pages {
+                create(
+                    content: $content,
+                    description: $description,
+                    editor: $editor,
+                    isPublished: $isPublished,
+                    isPrivate: $isPrivate,
+                    locale: $locale,
+                    path: $path,
+                    tags: $tags,
+                    title: $title
+                ) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        message
+                    }
+                    page {
+                        id
+                        path
+                        title
+                    }
+                }
+            }
+        }
+        """
+
+        variables = {
+            "path": path,
+            "title": title,
+            "content": content,
+            "description": description,
+            "tags": tags or [],
+            "isPublished": is_published,
+            "isPrivate": is_private,
+            "editor": editor,
+            "locale": "en"
+        }
+
+        data = await self._execute_query(mutation, variables)
+        result = data.get("pages", {}).get("create", {})
+
+        if not result.get("responseResult", {}).get("succeeded"):
+            error = result.get("responseResult", {})
+            raise Exception(f"Failed to create page: {error}")
+
+        logger.info(f"Created page: {path}")
+        return result.get("page")
+
+    async def update_page(
+        self,
+        page_id: int,
+        content: Optional[str] = None,
+        title: Optional[str] = None,
+        description: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        is_published: Optional[bool] = None
+    ) -> Dict[str, Any]:
+        """
+        Update existing page.
+
+        Args:
+            page_id: Page ID
+            content: New content (optional)
+            title: New title (optional)
+            description: New description (optional)
+            tags: New tags (optional)
+            is_published: Published status (optional)
+
+        Returns:
+            Updated page object
+
+        Raises:
+            Exception: If update fails
+        """
+        mutation = """
+        mutation UpdatePage(
+            $id: Int!,
+            $content: String,
+            $title: String,
+            $description: String,
+            $tags: [String],
+            $isPublished: Boolean
+        ) {
+            pages {
+                update(
+                    id: $id,
+                    content: $content,
+                    title: $title,
+                    description: $description,
+                    tags: $tags,
+                    isPublished: $isPublished
+                ) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        message
+                    }
+                    page {
+                        id
+                        updatedAt
+                    }
+                }
+            }
+        }
+        """
+
+        variables = {"id": page_id}
+        if content is not None:
+            variables["content"] = content
+        if title is not None:
+            variables["title"] = title
+        if description is not None:
+            variables["description"] = description
+        if tags is not None:
+            variables["tags"] = tags
+        if is_published is not None:
+            variables["isPublished"] = is_published
+
+        data = await self._execute_query(mutation, variables)
+        result = data.get("pages", {}).get("update", {})
+
+        if not result.get("responseResult", {}).get("succeeded"):
+            error = result.get("responseResult", {})
+            raise Exception(f"Failed to update page: {error}")
+
+        logger.info(f"Updated page: {page_id}")
+        return result.get("page")
+
+    async def delete_page(self, page_id: int):
+        """
+        Delete page.
+
+        Args:
+            page_id: Page ID
+
+        Raises:
+            Exception: If deletion fails
+        """
+        mutation = """
+        mutation DeletePage($id: Int!) {
+            pages {
+                delete(id: $id) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        message
+                    }
+                }
+            }
+        }
+        """
+
+        data = await self._execute_query(mutation, {"id": page_id})
+        result = data.get("pages", {}).get("delete", {})
+
+        if not result.get("responseResult", {}).get("succeeded"):
+            error = result.get("responseResult", {})
+            raise Exception(f"Failed to delete page: {error}")
+
+        logger.info(f"Deleted page: {page_id}")
+
+    async def search_pages(
+        self,
+        query: str,
+        path_prefix: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Search pages by content.
+
+        Args:
+            query: Search query
+            path_prefix: Optional path prefix filter
+
+        Returns:
+            List of matching pages
+        """
+        gql_query = """
+        query SearchPages($query: String!) {
+            pages {
+                search(query: $query) {
+                    results {
+                        id
+                        path
+                        title
+                        description
+                    }
+                }
+            }
+        }
+        """
+
+        data = await self._execute_query(gql_query, {"query": query})
+        results = data.get("pages", {}).get("search", {}).get("results", [])
+
+        # Filter by path prefix if provided
+        if path_prefix:
+            results = [r for r in results if r["path"].startswith(path_prefix)]
+
+        return results
+
+    async def get_page_tree(self, path: str = "/") -> List[Dict[str, Any]]:
+        """
+        Get page tree structure.
+
+        Args:
+            path: Root path
+
+        Returns:
+            Tree structure of pages
+        """
+        query = """
+        query GetPageTree($parent: Int, $mode: String!) {
+            pages {
+                tree(parent: $parent, mode: $mode) {
+                    id
+                    path
+                    title
+                    isFolder
+                    pageId
+                }
+            }
+        }
+        """
+
+        try:
+            data = await self._execute_query(
+                query,
+                {"parent": 0, "mode": "all"}
+            )
+            return data.get("pages", {}).get("tree", [])
+        except Exception as e:
+            logger.error(f"Failed to get page tree: {e}")
+            return []
+
+    async def move_page(
+        self,
+        page_id: int,
+        new_path: str,
+        locale: str = "en"
+    ) -> bool:
+        """
+        Move/rename page.
+
+        Args:
+            page_id: Page ID
+            new_path: New page path
+            locale: Page locale
+
+        Returns:
+            True if successful
+        """
+        mutation = """
+        mutation MovePage($id: Int!, $destinationPath: String!, $destinationLocale: String!) {
+            pages {
+                move(id: $id, destinationPath: $destinationPath, destinationLocale: $destinationLocale) {
+                    responseResult {
+                        succeeded
+                        errorCode
+                        message
+                    }
+                }
+            }
+        }
+        """
+
+        try:
+            data = await self._execute_query(
+                mutation,
+                {
+                    "id": page_id,
+                    "destinationPath": new_path,
+                    "destinationLocale": locale
+                }
+            )
+            result = data.get("pages", {}).get("move", {})
+            success = result.get("responseResult", {}).get("succeeded", False)
+
+            if success:
+                logger.info(f"Moved page {page_id} to {new_path}")
+            return success
+
+        except Exception as e:
+            logger.error(f"Failed to move page: {e}")
+            return False