""" HTTP client for the Library-Desk API. Provides async methods for all relevant library-desk endpoints: - HybridRAG queries - Wiki operations - Vector search - Knowledge graph queries """ from typing import Any import httpx from pydantic import BaseModel, Field from src.core.config import config from src.core.context import get_user from src.core.logging_config import get_logger logger = get_logger(__name__) # ============================================================================ # Response Models # ============================================================================ class WikiPage(BaseModel): """Wiki page from library-desk.""" id: int path: str title: str description: str | None = None content: str | None = None tags: list[str] = Field(default_factory=list) created_at: str | None = None updated_at: str | None = None class WikiSearchResult(BaseModel): """Search result from wiki search.""" id: int path: str title: str description: str | None = None locale: str | None = None class VectorSearchResult(BaseModel): """Result from semantic vector search.""" page_id: int page_path: str page_title: str chunk_text: str score: float chunk_index: int class HybridSearchResult(BaseModel): """Result from HybridRAG search.""" source: str # "vector", "graph", "web" title: str content: str url: str | None = None score: float page_id: int | None = None metadata: dict[str, Any] = Field(default_factory=dict) class HybridRAGResponse(BaseModel): """Full response from HybridRAG query.""" results: list[HybridSearchResult] = Field(default_factory=list) keywords: list[str] = Field(default_factory=list) synonyms: list[str] = Field(default_factory=list) related_dossiers: list[str] = Field(default_factory=list) formatted_context: str = "" search_id: str | None = None timing: dict[str, float] = Field(default_factory=dict) class GraphNode(BaseModel): """Node from knowledge graph.""" id: str labels: list[str] = Field(default_factory=list) properties: dict[str, Any] = Field(default_factory=dict) class Dossier(BaseModel): """A dossier (tag-based collection).""" name: str page_count: int class ResearchSummary(BaseModel): """Summary of research performed during smart-create.""" wiki_results: int = 0 web_results: int = 0 graph_entities: int = 0 keywords_extracted: int = 0 timing_ms: int = 0 class WebSearchResult(BaseModel): """Result from web search via /rag/search.""" title: str url: str content: str = "" # Full extracted text via Trafilatura snippet: str = "" # Original search engine snippet source: str = "" # Domain name published_date: str | None = None class WebSearchResponse(BaseModel): """Response from /rag/search endpoint.""" query: str search_type: str results: list[WebSearchResult] = Field(default_factory=list) total_results: int = 0 search_time_ms: int = 0 sources_summary: str = "" # Pre-formatted markdown citations class ContentExtractionResult(BaseModel): """Result from content extraction.""" url: str title: str | None = None content: str = "" author: str | None = None date: str | None = None language: str | None = None success: bool = True error: str | None = None class BatchExtractionResponse(BaseModel): """Response from batch content extraction.""" results: list[ContentExtractionResult] = Field(default_factory=list) total_urls: int = 0 successful: int = 0 failed: int = 0 extraction_time_ms: int = 0 class EntityLinking(BaseModel): """Entity linking results from smart-create.""" forward_links: int = 0 backward_links: int = 0 pages_updated: int = 0 class SmartCreateResponse(BaseModel): """Response from smart-create wiki page endpoint.""" page: WikiPage research_summary: ResearchSummary = Field(default_factory=ResearchSummary) sources_used: int = 0 search_id: str | None = None entity_linking: EntityLinking = Field(default_factory=EntityLinking) # ============================================================================ # Client # ============================================================================ class LibraryDeskClient: """ Async HTTP client for Library-Desk API. Usage: async with LibraryDeskClient() as client: results = await client.hybrid_search("docker kubernetes") """ def __init__( self, base_url: str | None = None, api_key: str | None = None, timeout: int = 60, ): """ Initialize the client. Args: base_url: Library-desk API URL (defaults to config) api_key: API key for authentication (defaults to config) timeout: Request timeout in seconds """ self.base_url = base_url or str(config.LIBRARY_DESK_HOST) self.api_key = api_key or config.LIBRARY_DESK_API_KEY self.timeout = timeout self._client: httpx.AsyncClient | None = None async def __aenter__(self) -> "LibraryDeskClient": """Create HTTP client on context entry.""" headers = {} if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" self._client = httpx.AsyncClient( base_url=self.base_url, headers=headers, timeout=self.timeout, ) return self async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Close HTTP client on context exit.""" if self._client: await self._client.aclose() self._client = None def _ensure_client(self) -> httpx.AsyncClient: """Ensure client is initialized.""" if self._client is None: raise RuntimeError( "Client not initialized. Use 'async with LibraryDeskClient() as client:'" ) return self._client # ======================================================================== # HybridRAG # ======================================================================== async def hybrid_search( self, query: str, user: str | None = None, vector_limit: int = 10, graph_limit: int = 10, web_limit: int = 5, document_limit: int = 5, volatile_limit: int = 3, enable_reranking: bool = True, final_result_count: int = 10, ) -> HybridRAGResponse: """ Execute HybridRAG search combining vector, graph, documents, volatile, and web. Args: query: Search query user: User identifier for multi-tenancy (defaults to request context) vector_limit: Max results from vector search (wiki pages) graph_limit: Max results from graph search web_limit: Max results from web search (0 to disable) document_limit: Max results from Paperless documents (0 to disable) volatile_limit: Max results from volatile cache (0 to disable) enable_reranking: Whether to rerank with LLM final_result_count: Number of final results after fusion Returns: HybridRAGResponse with ranked results and context """ user = user or get_user() client = self._ensure_client() payload = { "query": query, "config": { "vector_limit": vector_limit, "graph_limit": graph_limit, "web_limit": web_limit, "document_limit": document_limit, "volatile_limit": volatile_limit, "enable_documents": document_limit > 0, "enable_volatile": volatile_limit > 0, "enable_web": web_limit > 0, "enable_reranking": enable_reranking, "final_result_count": final_result_count, }, } logger.info("library_desk_hybrid_search", query=query, user=user) response = await client.post( "/query/hybrid", json=payload, params={"user": user}, ) response.raise_for_status() data = response.json() # Parse results results = [] for r in data.get("results", []): results.append(HybridSearchResult( source=r.get("source", "unknown"), title=r.get("title", ""), content=r.get("content", ""), url=r.get("url"), score=r.get("score", 0.0), page_id=r.get("page_id"), metadata=r.get("metadata", {}), )) # Handle keywords being either a list or a dict with core_keywords raw_keywords = data.get("keywords", []) if isinstance(raw_keywords, dict): keywords = raw_keywords.get("core_keywords", []) else: keywords = raw_keywords return HybridRAGResponse( results=results, keywords=keywords, synonyms=data.get("synonyms", []), related_dossiers=data.get("related_dossiers", []), formatted_context=data.get("formatted_context", ""), search_id=data.get("search_id"), timing=data.get("timing", {}), ) # ======================================================================== # Wiki Operations # ======================================================================== async def search_wiki( self, query: str, user: str | None = None, limit: int = 20, ) -> list[WikiSearchResult]: """ Search wiki pages by text. Args: query: Search query user: User identifier (defaults to request context) limit: Maximum results Returns: List of matching wiki pages """ user = user or get_user() client = self._ensure_client() logger.debug("library_desk_wiki_search", query=query, user=user) response = await client.get( "/wiki/search", params={"q": query, "user": user, "limit": limit}, ) response.raise_for_status() data = response.json() return [WikiSearchResult(**r) for r in data.get("results", [])] async def get_wiki_page( self, page_id: int, user: str | None = None, ) -> WikiPage: """ Get a wiki page by ID. Args: page_id: Page ID user: User identifier (defaults to request context) Returns: WikiPage with full content """ user = user or get_user() client = self._ensure_client() response = await client.get( f"/wiki/pages/{page_id}", params={"user": user}, ) response.raise_for_status() return WikiPage(**response.json()) async def list_wiki_pages( self, user: str | None = None, tag: str | None = None, limit: int = 50, ) -> list[WikiPage]: """ List wiki pages, optionally filtered by tag. Args: user: User identifier (defaults to request context) tag: Optional tag (dossier) to filter by limit: Maximum pages to return Returns: List of wiki pages """ user = user or get_user() client = self._ensure_client() params: dict[str, Any] = {"user": user, "limit": limit} if tag: params["tag"] = tag response = await client.get("/wiki/pages", params=params) response.raise_for_status() data = response.json() return [WikiPage(**p) for p in data.get("pages", [])] async def create_wiki_page( self, title: str, path: str, content: str, user: str | None = None, description: str = "", tags: list[str] | None = None, ) -> WikiPage: """ Create a new wiki page. Args: title: Page title path: Page path (e.g., "/projects/my-project") content: Markdown content user: User identifier (defaults to request context) description: Short description tags: List of tags (dossiers) Returns: Created WikiPage """ user = user or get_user() client = self._ensure_client() payload = { "title": title, "path": path, "content": content, "user": user, "description": description, "tags": tags or [], } logger.info("library_desk_create_page", title=title, path=path) response = await client.post("/wiki/pages", json=payload) response.raise_for_status() return WikiPage(**response.json()) async def update_wiki_page( self, page_id: int, user: str | None = None, content: str | None = None, title: str | None = None, tags: list[str] | None = None, description: str | None = None, ) -> WikiPage: """ Update an existing wiki page. Supports partial updates - only provided fields are updated. Automatically triggers vector re-indexing and graph extraction. Args: page_id: ID of the page to update user: User identifier (defaults to request context) content: New content (optional) title: New title (optional) tags: New tags list (optional) description: New description (optional) Returns: Updated WikiPage """ user = user or get_user() client = self._ensure_client() # Build update payload with only provided fields update_data: dict[str, Any] = {} if content is not None: update_data["content"] = content if title is not None: update_data["title"] = title if tags is not None: update_data["tags"] = tags if description is not None: update_data["description"] = description logger.info( "library_desk_update_page", page_id=page_id, fields=list(update_data.keys()), ) response = await client.put( f"/wiki/pages/{page_id}", params={"user": user}, json=update_data, ) response.raise_for_status() return WikiPage(**response.json()) async def smart_create_wiki_page( self, topic: str, tags: list[str], user: str | None = None, path: str | None = None, include_web_research: bool = True, include_wiki_search: bool = True, ) -> SmartCreateResponse: """ Create a wiki page with HybridRAG research. This endpoint: 1. Searches existing wiki, knowledge graph, and web for context 2. Uses LLM to synthesize findings into structured content 3. Creates the page with proper attribution 4. Automatically links entities bidirectionally Args: topic: The topic to research and create a page about tags: List of tags (dossiers) for the page user: User identifier path: Optional custom path (auto-generated from topic if not provided) include_web_research: Whether to include web search results include_wiki_search: Whether to include existing wiki content Returns: SmartCreateResponse with page and research metadata """ user = user or get_user() client = self._ensure_client() payload: dict[str, Any] = { "topic": topic, "tags": tags, "user": user, "include_web_research": include_web_research, "include_wiki_search": include_wiki_search, } if path is not None: payload["path"] = path logger.info( "library_desk_smart_create", topic=topic, tags=tags, include_web=include_web_research, ) response = await client.post("/wiki/pages/smart-create", json=payload) response.raise_for_status() data = response.json() # Parse nested response page = WikiPage(**data.get("page", {})) research_summary = ResearchSummary(**data.get("research_summary", {})) entity_linking = EntityLinking(**data.get("entity_linking", {})) return SmartCreateResponse( page=page, research_summary=research_summary, sources_used=data.get("sources_used", 0), search_id=data.get("search_id"), entity_linking=entity_linking, ) async def list_dossiers( self, user: str | None = None, ) -> list[Dossier]: """ List all dossiers (tag collections) for a user. Args: user: User identifier (defaults to request context) Returns: List of dossiers with page counts """ user = user or get_user() client = self._ensure_client() response = await client.get( "/wiki/dossiers", params={"user": user}, ) response.raise_for_status() data = response.json() return [Dossier(**d) for d in data.get("dossiers", [])] # ======================================================================== # Vector Search # ======================================================================== async def semantic_search( self, query: str, user: str | None = None, limit: int = 10, score_threshold: float = 0.5, ) -> list[VectorSearchResult]: """ Perform semantic (vector) search over documents. Args: query: Natural language query user: User identifier (defaults to request context) limit: Maximum results score_threshold: Minimum similarity score Returns: List of matching document chunks with scores """ user = user or get_user() client = self._ensure_client() payload = { "query": query, "user": user, "limit": limit, "score_threshold": score_threshold, } logger.debug("library_desk_semantic_search", query=query) response = await client.post("/vector/search", json=payload) response.raise_for_status() data = response.json() return [VectorSearchResult(**r) for r in data.get("results", [])] # ======================================================================== # Knowledge Graph # ======================================================================== async def query_graph( self, cypher_query: str, user: str | None = None, parameters: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """ Execute a Cypher query on the knowledge graph. Note: Query is automatically scoped to user's data. Args: cypher_query: Cypher query string user: User identifier (defaults to request context) parameters: Query parameters Returns: List of result records """ user = user or get_user() client = self._ensure_client() payload = { "query": cypher_query, "user": user, "parameters": parameters or {}, } logger.debug("library_desk_graph_query", query=cypher_query[:100]) response = await client.post("/graph/query", json=payload) response.raise_for_status() return response.json().get("records", []) async def list_graph_nodes( self, user: str | None = None, node_type: str | None = None, limit: int = 100, ) -> list[GraphNode]: """ List nodes in the knowledge graph. Args: user: User identifier (defaults to request context) node_type: Optional filter by type (Document, Person, Concept, etc.) limit: Maximum nodes Returns: List of graph nodes """ user = user or get_user() client = self._ensure_client() params: dict[str, Any] = {"user": user, "limit": limit} if node_type: params["node_type"] = node_type response = await client.get("/graph/nodes", params=params) response.raise_for_status() data = response.json() return [GraphNode(**n) for n in data.get("nodes", [])] async def get_graph_node( self, node_id: str, user: str | None = None, ) -> dict[str, Any]: """ Get detailed information about a graph node. Args: node_id: Node ID user: User identifier (defaults to request context) Returns: Node with relationships and connected nodes """ user = user or get_user() client = self._ensure_client() response = await client.get( f"/graph/nodes/{node_id}", params={"user": user}, ) response.raise_for_status() return response.json() # ======================================================================== # Health Check # ======================================================================== async def health_check(self) -> bool: """ Check if library-desk is healthy. Returns: True if healthy, False otherwise """ try: client = self._ensure_client() response = await client.get("/health") return response.status_code == 200 except Exception as e: logger.warning("library_desk_health_check_failed", error=str(e)) return False # ======================================================================== # RAG Search (Web Search with Content Extraction) # ======================================================================== async def search_web( self, query: str, user: str | None = None, search_type: str = "web", limit: int = 10, ) -> WebSearchResponse: """ Search the web and extract content from results. Uses SearXNG for search and Trafilatura for content extraction. Returns both snippets and full extracted text. Args: query: Search query (1-500 chars) user: User identifier for tracking search_type: "web", "news", or "images" limit: Number of results (1-20) Returns: WebSearchResponse with results and pre-formatted sources """ user = user or get_user() client = self._ensure_client() payload = { "query": query, "search_type": search_type, "limit": limit, "user": user or "tatlock-librarian", } logger.info("library_desk_web_search", query=query, limit=limit) response = await client.post("/rag/search", json=payload, timeout=30.0) response.raise_for_status() data = response.json() results = [ WebSearchResult( title=r.get("title", ""), url=r.get("url", ""), content=r.get("content", ""), snippet=r.get("snippet", ""), source=r.get("source", ""), published_date=r.get("published_date"), ) for r in data.get("results", []) ] return WebSearchResponse( query=data.get("query", query), search_type=data.get("search_type", search_type), results=results, total_results=data.get("total_results", len(results)), search_time_ms=data.get("search_time_ms", 0), sources_summary=data.get("sources_summary", ""), ) # ======================================================================== # Content Extraction # ======================================================================== async def extract_content( self, url: str, include_metadata: bool = True, max_length: int = 5000, ) -> ContentExtractionResult: """ Extract main content from a URL. Uses Trafilatura for intelligent content extraction, removing boilerplate, ads, and navigation. Note: Uses soft failure pattern - check result.success field. Args: url: URL to extract content from include_metadata: Whether to extract author, date, etc. max_length: Maximum content length Returns: ContentExtractionResult (check .success and .error fields) """ client = self._ensure_client() payload = { "url": url, "include_metadata": include_metadata, "max_length": max_length, } logger.debug("library_desk_extract_content", url=url) response = await client.post("/content/extract", json=payload, timeout=30.0) response.raise_for_status() data = response.json() result = data.get("result", {}) return ContentExtractionResult( url=result.get("url", url), title=result.get("title"), content=result.get("content", ""), author=result.get("author"), date=result.get("date"), language=result.get("language"), success=result.get("success", False), error=result.get("error"), ) async def extract_content_batch( self, urls: list[str], include_metadata: bool = True, max_length: int = 2000, ) -> BatchExtractionResponse: """ Extract content from multiple URLs in parallel. More efficient than sequential calls. Max 20 URLs per batch. Note: Uses soft failure pattern - individual failures don't throw errors, check each result's .success field. Args: urls: List of URLs to extract (max 20) include_metadata: Whether to extract author, date, etc. max_length: Maximum content length per URL Returns: BatchExtractionResponse with results and stats """ client = self._ensure_client() payload = { "urls": urls[:20], # Server limit "include_metadata": include_metadata, "max_length": max_length, } logger.info("library_desk_extract_batch", url_count=len(urls)) response = await client.post( "/content/extract/batch", json=payload, timeout=60.0, # Longer timeout for batch ) response.raise_for_status() data = response.json() results = [ ContentExtractionResult( url=r.get("url", ""), title=r.get("title"), content=r.get("content", ""), author=r.get("author"), date=r.get("date"), language=r.get("language"), success=r.get("success", False), error=r.get("error"), ) for r in data.get("results", []) ] return BatchExtractionResponse( results=results, total_urls=data.get("total_urls", len(urls)), successful=data.get("successful", 0), failed=data.get("failed", 0), extraction_time_ms=data.get("extraction_time_ms", 0), ) # Global client factory async def get_library_client() -> LibraryDeskClient: """ Get a library-desk client instance. Usage: async with get_library_client() as client: results = await client.hybrid_search("query") """ return LibraryDeskClient()