""" Librarian tools for PydanticAI agent. These tools wrap the library-desk API and are registered with The Librarian agent for research and knowledge management tasks. """ import httpx from pydantic_ai import ModelRetry from src.agents.librarian.client import HybridRAGResponse, LibraryDeskClient from src.core.logging_config import get_logger logger = get_logger(__name__) def _retry_if_transient(e: Exception, what: str) -> None: """ Convert transient HTTP errors into ModelRetry so the agent's retry budget (Agent(retries=2)) engages instead of the tool swallowing the failure. Only read tools call this - writes are never retried to avoid duplicate wiki pages. """ retryable = isinstance(e, httpx.TransportError) if isinstance(e, httpx.HTTPStatusError): status = e.response.status_code retryable = status >= 500 or status == 429 if retryable: raise ModelRetry(f"{what} is temporarily unavailable; please retry.") from e # Icons keyed by the values library-desk emits in each result's `sources` # list (search legs) and `source_type` (result origin). SOURCE_ICONS = { "vector": "📄", "graph": "🔗", "web": "🌐", "document": "📑", "documents": "📑", "volatile": "⚡", "wiki": "📄", } def _coverage_note( response: HybridRAGResponse, include_web: bool, include_documents: bool, include_volatile: bool, ) -> str: """ Build a one-line coverage note when the search was degraded or an enabled source leg contributed nothing, so outages stay visible to the model and the user instead of silently narrowing results. When the additive source_status/degraded contract is present it is authoritative and used EXCLUSIVELY - no count heuristics. Without it, absence from source_counts is only inferred for the optional legs this request explicitly enabled (web/documents/volatile); the always-on wiki legs (vector/graph) are never inferred, because source_counts only tallies the sources of the final top-N fused results, so their absence is normal ranking behavior, not an outage. """ if response.source_status: failed = sorted(leg for leg, status in response.source_status.items() if status == "failed") if failed: return ( "⚠️ *Coverage note: results are partial - " f"these sources failed: {', '.join(failed)}.*" ) if response.degraded: return ( "⚠️ *Coverage note: results are partial - " "one or more sources failed during this search.*" ) return "" if not response.source_counts: # Older library-desk without per-source reporting - nothing to infer return "" # Only legs the request explicitly enabled; never vector/graph (their # absence from the top-N counts is healthy, see docstring) expected = set() if include_web: expected.add("web") if include_documents: expected.add("documents") if include_volatile: expected.add("volatile") # Normalize count keys to leg names (document/documents) aliases = {"document": "documents"} reported = {aliases.get(key, key) for key in response.source_counts} missing = sorted(expected - reported) if missing: return ( "⚠️ *Coverage note: no results came from: " f"{', '.join(missing)} (source unavailable or nothing found).*" ) return "" # ============================================================================ # HybridRAG Search # ============================================================================ async def hybrid_search( query: str, include_web: bool = True, include_documents: bool = True, include_volatile: bool = True, ) -> str: """ Search across all knowledge sources using HybridRAG. This is the primary research tool, combining: - Vector search (semantic similarity over wiki pages) - Knowledge graph (entities and relationships) - Paperless documents (📑 indexed PDFs, scans, invoices) - Volatile cache (⚡ weather, news, stocks - for user's configured items) - Web search (current information from SearXNG) Results are fused and re-ranked by relevance. Volatile data gets priority when fresh. Args: query: Natural language research query include_web: Whether to include web results (default: True) include_documents: Whether to include Paperless documents (default: True) include_volatile: Whether to include volatile cache data (default: True) Returns: Formatted search results with sources and context Examples: hybrid_search("How does Docker orchestration work with Kubernetes?") hybrid_search("What projects use Neo4j?", include_web=False) hybrid_search("Find my electricity invoices", include_web=False, include_volatile=False) hybrid_search("What's the weather in Rotterdam?") # May hit volatile cache """ try: async with LibraryDeskClient() as client: response = await client.hybrid_search( query=query, web_limit=5 if include_web else 0, document_limit=5 if include_documents else 0, volatile_limit=3 if include_volatile else 0, ) if not response.results: return f"No results found for '{query}'" # Format results output_parts = [f"## Search Results for: {query}\n"] # Add keywords if extracted if response.keywords: output_parts.append(f"**Keywords:** {', '.join(response.keywords)}") # Add related dossiers if response.related_dossiers: output_parts.append(f"**Related Dossiers:** {', '.join(response.related_dossiers)}") output_parts.append("") # Add results for i, result in enumerate(response.results, 1): source_keys = result.sources or [result.source] source_icon = "".join( dict.fromkeys(SOURCE_ICONS.get(key, "•") for key in source_keys) ) output_parts.append( f"{i}. {source_icon} **{result.title}** (score: {result.score:.2f})" ) if result.url: output_parts.append(f" URL: {result.url}") output_parts.append(f" {result.content[:300]}...") output_parts.append("") # Surface degraded coverage so outages are visible downstream coverage_note = _coverage_note( response, include_web=include_web, include_documents=include_documents, include_volatile=include_volatile, ) if coverage_note: output_parts.append(coverage_note) logger.info( "librarian_hybrid_search", query=query, result_count=len(response.results), degraded=response.degraded, source_counts=response.source_counts, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_hybrid_search_error", error=str(e), query=query) _retry_if_transient(e, "The knowledge archive") return "I was unable to search the knowledge archives; the search service did not respond properly." # ============================================================================ # Wiki Operations # ============================================================================ async def search_wiki( query: str, limit: int = 10, ) -> str: """ Search the personal wiki for relevant pages. Performs full-text search over wiki page titles, descriptions, and content. Use this for finding specific documents. Args: query: Search query limit: Maximum results (default: 10) Returns: List of matching wiki pages with paths and descriptions Examples: search_wiki("docker setup guide") search_wiki("architecture", limit=5) """ try: async with LibraryDeskClient() as client: results = await client.search_wiki(query=query, limit=limit) if not results: return f"No wiki pages found for '{query}'" output_parts = [f"## Wiki Search: {query}\n"] # No ordinal numbering: small models pass the list position to # get_wiki_page instead of the page ID unless the ID is the only # number in sight. for page in results: output_parts.append(f"- **{page.title}** (page_id: {page.id})") output_parts.append(f" Path: {page.path}") if page.description: output_parts.append(f" {page.description}") output_parts.append("") return "\n".join(output_parts) except Exception as e: logger.error("librarian_wiki_search_error", error=str(e)) _retry_if_transient(e, "The wiki search") return "I was unable to search the wiki at this time." async def get_wiki_page( page_id: int, ) -> str: """ Get the full content of a wiki page. Use this after searching to read the complete content of a specific page. Args: page_id: The page ID from search results Returns: Full page content including title, path, and markdown content Examples: get_wiki_page(42) """ try: async with LibraryDeskClient() as client: page = await client.get_wiki_page(page_id=page_id) output_parts = [ f"# {page.title}", f"**Path:** {page.path}", ] if page.description: output_parts.append(f"**Description:** {page.description}") if page.tags: output_parts.append(f"**Tags:** {', '.join(page.tags)}") output_parts.append("") output_parts.append(page.content or "(No content)") return "\n".join(output_parts) except Exception as e: logger.error("librarian_get_page_error", error=str(e), page_id=page_id) _retry_if_transient(e, "The wiki") return f"I was unable to retrieve wiki page {page_id}." async def list_dossiers() -> str: """ List all research dossiers (tag collections). Dossiers are collections of wiki pages grouped by tag. Use this to discover what knowledge collections exist. Returns: List of dossiers with page counts Examples: list_dossiers() """ try: async with LibraryDeskClient() as client: dossiers = await client.list_dossiers() if not dossiers: return "No dossiers found" output_parts = ["## Research Dossiers\n"] for dossier in dossiers: output_parts.append(f"- **{dossier.name}** ({dossier.page_count} pages)") return "\n".join(output_parts) except Exception as e: logger.error("librarian_list_dossiers_error", error=str(e)) _retry_if_transient(e, "The dossier index") return "I was unable to retrieve the list of dossiers." async def get_dossier_pages( dossier_name: str, limit: int = 20, ) -> str: """ Get all pages in a dossier. Retrieves pages tagged with the specified dossier name. Args: dossier_name: Name of the dossier/tag limit: Maximum pages to return Returns: List of pages in the dossier Examples: get_dossier_pages("projects") get_dossier_pages("architecture", limit=10) """ try: async with LibraryDeskClient() as client: pages = await client.list_wiki_pages(tag=dossier_name, limit=limit) if not pages: return f"No pages found in dossier '{dossier_name}'" output_parts = [f"## Dossier: {dossier_name}\n"] for page in pages: output_parts.append(f"- **{page.title}** ({page.path})") if page.description: output_parts.append(f" {page.description}") return "\n".join(output_parts) except Exception as e: logger.error("librarian_get_dossier_error", error=str(e)) _retry_if_transient(e, "The dossier index") return f"I was unable to retrieve the dossier '{dossier_name}'." # ============================================================================ # Semantic Search # ============================================================================ async def semantic_search( query: str, limit: int = 10, ) -> str: """ Perform semantic (vector) search over documents. Finds documents similar in meaning to the query, even if they don't contain the exact words. Args: query: Natural language query limit: Maximum results Returns: Matching document chunks with similarity scores Examples: semantic_search("containerization best practices") semantic_search("how to handle authentication") """ try: async with LibraryDeskClient() as client: results = await client.semantic_search(query=query, limit=limit) if not results: return f"No semantically similar content found for '{query}'" output_parts = [f"## Semantic Search: {query}\n"] for i, result in enumerate(results, 1): output_parts.append(f"{i}. **{result.page_title}** (score: {result.score:.2f})") output_parts.append(f" Path: {result.page_path}") output_parts.append(f" {result.chunk_text[:200]}...") output_parts.append("") return "\n".join(output_parts) except Exception as e: logger.error("librarian_semantic_search_error", error=str(e)) _retry_if_transient(e, "The semantic search") return "I was unable to complete the semantic search." # ============================================================================ # Knowledge Graph # ============================================================================ async def explore_knowledge_graph( entity_type: str = "Document", limit: int = 20, ) -> str: """ Explore entities in the knowledge graph. Lists nodes of a specific type to understand what's in the knowledge base. Args: entity_type: Type of entity (Document, Person, Project, Concept, Technology) limit: Maximum nodes to return Returns: List of entities with their properties Examples: explore_knowledge_graph("Person") explore_knowledge_graph("Technology", limit=50) """ try: async with LibraryDeskClient() as client: nodes = await client.list_graph_nodes( node_type=entity_type, limit=limit, ) if not nodes: return f"No {entity_type} nodes found in knowledge graph" output_parts = [f"## Knowledge Graph: {entity_type} Entities\n"] for node in nodes: name = node.properties.get("name", node.properties.get("title", node.id)) output_parts.append(f"- **{name}**") # Show a few key properties for key in ["description", "url", "path"]: if key in node.properties: output_parts.append(f" {key}: {node.properties[key]}") return "\n".join(output_parts) except Exception as e: logger.error("librarian_explore_graph_error", error=str(e)) _retry_if_transient(e, "The knowledge graph") return "I was unable to explore the knowledge graph." async def find_related_entities( entity_name: str, ) -> str: """ Find entities related to a given concept or entity. Queries the knowledge graph to find documents, people, and concepts connected to the specified entity. Args: entity_name: Name of the entity to find relationships for Returns: Related entities and their relationships Examples: find_related_entities("Docker") find_related_entities("Kubernetes") """ try: async with LibraryDeskClient() as client: # Find entities mentioning or related to the search term cypher = """ MATCH (n) WHERE toLower(n.name) CONTAINS toLower($name) OR toLower(n.title) CONTAINS toLower($name) OPTIONAL MATCH (n)-[r]-(related) RETURN n, collect(DISTINCT {type: type(r), node: related})[0..10] as relationships LIMIT 10 """ results = await client.query_graph( cypher, parameters={"name": entity_name}, ) if not results: return f"No entities found related to '{entity_name}'" output_parts = [f"## Entities Related to: {entity_name}\n"] for record in results: node = record.get("n", {}) relationships = record.get("relationships", []) name = node.get("name", node.get("title", "Unknown")) labels = node.get("labels", []) output_parts.append(f"### {name}") if labels: output_parts.append(f"Type: {', '.join(labels)}") if relationships: output_parts.append("**Connections:**") for rel in relationships[:5]: # Limit to 5 relationships rel_type = rel.get("type", "RELATED_TO") related_node = rel.get("node", {}) related_name = related_node.get( "name", related_node.get("title", "Unknown") ) output_parts.append(f" - {rel_type} → {related_name}") output_parts.append("") return "\n".join(output_parts) except Exception as e: logger.error("librarian_find_related_error", error=str(e)) _retry_if_transient(e, "The knowledge graph") return f"I was unable to look up entities related to '{entity_name}'." # ============================================================================ # Web Search & Content Extraction # ============================================================================ async def search_web( query: str, limit: int = 10, search_type: str = "web", ) -> str: """ Search the web and extract content from results. This is the primary tool for finding current information online. Results include both snippets and full extracted text from pages. Search types: - "web": General web search (default) - "news": News articles - "images": Image search Args: query: Search query (1-500 chars) limit: Number of results (1-20, default: 10) search_type: Type of search ("web", "news", or "images") Returns: Formatted search results with sources and extracted content Examples: search_web("Python 3.12 new features") search_web("latest tech news", search_type="news", limit=5) """ try: async with LibraryDeskClient() as client: response = await client.search_web( query=query, limit=limit, search_type=search_type, ) if not response.results: return f"No results found for '{query}'" output_parts = [f"## Web Search: {query}\n"] output_parts.append( f"*Found {response.total_results} results in {response.search_time_ms}ms*\n" ) for i, result in enumerate(response.results, 1): output_parts.append(f"### {i}. {result.title}") output_parts.append(f"**Source:** {result.source}") output_parts.append(f"**URL:** {result.url}") if result.published_date: output_parts.append(f"**Date:** {result.published_date}") # Use full content if available, otherwise snippet content = result.content or result.snippet if content: # Truncate for readability if len(content) > 500: content = content[:500] + "..." output_parts.append(f"\n{content}") output_parts.append("") # Add pre-formatted sources for citations if response.sources_summary: output_parts.append("---") output_parts.append(response.sources_summary) logger.info( "librarian_web_search", query=query, result_count=response.total_results, search_type=search_type, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_web_search_error", error=str(e), query=query) _retry_if_transient(e, "The web search") return "I was unable to search the web at this time." async def read_url( url: str, max_length: int = 5000, ) -> str: """ Read and extract the main content from a URL. Use this when you have a specific URL to read, such as: - A link the user provided - A URL from search results you want to read in full - Documentation or article pages Extracts the main content, removing ads, navigation, and boilerplate. Args: url: The URL to read max_length: Maximum content length (default: 5000) Returns: Extracted page content with metadata Examples: read_url("https://docs.python.org/3/library/asyncio.html") read_url("https://example.com/article", max_length=10000) """ try: async with LibraryDeskClient() as client: result = await client.extract_content( url=url, include_metadata=True, max_length=max_length, ) if not result.success: return f"Could not read page: {result.error or 'Unknown error'}" output_parts = [] # Header with metadata if result.title: output_parts.append(f"# {result.title}") else: output_parts.append(f"# Content from {url}") output_parts.append(f"**URL:** {url}") if result.author: output_parts.append(f"**Author:** {result.author}") if result.date: output_parts.append(f"**Date:** {result.date}") if result.language and result.language != "en": output_parts.append(f"**Language:** {result.language}") output_parts.append("") # Main content if result.content: output_parts.append(result.content) else: output_parts.append("(No content could be extracted)") logger.info( "librarian_read_url", url=url, content_length=len(result.content) if result.content else 0, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_read_url_error", error=str(e), url=url) _retry_if_transient(e, "Content extraction") return f"I was unable to read the page at {url}." async def read_urls_batch( urls: list[str], max_length: int = 2000, ) -> str: """ Read and extract content from multiple URLs in parallel. More efficient than calling read_url multiple times. Max 20 URLs per batch. Note: Individual failures don't fail the entire batch - failed URLs are reported but other content is still returned. Args: urls: List of URLs to read (max 20) max_length: Maximum content length per URL (default: 2000) Returns: Extracted content from all successful URLs with failure report Examples: read_urls_batch(["https://example.com/1", "https://example.com/2"]) """ try: async with LibraryDeskClient() as client: response = await client.extract_content_batch( urls=urls, include_metadata=True, max_length=max_length, ) output_parts = [ "## Batch Content Extraction", f"*Extracted {response.successful}/{response.total_urls} URLs in {response.extraction_time_ms}ms*\n", ] # Show successful extractions for result in response.results: if result.success: title = result.title or result.url output_parts.append(f"### {title}") output_parts.append(f"**URL:** {result.url}") if result.content: # Truncate for readability in batch mode content = result.content if len(content) > max_length: content = content[:max_length] + "..." output_parts.append(f"\n{content}") output_parts.append("") # Report failures failed = [r for r in response.results if not r.success] if failed: output_parts.append("---") output_parts.append("### Failed Extractions") for result in failed: output_parts.append(f"- {result.url}: {result.error}") logger.info( "librarian_read_urls_batch", total=response.total_urls, successful=response.successful, failed=response.failed, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_read_urls_batch_error", error=str(e)) _retry_if_transient(e, "Content extraction") return "I was unable to read the requested pages." # ============================================================================ # Wiki Write Operations # ============================================================================ CLEAR_TAGS_SENTINEL = "__CLEAR__" async def update_wiki_page( page_id: int, content: str = "", title: str = "", tags: list[str] = [], # noqa: B006 - sentinel, never mutated description: str = "", ) -> str: """ Update an existing wiki page. Supports partial updates - only specify the fields you want to change. Changes trigger automatic vector re-indexing and knowledge graph updates. Use this for: - Correcting information in a page - Adding content to an existing page - Updating tags to organize pages into dossiers - Fixing descriptions or titles Note: empty values are sentinels for "leave unchanged" (Ollama's OpenAI-compatible API mishandles anyOf[X, null] parameter schemas). Args: page_id: ID of the page to update (get from search_wiki results) content: New markdown content (empty = leave unchanged) title: New title (empty = leave unchanged) tags: New tag list, replaces existing tags (empty = leave unchanged). To remove ALL tags from a page, pass exactly ["__CLEAR__"] (an empty list means "leave unchanged", not "clear") description: New description (empty = leave unchanged) Returns: Confirmation with updated page details Examples: update_wiki_page(42, content="# Updated Content\\n\\nNew information here") update_wiki_page(42, tags=["projects", "devops"]) # Add to dossiers update_wiki_page(42, tags=["__CLEAR__"]) # Remove all tags update_wiki_page(42, description="Updated description") """ # Empty list = leave unchanged; the explicit clear sentinel sends an # empty tag list to the service, which replaces (clears) all tags. clear_tags = tags == [CLEAR_TAGS_SENTINEL] try: async with LibraryDeskClient() as client: page = await client.update_wiki_page( page_id=page_id, content=content if content else None, title=title if title else None, tags=[] if clear_tags else (tags if tags else None), description=description if description else None, ) # Build update summary updated_fields = [] if content: updated_fields.append("content") if title: updated_fields.append("title") if clear_tags: updated_fields.append("tags (cleared)") elif tags: updated_fields.append("tags") if description: updated_fields.append("description") output_parts = [ f"## Page Updated: {page.title}", f"**Path:** {page.path}", f"**Updated fields:** {', '.join(updated_fields)}", ] if page.tags: output_parts.append(f"**Tags:** {', '.join(page.tags)}") output_parts.append( "\n*Vector embeddings and knowledge graph will be updated automatically.*" ) logger.info( "librarian_update_page", page_id=page_id, updated_fields=updated_fields, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_update_page_error", error=str(e), page_id=page_id) return f"I was unable to update wiki page {page_id}." async def create_wiki_page( title: str, path: str, content: str, tags: list[str], description: str = "", ) -> str: """ Create a new wiki page with user-provided content. Use this when: - User provides specific content to add - Creating simple notes or reminders - The content is already known/composed For research-backed pages where you need to gather information first, use smart_create_wiki_page instead. Args: title: Page title path: Page path (e.g., "/projects/my-project" or "/notes/meeting-2024") content: Markdown content for the page tags: List of tags/dossiers (e.g., ["projects", "devops"]) description: Short description of the page Returns: Confirmation with created page details Examples: create_wiki_page( title="SSL Renewal Reminder", path="/reminders/ssl-renewal", content="# SSL Renewal\\n\\nRemember to renew SSL cert on Jan 15", tags=["reminders", "infrastructure"], description="Certificate renewal reminder" ) """ try: async with LibraryDeskClient() as client: page = await client.create_wiki_page( title=title, path=path, content=content, tags=tags, description=description, ) output_parts = [ f"## Page Created: {page.title}", f"**ID:** {page.id}", f"**Path:** {page.path}", ] if page.tags: output_parts.append(f"**Tags:** {', '.join(page.tags)}") if page.description: output_parts.append(f"**Description:** {page.description}") output_parts.append( "\n*Vector embeddings and knowledge graph will be updated automatically.*" ) logger.info( "librarian_create_page", page_id=page.id, title=title, path=path, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_create_page_error", error=str(e), title=title) return f"I was unable to create the page '{title}'." async def smart_create_wiki_page( topic: str, tags: list[str], path: str = "", include_web_research: bool = True, include_wiki_search: bool = True, ) -> str: """ Create a wiki page with automatic research and content synthesis. This is the RECOMMENDED way to create pages about topics. It will: 1. Search existing wiki, knowledge graph, and web for relevant information 2. Use an LLM to synthesize findings into well-structured content 3. Create the page with proper source attribution 4. Automatically link entities bidirectionally in the knowledge graph Use this when: - User says "Create a page about X" - User says "Add information about X to the wiki" - You need to research a topic before writing - The topic would benefit from existing knowledge context Args: topic: The topic to research and create a page about tags: List of tags/dossiers for categorization path: Optional custom path (empty = auto-generated from topic) include_web_research: Whether to search the web (default: True) include_wiki_search: Whether to search existing wiki (default: True) Returns: Summary of created page with research statistics Examples: smart_create_wiki_page("Docker Compose", tags=["technology", "devops"]) smart_create_wiki_page("Home network architecture", tags=["infrastructure"], include_web_research=False) """ try: async with LibraryDeskClient() as client: response = await client.smart_create_wiki_page( topic=topic, tags=tags, path=path if path else None, include_web_research=include_web_research, include_wiki_search=include_wiki_search, ) page = response.page research = response.research_summary linking = response.entity_linking output_parts = [ f"## Page Created: {page.title}", f"**ID:** {page.id}", f"**Path:** {page.path}", ] if page.tags: output_parts.append(f"**Tags:** {', '.join(page.tags)}") # Research summary output_parts.append("\n### Research Summary") output_parts.append(f"- **Wiki results used:** {research.wiki_results}") output_parts.append(f"- **Web results used:** {research.web_results}") output_parts.append(f"- **Graph entities found:** {research.graph_entities}") output_parts.append(f"- **Keywords extracted:** {research.keywords_extracted}") output_parts.append(f"- **Total sources:** {response.sources_used}") output_parts.append(f"- **Research time:** {research.timing_ms}ms") # Entity linking if linking.forward_links > 0 or linking.backward_links > 0: output_parts.append("\n### Knowledge Graph Updates") output_parts.append(f"- **Forward links created:** {linking.forward_links}") output_parts.append(f"- **Backward links created:** {linking.backward_links}") output_parts.append(f"- **Related pages updated:** {linking.pages_updated}") logger.info( "librarian_smart_create", topic=topic, page_id=page.id, sources_used=response.sources_used, ) return "\n".join(output_parts) except Exception as e: logger.error("librarian_smart_create_error", error=str(e), topic=topic) return f"I was unable to create a page about '{topic}'." # ============================================================================ # Tool Collection for Registration # ============================================================================ # All tools available to The Librarian LIBRARIAN_TOOLS = [ # Research tools (internal knowledge) hybrid_search, search_wiki, get_wiki_page, list_dossiers, get_dossier_pages, semantic_search, explore_knowledge_graph, find_related_entities, # Web search & content extraction search_web, read_url, read_urls_batch, # Write tools create_wiki_page, update_wiki_page, smart_create_wiki_page, ]