jpmschweitzerandClaude Fable 5
6ec77091b6
Mechanical Optional[X] -> X | None and f-string cleanups so subsequent librarian changes lint clean against the dirty baseline. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
936 lines
28 KiB
Python
936 lines
28 KiB
Python
"""
|
|
HTTP client for the Library-Desk API.
|
|
|
|
Provides async methods for all relevant library-desk endpoints:
|
|
- HybridRAG queries
|
|
- Wiki operations
|
|
- Vector search
|
|
- Knowledge graph queries
|
|
"""
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from pydantic import BaseModel, Field
|
|
|
|
from src.core.config import config
|
|
from src.core.context import get_user
|
|
from src.core.logging_config import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
# ============================================================================
|
|
# Response Models
|
|
# ============================================================================
|
|
|
|
class WikiPage(BaseModel):
|
|
"""Wiki page from library-desk."""
|
|
id: int
|
|
path: str
|
|
title: str
|
|
description: str | None = None
|
|
content: str | None = None
|
|
tags: list[str] = Field(default_factory=list)
|
|
created_at: str | None = None
|
|
updated_at: str | None = None
|
|
|
|
|
|
class WikiSearchResult(BaseModel):
|
|
"""Search result from wiki search."""
|
|
id: int
|
|
path: str
|
|
title: str
|
|
description: str | None = None
|
|
locale: str | None = None
|
|
|
|
|
|
class VectorSearchResult(BaseModel):
|
|
"""Result from semantic vector search."""
|
|
page_id: int
|
|
page_path: str
|
|
page_title: str
|
|
chunk_text: str
|
|
score: float
|
|
chunk_index: int
|
|
|
|
|
|
class HybridSearchResult(BaseModel):
|
|
"""Result from HybridRAG search."""
|
|
source: str # "vector", "graph", "web"
|
|
title: str
|
|
content: str
|
|
url: str | None = None
|
|
score: float
|
|
page_id: int | None = None
|
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
|
|
class HybridRAGResponse(BaseModel):
|
|
"""Full response from HybridRAG query."""
|
|
results: list[HybridSearchResult] = Field(default_factory=list)
|
|
keywords: list[str] = Field(default_factory=list)
|
|
synonyms: list[str] = Field(default_factory=list)
|
|
related_dossiers: list[str] = Field(default_factory=list)
|
|
formatted_context: str = ""
|
|
search_id: str | None = None
|
|
timing: dict[str, float] = Field(default_factory=dict)
|
|
|
|
|
|
class GraphNode(BaseModel):
|
|
"""Node from knowledge graph."""
|
|
id: str
|
|
labels: list[str] = Field(default_factory=list)
|
|
properties: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
|
|
class Dossier(BaseModel):
|
|
"""A dossier (tag-based collection)."""
|
|
name: str
|
|
page_count: int
|
|
|
|
|
|
class ResearchSummary(BaseModel):
|
|
"""Summary of research performed during smart-create."""
|
|
wiki_results: int = 0
|
|
web_results: int = 0
|
|
graph_entities: int = 0
|
|
keywords_extracted: int = 0
|
|
timing_ms: int = 0
|
|
|
|
|
|
class WebSearchResult(BaseModel):
|
|
"""Result from web search via /rag/search."""
|
|
title: str
|
|
url: str
|
|
content: str = "" # Full extracted text via Trafilatura
|
|
snippet: str = "" # Original search engine snippet
|
|
source: str = "" # Domain name
|
|
published_date: str | None = None
|
|
|
|
|
|
class WebSearchResponse(BaseModel):
|
|
"""Response from /rag/search endpoint."""
|
|
query: str
|
|
search_type: str
|
|
results: list[WebSearchResult] = Field(default_factory=list)
|
|
total_results: int = 0
|
|
search_time_ms: int = 0
|
|
sources_summary: str = "" # Pre-formatted markdown citations
|
|
|
|
|
|
class ContentExtractionResult(BaseModel):
|
|
"""Result from content extraction."""
|
|
url: str
|
|
title: str | None = None
|
|
content: str = ""
|
|
author: str | None = None
|
|
date: str | None = None
|
|
language: str | None = None
|
|
success: bool = True
|
|
error: str | None = None
|
|
|
|
|
|
class BatchExtractionResponse(BaseModel):
|
|
"""Response from batch content extraction."""
|
|
results: list[ContentExtractionResult] = Field(default_factory=list)
|
|
total_urls: int = 0
|
|
successful: int = 0
|
|
failed: int = 0
|
|
extraction_time_ms: int = 0
|
|
|
|
|
|
class EntityLinking(BaseModel):
|
|
"""Entity linking results from smart-create."""
|
|
forward_links: int = 0
|
|
backward_links: int = 0
|
|
pages_updated: int = 0
|
|
|
|
|
|
class SmartCreateResponse(BaseModel):
|
|
"""Response from smart-create wiki page endpoint."""
|
|
page: WikiPage
|
|
research_summary: ResearchSummary = Field(default_factory=ResearchSummary)
|
|
sources_used: int = 0
|
|
search_id: str | None = None
|
|
entity_linking: EntityLinking = Field(default_factory=EntityLinking)
|
|
|
|
|
|
# ============================================================================
|
|
# Client
|
|
# ============================================================================
|
|
|
|
class LibraryDeskClient:
|
|
"""
|
|
Async HTTP client for Library-Desk API.
|
|
|
|
Usage:
|
|
async with LibraryDeskClient() as client:
|
|
results = await client.hybrid_search("docker kubernetes")
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
base_url: str | None = None,
|
|
api_key: str | None = None,
|
|
timeout: int = 60,
|
|
):
|
|
"""
|
|
Initialize the client.
|
|
|
|
Args:
|
|
base_url: Library-desk API URL (defaults to config)
|
|
api_key: API key for authentication (defaults to config)
|
|
timeout: Request timeout in seconds
|
|
"""
|
|
self.base_url = base_url or str(config.LIBRARY_DESK_HOST)
|
|
self.api_key = api_key or config.LIBRARY_DESK_API_KEY
|
|
self.timeout = timeout
|
|
self._client: httpx.AsyncClient | None = None
|
|
|
|
async def __aenter__(self) -> "LibraryDeskClient":
|
|
"""Create HTTP client on context entry."""
|
|
headers = {}
|
|
if self.api_key:
|
|
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
|
|
self._client = httpx.AsyncClient(
|
|
base_url=self.base_url,
|
|
headers=headers,
|
|
timeout=self.timeout,
|
|
)
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
"""Close HTTP client on context exit."""
|
|
if self._client:
|
|
await self._client.aclose()
|
|
self._client = None
|
|
|
|
def _ensure_client(self) -> httpx.AsyncClient:
|
|
"""Ensure client is initialized."""
|
|
if self._client is None:
|
|
raise RuntimeError(
|
|
"Client not initialized. Use 'async with LibraryDeskClient() as client:'"
|
|
)
|
|
return self._client
|
|
|
|
# ========================================================================
|
|
# HybridRAG
|
|
# ========================================================================
|
|
|
|
async def hybrid_search(
|
|
self,
|
|
query: str,
|
|
user: str | None = None,
|
|
vector_limit: int = 10,
|
|
graph_limit: int = 10,
|
|
web_limit: int = 5,
|
|
document_limit: int = 5,
|
|
volatile_limit: int = 3,
|
|
enable_reranking: bool = True,
|
|
final_result_count: int = 10,
|
|
) -> HybridRAGResponse:
|
|
"""
|
|
Execute HybridRAG search combining vector, graph, documents, volatile, and web.
|
|
|
|
Args:
|
|
query: Search query
|
|
user: User identifier for multi-tenancy (defaults to request context)
|
|
vector_limit: Max results from vector search (wiki pages)
|
|
graph_limit: Max results from graph search
|
|
web_limit: Max results from web search (0 to disable)
|
|
document_limit: Max results from Paperless documents (0 to disable)
|
|
volatile_limit: Max results from volatile cache (0 to disable)
|
|
enable_reranking: Whether to rerank with LLM
|
|
final_result_count: Number of final results after fusion
|
|
|
|
Returns:
|
|
HybridRAGResponse with ranked results and context
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"query": query,
|
|
"config": {
|
|
"vector_limit": vector_limit,
|
|
"graph_limit": graph_limit,
|
|
"web_limit": web_limit,
|
|
"document_limit": document_limit,
|
|
"volatile_limit": volatile_limit,
|
|
"enable_documents": document_limit > 0,
|
|
"enable_volatile": volatile_limit > 0,
|
|
"enable_web": web_limit > 0,
|
|
"enable_reranking": enable_reranking,
|
|
"final_result_count": final_result_count,
|
|
},
|
|
}
|
|
|
|
logger.info("library_desk_hybrid_search", query=query, user=user)
|
|
|
|
response = await client.post(
|
|
"/query/hybrid",
|
|
json=payload,
|
|
params={"user": user},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
# Parse results
|
|
results = []
|
|
for r in data.get("results", []):
|
|
results.append(HybridSearchResult(
|
|
source=r.get("source", "unknown"),
|
|
title=r.get("title", ""),
|
|
content=r.get("content", ""),
|
|
url=r.get("url"),
|
|
score=r.get("score", 0.0),
|
|
page_id=r.get("page_id"),
|
|
metadata=r.get("metadata", {}),
|
|
))
|
|
|
|
# Handle keywords being either a list or a dict with core_keywords
|
|
raw_keywords = data.get("keywords", [])
|
|
if isinstance(raw_keywords, dict):
|
|
keywords = raw_keywords.get("core_keywords", [])
|
|
else:
|
|
keywords = raw_keywords
|
|
|
|
return HybridRAGResponse(
|
|
results=results,
|
|
keywords=keywords,
|
|
synonyms=data.get("synonyms", []),
|
|
related_dossiers=data.get("related_dossiers", []),
|
|
formatted_context=data.get("formatted_context", ""),
|
|
search_id=data.get("search_id"),
|
|
timing=data.get("timing", {}),
|
|
)
|
|
|
|
# ========================================================================
|
|
# Wiki Operations
|
|
# ========================================================================
|
|
|
|
async def search_wiki(
|
|
self,
|
|
query: str,
|
|
user: str | None = None,
|
|
limit: int = 20,
|
|
) -> list[WikiSearchResult]:
|
|
"""
|
|
Search wiki pages by text.
|
|
|
|
Args:
|
|
query: Search query
|
|
user: User identifier (defaults to request context)
|
|
limit: Maximum results
|
|
|
|
Returns:
|
|
List of matching wiki pages
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
logger.debug("library_desk_wiki_search", query=query, user=user)
|
|
|
|
response = await client.get(
|
|
"/wiki/search",
|
|
params={"q": query, "user": user, "limit": limit},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
return [WikiSearchResult(**r) for r in data.get("results", [])]
|
|
|
|
async def get_wiki_page(
|
|
self,
|
|
page_id: int,
|
|
user: str | None = None,
|
|
) -> WikiPage:
|
|
"""
|
|
Get a wiki page by ID.
|
|
|
|
Args:
|
|
page_id: Page ID
|
|
user: User identifier (defaults to request context)
|
|
|
|
Returns:
|
|
WikiPage with full content
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
response = await client.get(
|
|
f"/wiki/pages/{page_id}",
|
|
params={"user": user},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
return WikiPage(**response.json())
|
|
|
|
async def list_wiki_pages(
|
|
self,
|
|
user: str | None = None,
|
|
tag: str | None = None,
|
|
limit: int = 50,
|
|
) -> list[WikiPage]:
|
|
"""
|
|
List wiki pages, optionally filtered by tag.
|
|
|
|
Args:
|
|
user: User identifier (defaults to request context)
|
|
tag: Optional tag (dossier) to filter by
|
|
limit: Maximum pages to return
|
|
|
|
Returns:
|
|
List of wiki pages
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
params: dict[str, Any] = {"user": user, "limit": limit}
|
|
if tag:
|
|
params["tag"] = tag
|
|
|
|
response = await client.get("/wiki/pages", params=params)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
return [WikiPage(**p) for p in data.get("pages", [])]
|
|
|
|
async def create_wiki_page(
|
|
self,
|
|
title: str,
|
|
path: str,
|
|
content: str,
|
|
user: str | None = None,
|
|
description: str = "",
|
|
tags: list[str] | None = None,
|
|
) -> WikiPage:
|
|
"""
|
|
Create a new wiki page.
|
|
|
|
Args:
|
|
title: Page title
|
|
path: Page path (e.g., "/projects/my-project")
|
|
content: Markdown content
|
|
user: User identifier (defaults to request context)
|
|
description: Short description
|
|
tags: List of tags (dossiers)
|
|
|
|
Returns:
|
|
Created WikiPage
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"title": title,
|
|
"path": path,
|
|
"content": content,
|
|
"user": user,
|
|
"description": description,
|
|
"tags": tags or [],
|
|
}
|
|
|
|
logger.info("library_desk_create_page", title=title, path=path)
|
|
|
|
response = await client.post("/wiki/pages", json=payload)
|
|
response.raise_for_status()
|
|
|
|
return WikiPage(**response.json())
|
|
|
|
async def update_wiki_page(
|
|
self,
|
|
page_id: int,
|
|
user: str | None = None,
|
|
content: str | None = None,
|
|
title: str | None = None,
|
|
tags: list[str] | None = None,
|
|
description: str | None = None,
|
|
) -> WikiPage:
|
|
"""
|
|
Update an existing wiki page.
|
|
|
|
Supports partial updates - only provided fields are updated.
|
|
Automatically triggers vector re-indexing and graph extraction.
|
|
|
|
Args:
|
|
page_id: ID of the page to update
|
|
user: User identifier (defaults to request context)
|
|
content: New content (optional)
|
|
title: New title (optional)
|
|
tags: New tags list (optional)
|
|
description: New description (optional)
|
|
|
|
Returns:
|
|
Updated WikiPage
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
# Build update payload with only provided fields
|
|
update_data: dict[str, Any] = {}
|
|
if content is not None:
|
|
update_data["content"] = content
|
|
if title is not None:
|
|
update_data["title"] = title
|
|
if tags is not None:
|
|
update_data["tags"] = tags
|
|
if description is not None:
|
|
update_data["description"] = description
|
|
|
|
logger.info(
|
|
"library_desk_update_page",
|
|
page_id=page_id,
|
|
fields=list(update_data.keys()),
|
|
)
|
|
|
|
response = await client.put(
|
|
f"/wiki/pages/{page_id}",
|
|
params={"user": user},
|
|
json=update_data,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
return WikiPage(**response.json())
|
|
|
|
async def smart_create_wiki_page(
|
|
self,
|
|
topic: str,
|
|
tags: list[str],
|
|
user: str | None = None,
|
|
path: str | None = None,
|
|
include_web_research: bool = True,
|
|
include_wiki_search: bool = True,
|
|
) -> SmartCreateResponse:
|
|
"""
|
|
Create a wiki page with HybridRAG research.
|
|
|
|
This endpoint:
|
|
1. Searches existing wiki, knowledge graph, and web for context
|
|
2. Uses LLM to synthesize findings into structured content
|
|
3. Creates the page with proper attribution
|
|
4. Automatically links entities bidirectionally
|
|
|
|
Args:
|
|
topic: The topic to research and create a page about
|
|
tags: List of tags (dossiers) for the page
|
|
user: User identifier
|
|
path: Optional custom path (auto-generated from topic if not provided)
|
|
include_web_research: Whether to include web search results
|
|
include_wiki_search: Whether to include existing wiki content
|
|
|
|
Returns:
|
|
SmartCreateResponse with page and research metadata
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload: dict[str, Any] = {
|
|
"topic": topic,
|
|
"tags": tags,
|
|
"user": user,
|
|
"include_web_research": include_web_research,
|
|
"include_wiki_search": include_wiki_search,
|
|
}
|
|
if path is not None:
|
|
payload["path"] = path
|
|
|
|
logger.info(
|
|
"library_desk_smart_create",
|
|
topic=topic,
|
|
tags=tags,
|
|
include_web=include_web_research,
|
|
)
|
|
|
|
response = await client.post("/wiki/pages/smart-create", json=payload)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
# Parse nested response
|
|
page = WikiPage(**data.get("page", {}))
|
|
research_summary = ResearchSummary(**data.get("research_summary", {}))
|
|
entity_linking = EntityLinking(**data.get("entity_linking", {}))
|
|
|
|
return SmartCreateResponse(
|
|
page=page,
|
|
research_summary=research_summary,
|
|
sources_used=data.get("sources_used", 0),
|
|
search_id=data.get("search_id"),
|
|
entity_linking=entity_linking,
|
|
)
|
|
|
|
async def list_dossiers(
|
|
self,
|
|
user: str | None = None,
|
|
) -> list[Dossier]:
|
|
"""
|
|
List all dossiers (tag collections) for a user.
|
|
|
|
Args:
|
|
user: User identifier (defaults to request context)
|
|
|
|
Returns:
|
|
List of dossiers with page counts
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
response = await client.get(
|
|
"/wiki/dossiers",
|
|
params={"user": user},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
return [Dossier(**d) for d in data.get("dossiers", [])]
|
|
|
|
# ========================================================================
|
|
# Vector Search
|
|
# ========================================================================
|
|
|
|
async def semantic_search(
|
|
self,
|
|
query: str,
|
|
user: str | None = None,
|
|
limit: int = 10,
|
|
score_threshold: float = 0.5,
|
|
) -> list[VectorSearchResult]:
|
|
"""
|
|
Perform semantic (vector) search over documents.
|
|
|
|
Args:
|
|
query: Natural language query
|
|
user: User identifier (defaults to request context)
|
|
limit: Maximum results
|
|
score_threshold: Minimum similarity score
|
|
|
|
Returns:
|
|
List of matching document chunks with scores
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"query": query,
|
|
"user": user,
|
|
"limit": limit,
|
|
"score_threshold": score_threshold,
|
|
}
|
|
|
|
logger.debug("library_desk_semantic_search", query=query)
|
|
|
|
response = await client.post("/vector/search", json=payload)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
return [VectorSearchResult(**r) for r in data.get("results", [])]
|
|
|
|
# ========================================================================
|
|
# Knowledge Graph
|
|
# ========================================================================
|
|
|
|
async def query_graph(
|
|
self,
|
|
cypher_query: str,
|
|
user: str | None = None,
|
|
parameters: dict[str, Any] | None = None,
|
|
) -> list[dict[str, Any]]:
|
|
"""
|
|
Execute a Cypher query on the knowledge graph.
|
|
|
|
Note: Query is automatically scoped to user's data.
|
|
|
|
Args:
|
|
cypher_query: Cypher query string
|
|
user: User identifier (defaults to request context)
|
|
parameters: Query parameters
|
|
|
|
Returns:
|
|
List of result records
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"query": cypher_query,
|
|
"user": user,
|
|
"parameters": parameters or {},
|
|
}
|
|
|
|
logger.debug("library_desk_graph_query", query=cypher_query[:100])
|
|
|
|
response = await client.post("/graph/query", json=payload)
|
|
response.raise_for_status()
|
|
|
|
return response.json().get("records", [])
|
|
|
|
async def list_graph_nodes(
|
|
self,
|
|
user: str | None = None,
|
|
node_type: str | None = None,
|
|
limit: int = 100,
|
|
) -> list[GraphNode]:
|
|
"""
|
|
List nodes in the knowledge graph.
|
|
|
|
Args:
|
|
user: User identifier (defaults to request context)
|
|
node_type: Optional filter by type (Document, Person, Concept, etc.)
|
|
limit: Maximum nodes
|
|
|
|
Returns:
|
|
List of graph nodes
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
params: dict[str, Any] = {"user": user, "limit": limit}
|
|
if node_type:
|
|
params["node_type"] = node_type
|
|
|
|
response = await client.get("/graph/nodes", params=params)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
return [GraphNode(**n) for n in data.get("nodes", [])]
|
|
|
|
async def get_graph_node(
|
|
self,
|
|
node_id: str,
|
|
user: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""
|
|
Get detailed information about a graph node.
|
|
|
|
Args:
|
|
node_id: Node ID
|
|
user: User identifier (defaults to request context)
|
|
|
|
Returns:
|
|
Node with relationships and connected nodes
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
response = await client.get(
|
|
f"/graph/nodes/{node_id}",
|
|
params={"user": user},
|
|
)
|
|
response.raise_for_status()
|
|
|
|
return response.json()
|
|
|
|
# ========================================================================
|
|
# Health Check
|
|
# ========================================================================
|
|
|
|
async def health_check(self) -> bool:
|
|
"""
|
|
Check if library-desk is healthy.
|
|
|
|
Returns:
|
|
True if healthy, False otherwise
|
|
"""
|
|
try:
|
|
client = self._ensure_client()
|
|
response = await client.get("/health")
|
|
return response.status_code == 200
|
|
except Exception as e:
|
|
logger.warning("library_desk_health_check_failed", error=str(e))
|
|
return False
|
|
|
|
# ========================================================================
|
|
# RAG Search (Web Search with Content Extraction)
|
|
# ========================================================================
|
|
|
|
async def search_web(
|
|
self,
|
|
query: str,
|
|
user: str | None = None,
|
|
search_type: str = "web",
|
|
limit: int = 10,
|
|
) -> WebSearchResponse:
|
|
"""
|
|
Search the web and extract content from results.
|
|
|
|
Uses SearXNG for search and Trafilatura for content extraction.
|
|
Returns both snippets and full extracted text.
|
|
|
|
Args:
|
|
query: Search query (1-500 chars)
|
|
user: User identifier for tracking
|
|
search_type: "web", "news", or "images"
|
|
limit: Number of results (1-20)
|
|
|
|
Returns:
|
|
WebSearchResponse with results and pre-formatted sources
|
|
"""
|
|
user = user or get_user()
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"query": query,
|
|
"search_type": search_type,
|
|
"limit": limit,
|
|
"user": user or "tatlock-librarian",
|
|
}
|
|
|
|
logger.info("library_desk_web_search", query=query, limit=limit)
|
|
|
|
response = await client.post("/rag/search", json=payload, timeout=30.0)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
results = [
|
|
WebSearchResult(
|
|
title=r.get("title", ""),
|
|
url=r.get("url", ""),
|
|
content=r.get("content", ""),
|
|
snippet=r.get("snippet", ""),
|
|
source=r.get("source", ""),
|
|
published_date=r.get("published_date"),
|
|
)
|
|
for r in data.get("results", [])
|
|
]
|
|
|
|
return WebSearchResponse(
|
|
query=data.get("query", query),
|
|
search_type=data.get("search_type", search_type),
|
|
results=results,
|
|
total_results=data.get("total_results", len(results)),
|
|
search_time_ms=data.get("search_time_ms", 0),
|
|
sources_summary=data.get("sources_summary", ""),
|
|
)
|
|
|
|
# ========================================================================
|
|
# Content Extraction
|
|
# ========================================================================
|
|
|
|
async def extract_content(
|
|
self,
|
|
url: str,
|
|
include_metadata: bool = True,
|
|
max_length: int = 5000,
|
|
) -> ContentExtractionResult:
|
|
"""
|
|
Extract main content from a URL.
|
|
|
|
Uses Trafilatura for intelligent content extraction,
|
|
removing boilerplate, ads, and navigation.
|
|
|
|
Note: Uses soft failure pattern - check result.success field.
|
|
|
|
Args:
|
|
url: URL to extract content from
|
|
include_metadata: Whether to extract author, date, etc.
|
|
max_length: Maximum content length
|
|
|
|
Returns:
|
|
ContentExtractionResult (check .success and .error fields)
|
|
"""
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"url": url,
|
|
"include_metadata": include_metadata,
|
|
"max_length": max_length,
|
|
}
|
|
|
|
logger.debug("library_desk_extract_content", url=url)
|
|
|
|
response = await client.post("/content/extract", json=payload, timeout=30.0)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
result = data.get("result", {})
|
|
|
|
return ContentExtractionResult(
|
|
url=result.get("url", url),
|
|
title=result.get("title"),
|
|
content=result.get("content", ""),
|
|
author=result.get("author"),
|
|
date=result.get("date"),
|
|
language=result.get("language"),
|
|
success=result.get("success", False),
|
|
error=result.get("error"),
|
|
)
|
|
|
|
async def extract_content_batch(
|
|
self,
|
|
urls: list[str],
|
|
include_metadata: bool = True,
|
|
max_length: int = 2000,
|
|
) -> BatchExtractionResponse:
|
|
"""
|
|
Extract content from multiple URLs in parallel.
|
|
|
|
More efficient than sequential calls. Max 20 URLs per batch.
|
|
|
|
Note: Uses soft failure pattern - individual failures don't
|
|
throw errors, check each result's .success field.
|
|
|
|
Args:
|
|
urls: List of URLs to extract (max 20)
|
|
include_metadata: Whether to extract author, date, etc.
|
|
max_length: Maximum content length per URL
|
|
|
|
Returns:
|
|
BatchExtractionResponse with results and stats
|
|
"""
|
|
client = self._ensure_client()
|
|
|
|
payload = {
|
|
"urls": urls[:20], # Server limit
|
|
"include_metadata": include_metadata,
|
|
"max_length": max_length,
|
|
}
|
|
|
|
logger.info("library_desk_extract_batch", url_count=len(urls))
|
|
|
|
response = await client.post(
|
|
"/content/extract/batch",
|
|
json=payload,
|
|
timeout=60.0, # Longer timeout for batch
|
|
)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
results = [
|
|
ContentExtractionResult(
|
|
url=r.get("url", ""),
|
|
title=r.get("title"),
|
|
content=r.get("content", ""),
|
|
author=r.get("author"),
|
|
date=r.get("date"),
|
|
language=r.get("language"),
|
|
success=r.get("success", False),
|
|
error=r.get("error"),
|
|
)
|
|
for r in data.get("results", [])
|
|
]
|
|
|
|
return BatchExtractionResponse(
|
|
results=results,
|
|
total_urls=data.get("total_urls", len(urls)),
|
|
successful=data.get("successful", 0),
|
|
failed=data.get("failed", 0),
|
|
extraction_time_ms=data.get("extraction_time_ms", 0),
|
|
)
|
|
|
|
|
|
# Global client factory
|
|
async def get_library_client() -> LibraryDeskClient:
|
|
"""
|
|
Get a library-desk client instance.
|
|
|
|
Usage:
|
|
async with get_library_client() as client:
|
|
results = await client.hybrid_search("query")
|
|
"""
|
|
return LibraryDeskClient()
|