Initial commit: library-desk service extraction from portainer-core
Build and Push / build (release) Successful in 36s

This commit is contained in:
2025-12-11 17:28:23 +01:00
commit 95852190ba
59 changed files with 17146 additions and 0 deletions
View File
+49
View File
@@ -0,0 +1,49 @@
"""
Knowledge Consolidation models for Librarian processing.
Used by the consolidation endpoint to process SearchQuery nodes
and consolidate knowledge into wiki pages.
"""
from pydantic import BaseModel, Field
from typing import List, Optional, Dict, Any
class ConsolidationRequest(BaseModel):
"""Request for knowledge consolidation from search results."""
process_limit: int = Field(default=10, ge=1, le=100, description="Max searches to process")
lookback_days: int = Field(default=7, ge=1, le=90, description="Process searches from last N days")
min_web_results: int = Field(default=2, ge=1, le=20, description="Minimum web results needed")
dry_run: bool = Field(default=False, description="If true, analyze but don't create pages")
class SearchQueryInfo(BaseModel):
"""Information about a search query to process."""
id: str
query: str
user: str
timestamp: str
total_results: int
web_count: int
keywords: List[str] = []
class ConsolidationResult(BaseModel):
"""Result of processing a single search query."""
search_id: str
query: str
pages_created: int = 0
pages_updated: int = 0
entities_added: int = 0
error: Optional[str] = None
class ConsolidationResponse(BaseModel):
"""Response from knowledge consolidation."""
total_found: int = Field(description="Total unprocessed searches found")
processed_count: int = Field(description="Successfully processed searches")
pages_created: int = Field(description="New wiki pages created")
pages_updated: int = Field(description="Existing pages updated")
entities_added: int = Field(description="New entities added to graph")
errors: List[str] = Field(default=[], description="Error messages")
results: List[ConsolidationResult] = Field(description="Per-search results")
dry_run: bool = Field(description="Whether this was a dry run")
+126
View File
@@ -0,0 +1,126 @@
"""
Graph models for Library Desk Neo4j operations.
Provides models for knowledge graph nodes, relationships, and queries.
"""
from pydantic import BaseModel, Field
from typing import List, Dict, Any, Optional
from datetime import datetime
class GraphNode(BaseModel):
"""Graph node representation."""
id: str = Field(..., description="Node ID")
labels: List[str] = Field(..., description="Node labels")
properties: Dict[str, Any] = Field(default_factory=dict, description="Node properties")
class GraphRelationship(BaseModel):
"""Graph relationship representation."""
id: str = Field(..., description="Relationship ID")
type: str = Field(..., description="Relationship type")
start_node: str = Field(..., description="Start node ID")
end_node: str = Field(..., description="End node ID")
properties: Dict[str, Any] = Field(default_factory=dict, description="Relationship properties")
class GraphNodeDetail(BaseModel):
"""Detailed node with relationships."""
node: GraphNode = Field(..., description="Node data")
relationships: List[GraphRelationship] = Field(
default_factory=list,
description="Connected relationships"
)
related_nodes: List[GraphNode] = Field(
default_factory=list,
description="Connected nodes"
)
class CypherQueryRequest(BaseModel):
"""Request to execute a Cypher query."""
query: str = Field(..., description="Cypher query to execute")
parameters: Dict[str, Any] = Field(
default_factory=dict,
description="Query parameters"
)
user: str = Field(
default="jpmschweitzer",
description="User for filtering (automatically scopes query)"
)
class CypherQueryResponse(BaseModel):
"""Response from Cypher query execution."""
results: List[Dict[str, Any]] = Field(..., description="Query results")
count: int = Field(..., description="Number of results")
query_time_ms: float = Field(..., description="Query execution time in milliseconds")
class UpdateFromPageRequest(BaseModel):
"""Request to update graph from a wiki page."""
page_id: int = Field(..., description="Wiki page ID to process")
user: str = Field(
default="jpmschweitzer",
description="User identifier for namespace scoping"
)
force_refresh: bool = Field(
default=False,
description="Force re-extraction even if page hasn't changed"
)
class EntityMention(BaseModel):
"""Extracted entity mention."""
text: str = Field(..., description="Entity text")
type: str = Field(..., description="Entity type (Person, Project, Concept, etc.)")
confidence: float = Field(default=1.0, description="Extraction confidence (0-1)")
class GraphUpdateSummary(BaseModel):
"""Summary of graph update operation."""
page_id: int = Field(..., description="Page ID processed")
page_title: str = Field(..., description="Page title")
nodes_created: int = Field(default=0, description="New nodes created")
nodes_updated: int = Field(default=0, description="Existing nodes updated")
relationships_created: int = Field(default=0, description="New relationships created")
entities_extracted: List[EntityMention] = Field(
default_factory=list,
description="Entities extracted from page"
)
processing_time_ms: float = Field(..., description="Processing time in milliseconds")
success: bool = Field(default=True, description="Whether update succeeded")
error_message: Optional[str] = Field(default=None, description="Error message if failed")
class NodeListResponse(BaseModel):
"""Response for node listing."""
nodes: List[GraphNode] = Field(..., description="List of nodes")
total: int = Field(..., description="Total number of nodes")
user: str = Field(..., description="User filter applied")
class MindMapNode(BaseModel):
"""Mind map node for visualization."""
id: str = Field(..., description="Node ID")
label: str = Field(..., description="Node label/name")
type: str = Field(..., description="Node type")
size: int = Field(default=10, description="Visual size")
color: Optional[str] = Field(default=None, description="Node color")
class MindMapLink(BaseModel):
"""Mind map link for visualization."""
source: str = Field(..., description="Source node ID")
target: str = Field(..., description="Target node ID")
type: str = Field(..., description="Relationship type")
strength: float = Field(default=1.0, description="Link strength")
class MindMapResponse(BaseModel):
"""Mind map data for D3.js or similar visualization."""
nodes: List[MindMapNode] = Field(..., description="Graph nodes")
links: List[MindMapLink] = Field(..., description="Graph edges")
center_node: str = Field(..., description="Central node ID")
depth: int = Field(..., description="Traversal depth")
+87
View File
@@ -0,0 +1,87 @@
"""
HybridRAG models for multi-source search with RRF fusion.
Combines vector search (Qdrant), knowledge graph (Neo4j), and web search (SearXNG)
with Reciprocal Rank Fusion and LLM re-ranking.
"""
from pydantic import BaseModel, Field
from typing import List, Optional, Dict, Any
class HybridRAGConfig(BaseModel):
"""Configuration for HybridRAG query."""
vector_limit: int = Field(default=10, ge=1, le=50, description="Max vector results")
graph_limit: int = Field(default=10, ge=1, le=50, description="Max graph results")
web_limit: int = Field(default=5, ge=1, le=20, description="Max web results")
enable_vector: bool = Field(default=True, description="Enable vector search")
enable_graph: bool = Field(default=True, description="Enable graph search")
enable_web: bool = Field(default=True, description="Enable web search")
enable_reranking: bool = Field(default=True, description="Enable LLM re-ranking")
enable_enrichment: bool = Field(default=True, description="Enable graph enrichment")
final_result_count: int = Field(default=10, ge=1, le=50, description="Final results to return")
rrf_k: int = Field(default=60, ge=1, le=100, description="RRF constant")
class RelatedDossier(BaseModel):
"""Related document metadata from graph enrichment."""
page_id: int
title: str
path: str
tag: str
shared_entities: int
class HybridRAGResult(BaseModel):
"""Single result from HybridRAG query."""
source_type: str = Field(..., description="Source: 'vector', 'graph', 'web'")
title: str
content: str
url: Optional[str] = Field(None, description="URL for web results")
page_id: Optional[int] = Field(None, description="Page ID for wiki results")
page_path: Optional[str] = Field(None, description="Wiki page path")
rrf_score: float = Field(..., description="Reciprocal Rank Fusion score")
final_rank: int = Field(..., description="Final rank after re-ranking")
sources: List[str] = Field(..., description="Which sources included this result")
related_dossiers: List[RelatedDossier] = Field(default=[], description="Related documents via shared entities")
metadata: Dict[str, Any] = Field(default={}, description="Additional metadata")
class TimingBreakdown(BaseModel):
"""Performance timing breakdown for each phase."""
query_enhancement_ms: float = Field(..., description="Phase 0: Keyword/synonym extraction")
vector_ms: float = Field(..., description="Phase 1: Vector search")
graph_ms: float = Field(..., description="Phase 1: Graph search")
web_ms: float = Field(..., description="Phase 1: Web search")
fusion_ms: float = Field(..., description="Phase 2: RRF fusion")
enrichment_ms: float = Field(..., description="Phase 3: Graph enrichment")
reranking_ms: float = Field(..., description="Phase 4: LLM re-ranking")
persistence_ms: float = Field(..., description="Phase 6: Search persistence")
total_ms: float = Field(..., description="Total end-to-end time")
class KeywordExtraction(BaseModel):
"""Extracted keywords and synonyms from query enhancement."""
core_keywords: List[str] = Field(default=[], description="Primary keywords")
entities: List[str] = Field(default=[], description="Named entities")
synonyms: Dict[str, List[str]] = Field(default={}, description="Synonyms map")
expansions: Dict[str, List[str]] = Field(default={}, description="Abbreviation expansions")
class HybridRAGResponse(BaseModel):
"""Response from HybridRAG query."""
query: str = Field(..., description="Original search query")
keywords: KeywordExtraction = Field(..., description="Extracted keywords/synonyms")
results: List[HybridRAGResult] = Field(..., description="Ranked search results")
context: str = Field(..., description="Formatted context for LLM consumption")
source_counts: Dict[str, int] = Field(..., description="Result counts by source")
total_results: int = Field(..., description="Total number of results")
timing: TimingBreakdown = Field(..., description="Performance breakdown")
config_used: HybridRAGConfig = Field(..., description="Configuration used")
search_id: Optional[str] = Field(None, description="Search ID for Librarian tracking")
class HybridRAGRequest(BaseModel):
"""Request for HybridRAG query."""
query: str = Field(..., min_length=1, max_length=500, description="Search query")
config: Optional[HybridRAGConfig] = Field(None, description="Custom configuration")
+67
View File
@@ -0,0 +1,67 @@
"""
Pydantic models for Document Ingestion system.
"""
from pydantic import BaseModel, Field
from typing import Optional, List, Dict, Any
from datetime import datetime
class IngestionRequest(BaseModel):
"""Request to ingest a wiki page."""
page_id: int = Field(..., description="Wiki page ID to ingest")
user: str = Field(default="jpmschweitzer", description="User identifier")
force_refresh: bool = Field(
default=False,
description="Force re-ingestion even if page hasn't changed"
)
skip_vectors: bool = Field(default=False, description="Skip vector embedding generation")
skip_graph: bool = Field(default=False, description="Skip graph entity extraction")
class BatchIngestionRequest(BaseModel):
"""Request to ingest multiple wiki pages."""
page_ids: List[int] = Field(..., description="List of wiki page IDs to ingest")
user: str = Field(default="jpmschweitzer", description="User identifier")
force_refresh: bool = Field(default=False)
skip_vectors: bool = Field(default=False)
skip_graph: bool = Field(default=False)
max_concurrent: int = Field(
default=3,
ge=1,
le=10,
description="Maximum concurrent ingestion tasks"
)
class IngestionResult(BaseModel):
"""Result of a single page ingestion."""
page_id: int
page_title: str
page_path: Optional[str] = None
success: bool
error: Optional[str] = None
vector_chunks_created: int = 0
graph_entities_extracted: int = 0
graph_relationships_created: int = 0
processing_time_ms: float
class BatchIngestionResult(BaseModel):
"""Result of batch ingestion."""
total_pages: int
successful: int
failed: int
results: List[IngestionResult]
total_processing_time_ms: float
class IngestionStatus(BaseModel):
"""Status of an ingestion job."""
job_id: str
status: str # "queued", "processing", "completed", "failed"
progress: int # 0-100
page_id: Optional[int] = None
result: Optional[IngestionResult] = None
created_at: datetime
started_at: Optional[datetime] = None
completed_at: Optional[datetime] = None
+61
View File
@@ -0,0 +1,61 @@
"""
Tool catalog models for Library Desk.
Provides simplified tool definitions optimized for AI agent consumption.
"""
from pydantic import BaseModel, Field
from typing import List, Dict, Any, Optional
from enum import Enum
class ParameterType(str, Enum):
"""Parameter data types."""
STRING = "string"
INTEGER = "integer"
BOOLEAN = "boolean"
ARRAY = "array"
OBJECT = "object"
class ToolParameter(BaseModel):
"""Tool parameter definition."""
name: str = Field(..., description="Parameter name")
type: ParameterType = Field(..., description="Parameter type")
description: str = Field(..., description="Parameter description")
required: bool = Field(default=False, description="Whether parameter is required")
default: Optional[Any] = Field(default=None, description="Default value if not required")
example: Optional[Any] = Field(default=None, description="Example value")
class ToolDefinition(BaseModel):
"""Individual tool definition."""
name: str = Field(..., description="Tool identifier (e.g., 'wiki_create_page')")
category: str = Field(..., description="Tool category (e.g., 'wiki', 'graph')")
description: str = Field(..., description="What this tool does")
method: str = Field(..., description="HTTP method (GET, POST, PUT, DELETE)")
endpoint: str = Field(..., description="API endpoint path")
parameters: List[ToolParameter] = Field(default_factory=list, description="Tool parameters")
returns: str = Field(..., description="What the tool returns")
example: Optional[Dict[str, Any]] = Field(default=None, description="Example request")
fast: bool = Field(default=True, description="Whether operation completes quickly (<5s)")
class CategoryInfo(BaseModel):
"""Tool category information."""
name: str = Field(..., description="Category name")
description: str = Field(..., description="Category description")
tool_count: int = Field(..., description="Number of tools in category")
class ToolCatalog(BaseModel):
"""Complete tool catalog response."""
service: str = Field(default="library-desk", description="Service name")
version: str = Field(default="1.0.0", description="API version")
base_url: str = Field(..., description="Base URL for API")
categories: List[CategoryInfo] = Field(..., description="Available categories")
tools: List[ToolDefinition] = Field(..., description="All available tools")
authentication: str = Field(
default="Bearer token via Authorization header",
description="Authentication method"
)
+100
View File
@@ -0,0 +1,100 @@
"""
Vector models for Library Desk Qdrant operations.
Provides models for semantic search, document chunks, and embeddings.
"""
from pydantic import BaseModel, Field
from typing import List, Dict, Any, Optional
class DocumentChunk(BaseModel):
"""Document chunk with embedding."""
chunk_id: str = Field(..., description="Unique chunk ID (page_id:chunk_index)")
page_id: int = Field(..., description="Wiki page ID")
chunk_index: int = Field(..., description="Chunk index within document")
content: str = Field(..., description="Chunk text content")
metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
class SearchResult(BaseModel):
"""Semantic search result."""
chunk_id: str = Field(..., description="Chunk ID")
page_id: int = Field(..., description="Wiki page ID")
page_title: Optional[str] = Field(None, description="Page title")
page_path: Optional[str] = Field(None, description="Page path")
chunk_index: int = Field(..., description="Chunk index")
content: str = Field(..., description="Chunk content")
score: float = Field(..., description="Similarity score (0-1)")
metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
class SearchRequest(BaseModel):
"""Semantic search request."""
query: str = Field(..., min_length=1, description="Search query")
user: str = Field(default="jpmschweitzer", description="User identifier")
limit: int = Field(default=10, ge=1, le=100, description="Maximum results")
score_threshold: float = Field(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score")
class SearchResponse(BaseModel):
"""Semantic search response."""
query: str = Field(..., description="Search query")
results: List[SearchResult] = Field(..., description="Search results")
total: int = Field(..., description="Number of results")
user: str = Field(..., description="User filter applied")
class VectorUpdateRequest(BaseModel):
"""Request to update vectors from a wiki page."""
page_id: int = Field(..., description="Wiki page ID to process")
user: str = Field(
default="jpmschweitzer",
description="User identifier for namespace scoping"
)
force_refresh: bool = Field(
default=False,
description="Force re-embedding even if page hasn't changed"
)
class VectorUpdateSummary(BaseModel):
"""Summary of vector update operation."""
page_id: int = Field(..., description="Page ID processed")
page_title: str = Field(..., description="Page title")
chunks_created: int = Field(default=0, description="New chunks created")
chunks_updated: int = Field(default=0, description="Existing chunks updated")
chunks_deleted: int = Field(default=0, description="Old chunks deleted")
total_chunks: int = Field(default=0, description="Total chunks for this page")
embedding_dim: int = Field(default=768, description="Embedding dimensionality")
processing_time_ms: float = Field(..., description="Processing time in milliseconds")
success: bool = Field(default=True, description="Whether update succeeded")
error_message: Optional[str] = Field(default=None, description="Error message if failed")
class CollectionInfo(BaseModel):
"""Qdrant collection information."""
name: str = Field(..., description="Collection name")
vectors_count: int = Field(..., description="Number of vectors")
points_count: int = Field(..., description="Number of points")
segments_count: int = Field(..., description="Number of segments")
class CollectionListResponse(BaseModel):
"""List of Qdrant collections."""
collections: List[CollectionInfo] = Field(..., description="List of collections")
total: int = Field(..., description="Total number of collections")
class DeletePageChunksRequest(BaseModel):
"""Request to delete all chunks for a page."""
page_id: int = Field(..., description="Wiki page ID")
user: str = Field(default="jpmschweitzer", description="User identifier")
class DeletePageChunksResponse(BaseModel):
"""Response from deleting page chunks."""
page_id: int = Field(..., description="Page ID")
chunks_deleted: int = Field(..., description="Number of chunks deleted")
success: bool = Field(..., description="Whether deletion succeeded")
message: str = Field(..., description="Result message")
+192
View File
@@ -0,0 +1,192 @@
"""
Pydantic models for Wiki.js operations.
Models for:
- Wiki pages (CRUD operations)
- Dossiers (tag-based collections)
- Search results
"""
from pydantic import BaseModel, Field, field_validator
from typing import Optional, List
from datetime import datetime
# Base models
class WikiPageBase(BaseModel):
"""Base wiki page fields."""
title: str = Field(..., min_length=1, max_length=500, description="Page title")
description: Optional[str] = Field(None, max_length=1000, description="Page description")
tags: List[str] = Field(default_factory=list, description="Tags (for dossier organization)")
is_published: bool = Field(default=True, description="Whether page is published")
@field_validator("tags")
@classmethod
def validate_tags(cls, v: List[str]) -> List[str]:
"""Validate and clean tags."""
# Remove empty tags and strip whitespace
cleaned = [tag.strip() for tag in v if tag.strip()]
# Ensure uniqueness
return list(set(cleaned))
class WikiPageCreate(WikiPageBase):
"""Request model for creating a wiki page."""
content: str = Field(..., description="Page content (markdown)")
path: str = Field(..., min_length=1, max_length=500, description="Page path (e.g., '/projects/library-desk')")
editor: str = Field(default="markdown", description="Editor type")
user: Optional[str] = Field(None, description="User identifier (defaults to configured user)")
@field_validator("path")
@classmethod
def validate_path(cls, v: str) -> str:
"""Validate page path."""
# Ensure path starts with /
if not v.startswith("/"):
v = f"/{v}"
# Remove trailing slash
if v.endswith("/") and v != "/":
v = v.rstrip("/")
return v
class WikiPageUpdate(BaseModel):
"""Request model for updating a wiki page."""
content: Optional[str] = Field(None, description="Updated content")
title: Optional[str] = Field(None, min_length=1, max_length=500, description="Updated title")
description: Optional[str] = Field(None, max_length=1000, description="Updated description")
tags: Optional[List[str]] = Field(None, description="Updated tags")
@field_validator("tags")
@classmethod
def validate_tags(cls, v: Optional[List[str]]) -> Optional[List[str]]:
"""Validate and clean tags."""
if v is None:
return None
cleaned = [tag.strip() for tag in v if tag.strip()]
return list(set(cleaned))
class WikiPage(WikiPageBase):
"""Response model for a wiki page."""
id: int = Field(..., description="Page ID")
path: str = Field(..., description="Page path")
content: Optional[str] = Field(None, description="Page content")
created_at: Optional[str] = Field(None, description="Creation timestamp")
updated_at: Optional[str] = Field(None, description="Last update timestamp")
editor: Optional[str] = Field(None, description="Editor type")
class Config:
from_attributes = True
class WikiPageSummary(BaseModel):
"""Summarized wiki page (for list responses)."""
id: int = Field(..., description="Page ID")
path: str = Field(..., description="Page path")
title: str = Field(..., description="Page title")
description: Optional[str] = Field(None, description="Page description")
tags: List[str] = Field(default_factory=list, description="Page tags")
updated_at: Optional[str] = Field(None, description="Last update timestamp")
is_published: bool = Field(..., description="Publication status")
class WikiPageList(BaseModel):
"""Response model for list of pages."""
pages: List[WikiPageSummary] = Field(..., description="List of pages")
total: int = Field(..., description="Total number of pages")
filtered_by_tag: Optional[str] = Field(None, description="Tag filter applied")
user: str = Field(..., description="User namespace")
# Dossier models
class DossierCreate(BaseModel):
"""Request model for creating a dossier."""
name: str = Field(..., min_length=1, max_length=100, description="Dossier name (becomes a tag)")
title: str = Field(..., min_length=1, max_length=200, description="Human-readable title")
description: str = Field(..., min_length=1, description="Dossier description")
create_index_page: bool = Field(default=True, description="Create an index page for the dossier")
user: Optional[str] = Field(None, description="User identifier")
@field_validator("name")
@classmethod
def validate_name(cls, v: str) -> str:
"""Validate dossier name (will be used as tag)."""
# Convert to lowercase, replace spaces with hyphens
name = v.lower().strip()
name = name.replace(" ", "-")
# Remove special characters except hyphens and underscores
name = "".join(c for c in name if c.isalnum() or c in "-_")
if not name:
raise ValueError("Dossier name must contain alphanumeric characters")
return name
class DossierInfo(BaseModel):
"""Response model for dossier information."""
name: str = Field(..., description="Dossier name (tag)")
title: str = Field(..., description="Dossier title")
description: str = Field(..., description="Dossier description")
page_count: int = Field(..., description="Number of pages in dossier")
index_page_id: Optional[int] = Field(None, description="ID of index page")
index_page_path: Optional[str] = Field(None, description="Path to index page")
created_at: Optional[str] = Field(None, description="Creation timestamp")
class DossierList(BaseModel):
"""Response model for list of dossiers."""
dossiers: List[DossierInfo] = Field(..., description="List of dossiers")
total: int = Field(..., description="Total number of dossiers")
user: str = Field(..., description="User namespace")
# Search models
class WikiSearchResult(BaseModel):
"""Search result item."""
id: int = Field(..., description="Page ID")
path: str = Field(..., description="Page path")
title: str = Field(..., description="Page title")
description: Optional[str] = Field(None, description="Page description")
relevance: Optional[float] = Field(None, description="Search relevance score")
class WikiSearchResponse(BaseModel):
"""Response model for search results."""
results: List[WikiSearchResult] = Field(..., description="Search results")
query: str = Field(..., description="Search query")
total: int = Field(..., description="Total results found")
# Move/rename models
class WikiPageMove(BaseModel):
"""Request model for moving/renaming a page."""
new_path: str = Field(..., min_length=1, description="New page path")
locale: str = Field(default="en", description="Page locale")
@field_validator("new_path")
@classmethod
def validate_new_path(cls, v: str) -> str:
"""Validate new path."""
if not v.startswith("/"):
v = f"/{v}"
if v.endswith("/") and v != "/":
v = v.rstrip("/")
return v
# Response models for operations
class WikiOperationResponse(BaseModel):
"""Generic response for wiki operations."""
success: bool = Field(..., description="Whether operation succeeded")
message: str = Field(..., description="Operation message")
page_id: Optional[int] = Field(None, description="Page ID (if applicable)")
page_path: Optional[str] = Field(None, description="Page path (if applicable)")
class DossierOperationResponse(BaseModel):
"""Response for dossier operations."""
success: bool = Field(..., description="Whether operation succeeded")
message: str = Field(..., description="Operation message")
dossier_name: str = Field(..., description="Dossier name")
index_page_id: Optional[int] = Field(None, description="Index page ID (if created)")
index_page_path: Optional[str] = Field(None, description="Index page path (if created)")