Initial commit: library-desk service extraction from portainer-core
Build and Push / build (release) Successful in 36s

This commit is contained in:
2025-12-11 17:28:23 +01:00
commit 95852190ba
59 changed files with 17146 additions and 0 deletions
View File
+160
View File
@@ -0,0 +1,160 @@
"""
Knowledge Consolidation router for Librarian processing.
Provides endpoints for the Scheduler to trigger knowledge consolidation
from HybridRAG search results into wiki pages.
"""
from fastapi import APIRouter, HTTPException, Depends
import logging
from src.models.consolidation import ConsolidationRequest, ConsolidationResponse
from src.services.consolidation_service import ConsolidationService
from src.core.dependencies import (
Neo4jDep, OllamaDep, WikiJSDep,
verify_api_key, get_settings, get_ingestion_service
)
from src.config import Settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/consolidate", tags=["Consolidation"])
# Dependency to get Consolidation service
def get_consolidation_service(
neo4j_client: Neo4jDep,
ollama_client: OllamaDep,
wiki_client: WikiJSDep,
settings: Settings = Depends(get_settings)
) -> ConsolidationService:
"""Get ConsolidationService instance with all dependencies."""
return ConsolidationService(
neo4j=neo4j_client,
ollama=ollama_client,
wiki=wiki_client,
settings=settings,
ingestion_service=get_ingestion_service()
)
@router.post("/knowledge", response_model=ConsolidationResponse)
async def consolidate_knowledge(
request: ConsolidationRequest,
consolidation_service: ConsolidationService = Depends(get_consolidation_service),
api_key: str = Depends(verify_api_key)
):
"""
Consolidate knowledge from HybridRAG search results into wiki pages.
**Librarian Task** - Processes unprocessed SearchQuery nodes to:
1. Find searches with web results from last N days
2. Analyze web content with LLM for novel information
3. Create new wiki pages for new concepts/technologies
4. Update existing pages with new facts and citations
5. Add new entities to knowledge graph
6. Mark SearchQuery nodes as processed
**Typically called by The Scheduler** on a periodic basis (e.g., hourly).
**Parameters:**
- `process_limit`: Maximum searches to process per run (default: 10)
- `lookback_days`: Only process searches from last N days (default: 7)
- `min_web_results`: Minimum web results required to consolidate (default: 2)
- `dry_run`: If true, analyze but don't create pages (default: false)
**Returns:**
- `total_found`: Number of unprocessed searches found
- `processed_count`: Successfully processed searches
- `pages_created`: New wiki pages created
- `pages_updated`: Existing pages updated with new facts
- `entities_added`: New entities added to knowledge graph
- `errors`: List of error messages if any failed
- `results`: Per-search processing results
**Example Request:**
```json
{
"process_limit": 10,
"lookback_days": 7,
"min_web_results": 2,
"dry_run": false
}
```
**Example Response:**
```json
{
"total_found": 5,
"processed_count": 4,
"pages_created": 2,
"pages_updated": 3,
"entities_added": 7,
"errors": ["Search abc123: Failed to parse response"],
"results": [
{
"search_id": "uuid-1",
"query": "docker orchestration kubernetes",
"pages_created": 1,
"pages_updated": 1,
"entities_added": 3
}
],
"dry_run": false
}
```
**Scheduler Task Configuration:**
```json
{
"task_name": "knowledge_consolidation",
"service": "library-desk",
"executor": "rest_api_executor",
"priority": 50,
"minute": 0,
"hour": -1,
"description": "Hourly knowledge consolidation from search results",
"config": {
"url": "http://library-desk:8089/consolidate/knowledge",
"method": "POST",
"payload": {
"process_limit": 10,
"lookback_days": 7,
"min_web_results": 2,
"dry_run": false
},
"auth": {
"type": "bearer",
"token": "${LIBRARY_DESK_API_KEY}"
}
}
}
```
"""
try:
logger.info(
f"Knowledge consolidation requested: "
f"limit={request.process_limit}, lookback={request.lookback_days}d, "
f"dry_run={request.dry_run}"
)
response = await consolidation_service.consolidate_knowledge(
process_limit=request.process_limit,
lookback_days=request.lookback_days,
min_web_results=request.min_web_results,
dry_run=request.dry_run
)
logger.info(
f"Consolidation completed: {response.processed_count}/{response.total_found} searches, "
f"{response.pages_created} pages created, {response.pages_updated} updated"
)
return response
except ValueError as e:
logger.error(f"Invalid request: {e}")
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Knowledge consolidation failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Consolidation failed")
+474
View File
@@ -0,0 +1,474 @@
"""
Entity Linking Router
Finds and links mentions of existing entities in wiki pages to the knowledge graph.
Creates both:
1. Graph relationships (MENTIONS) in Neo4j
2. Hyperlinks in wiki page content (markdown links)
"""
import logging
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from typing import List, Dict, Any, Optional, Tuple
import re
from src.config import get_settings
from src.core.dependencies import (
get_wiki_service,
get_graph_service,
get_ingestion_service,
verify_api_key
)
from src.services.wiki_service import WikiService
from src.services.graph_service import GraphService
from src.services.ingestion_service import IngestionService
from src.models.wiki import WikiPageUpdate
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/entity-linking", tags=["Entity Linking"])
class EntityLinkingResult(BaseModel):
"""Result of entity linking operation."""
page_id: int
page_title: str
entities_found: List[Dict[str, Any]]
new_graph_links_created: int # New MENTIONS relationships in Neo4j
total_graph_links: int
content_links_added: int # New hyperlinks added to wiki page content
content_updated: bool # Whether wiki page content was modified
re_indexed: bool
processing_time_ms: float
class EntityLinkingRequest(BaseModel):
"""Request for entity linking."""
user: str
page_id: int
create_relationships: bool = True
re_index_if_changed: bool = True
@router.post("/link-page", response_model=EntityLinkingResult)
async def link_entities_in_page(
request: EntityLinkingRequest,
wiki_service: WikiService = Depends(get_wiki_service),
graph_service: GraphService = Depends(get_graph_service),
ingestion_service: IngestionService = Depends(get_ingestion_service),
api_key: str = Depends(verify_api_key)
) -> EntityLinkingResult:
"""
Find and link entities mentioned in a wiki page.
This endpoint:
1. Retrieves the page content from Wiki.js
2. Fetches all existing entities from the knowledge graph
3. Finds mentions of those entities in the page text
4. Creates MENTIONS relationships in Neo4j
5. Optionally re-indexes the page if new links were created
Args:
request: EntityLinkingRequest with page_id and user
Returns:
EntityLinkingResult with statistics about linked entities
"""
import time
start_time = time.time()
try:
logger.info(f"Entity linking for page {request.page_id} (user: {request.user})")
# Step 1: Get page content
page = await wiki_service.get_page(request.page_id, request.user)
if not page:
raise HTTPException(status_code=404, detail=f"Page {request.page_id} not found")
page_title = page.title
page_content = page.content
original_content = page_content # Keep original for comparison
logger.info(f"Processing page: {page_title}")
# Step 2: Get all entities from knowledge graph with their wiki page paths
entities = await get_entities_with_paths(graph_service, request.user)
logger.info(f"Found {len(entities)} existing entities in graph")
# Step 3: Find entity mentions in page content
found_entities = find_entity_mentions(page_content, entities)
logger.info(f"Found {len(found_entities)} entity mentions in page")
# Filter out self-referential links (entities linking to the current page)
found_entities = [
e for e in found_entities
if e.get("doc_page_id") != request.page_id
]
logger.info(f"After filtering self-references: {len(found_entities)} entities to link")
# Step 4: Add hyperlinks to wiki content for entities with pages
content_links_added = 0
content_updated = False
if found_entities:
updated_content, content_links_added = add_entity_links_to_content(
page_content,
found_entities
)
if updated_content != original_content:
content_updated = True
logger.info(f"Added {content_links_added} hyperlinks to page content")
# Update the wiki page (preserve existing title, description, tags)
try:
update_data = WikiPageUpdate(
content=updated_content,
title=page.title,
description=page.description if hasattr(page, 'description') else None,
tags=page.tags if hasattr(page, 'tags') else None
)
await wiki_service.update_page(
page_id=request.page_id,
page_data=update_data,
user=request.user
)
logger.info(f"Updated wiki page {request.page_id} with entity links")
except Exception as e:
logger.error(f"Failed to update wiki page content: {e}")
# Continue anyway - graph links can still be created
# Step 5: Create graph relationships if requested
new_graph_links = 0
total_graph_links = 0
if request.create_relationships and found_entities:
new_graph_links = await graph_service.create_entity_mentions(
page_id=request.page_id,
user=request.user,
entity_names=found_entities
)
total_graph_links = len(found_entities)
logger.info(f"Created {new_graph_links} new MENTIONS relationships")
# Step 6: Re-index if changes were made (content or graph)
re_indexed = False
if request.re_index_if_changed and (content_updated or new_graph_links > 0):
logger.info(f"Re-indexing page {request.page_id} due to entity linking changes")
try:
await ingestion_service.ingest_page(request.page_id, request.user)
re_indexed = True
except Exception as e:
logger.error(f"Re-indexing failed: {e}")
# Don't fail the whole operation if re-indexing fails
processing_time = (time.time() - start_time) * 1000
return EntityLinkingResult(
page_id=request.page_id,
page_title=page_title,
entities_found=found_entities,
new_graph_links_created=new_graph_links,
total_graph_links=total_graph_links,
content_links_added=content_links_added,
content_updated=content_updated,
re_indexed=re_indexed,
processing_time_ms=processing_time
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Entity linking failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=f"Entity linking failed: {str(e)}")
def find_entity_mentions(content: str, entities: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Find mentions of entities in the page content.
Uses case-insensitive regex matching to find entity names in the text.
Args:
content: Page content to search
entities: List of entities with 'name', 'type', and optionally 'path' fields
Returns:
List of found entities with additional 'mentions' field
"""
found = []
content_lower = content.lower()
for entity in entities:
entity_name = entity.get("name", "")
if not entity_name or len(entity_name) < 3:
continue
# Create regex pattern for whole word matching
# This avoids matching "John" in "Johnson"
pattern = r'\b' + re.escape(entity_name.lower()) + r'\b'
# Find all matches
matches = list(re.finditer(pattern, content_lower))
if matches:
found.append({
"name": entity_name,
"type": entity.get("type", "unknown"),
"mentions": len(matches),
"entity_id": entity.get("id"),
"path": entity.get("path"), # Include path if available
"doc_page_id": entity.get("doc_page_id") # Include for self-reference filtering
})
# Sort by number of mentions (descending)
found.sort(key=lambda x: x["mentions"], reverse=True)
return found
def fuzzy_match_entity_to_document(
entity_name: str,
doc_title: str,
settings
) -> Tuple[bool, float]:
"""
Match entity name to document title with tolerance for variations.
Uses multiple strategies with confidence scoring:
1. Exact match (confidence: 1.0)
2. Containment match (confidence: 0.85-0.80)
3. Token overlap match (confidence: 0.60-0.80)
Args:
entity_name: Entity name to match
doc_title: Document title to match against
settings: Application settings with matching thresholds
Returns:
Tuple of (is_match, confidence_score)
"""
e_lower = entity_name.lower().strip()
d_lower = doc_title.lower().strip()
# Strategy 1: Exact match (confidence: 1.0)
if e_lower == d_lower:
return True, 1.0
# Strategy 2: Containment match (confidence: 0.85-0.80)
# Require minimum length to avoid false positives
if len(entity_name) >= settings.entity_linking_min_entity_length:
# Entity is substring of title: "lingecollege" in "RSG Lingecollege"
if e_lower in d_lower:
confidence = len(entity_name) / len(doc_title)
if confidence >= settings.entity_linking_min_containment_ratio:
return True, 0.85
# Title is substring of entity: "Google" in "Google Cloud Platform"
if d_lower in e_lower:
confidence = len(doc_title) / len(entity_name)
if confidence >= settings.entity_linking_min_containment_ratio:
return True, 0.80
# Strategy 3: Word-level overlap (confidence: 0.60-0.80)
entity_tokens = set(e_lower.split())
title_tokens = set(d_lower.split())
# Remove common stop words to reduce false positives
stop_words = {'the', 'a', 'an', 'of', 'for', 'and', 'or', 'in', 'on', 'at', 'to'}
entity_tokens -= stop_words
title_tokens -= stop_words
if entity_tokens and title_tokens:
overlap = len(entity_tokens & title_tokens)
total = len(entity_tokens | title_tokens)
overlap_ratio = overlap / total
# Require significant overlap to avoid weak matches
if overlap_ratio >= settings.entity_linking_min_token_overlap:
confidence = 0.75 * overlap_ratio # Scale: 0.45-0.75
return True, confidence
return False, 0.0
async def get_entities_with_paths(graph_service: GraphService, user: str) -> List[Dict[str, Any]]:
"""
Get all entities and check which ones have corresponding wiki pages.
Uses fuzzy matching to handle name variations, typos, and partial matches
while minimizing false positives through confidence thresholds.
Returns entities with their names, types, paths, and match confidence.
"""
from src.core.multi_tenancy import get_neo4j_user_base_label
settings = get_settings()
user_base_label = get_neo4j_user_base_label(user)
# Step 1: Get all entities
entities_query = f"""
MATCH (e:{user_base_label})
WHERE NOT e:Document
RETURN e.name as name,
e.type as type,
e.id as id
"""
# Step 2: Get all documents
documents_query = """
MATCH (d:Document)
RETURN d.title as title,
d.path as path,
d.page_id as page_id
"""
try:
entities = await graph_service.neo4j.execute_query(entities_query, {})
documents = await graph_service.neo4j.execute_query(documents_query, {})
# Step 3: Fuzzy match entities to documents
results = []
for entity in entities:
entity_name = entity["name"]
if not entity_name:
continue
best_match = None
best_confidence = 0.0
# Try to match this entity to any document
for doc in documents:
doc_title = doc["title"]
if not doc_title:
continue
is_match, confidence = fuzzy_match_entity_to_document(
entity_name,
doc_title,
settings
)
if is_match and confidence > best_confidence:
best_match = doc
best_confidence = confidence
# Only include matches above minimum confidence threshold
if best_match and best_confidence >= settings.entity_linking_min_confidence:
results.append({
"name": entity_name,
"type": entity.get("type", "unknown"),
"id": entity.get("id"),
"path": best_match["path"],
"doc_page_id": best_match["page_id"],
"match_confidence": best_confidence # NEW: track confidence
})
else:
# No matching document found (or below threshold)
results.append({
"name": entity_name,
"type": entity.get("type", "unknown"),
"id": entity.get("id"),
"path": None,
"doc_page_id": None,
"match_confidence": 0.0
})
logger.info(f"Matched {len([r for r in results if r['path']])} entities to documents (threshold: {settings.entity_linking_min_confidence})")
return results
except Exception as e:
logger.error(f"Failed to get entities with paths: {e}")
return []
def add_entity_links_to_content(
content: str,
entities_with_paths: List[Dict[str, Any]]
) -> Tuple[str, int]:
"""
Add markdown hyperlinks for entities in the content.
Only links entities that:
1. Have a wiki page (path is not None)
2. Are not already inside existing links
3. Are not already linked in the content
Returns:
Tuple of (updated_content, links_added_count)
"""
if not entities_with_paths:
return content, 0
# Filter to only entities with paths
linkable_entities = [e for e in entities_with_paths if e.get("path")]
if not linkable_entities:
return content, 0
# Sort by length (longest first) to avoid partial replacements
# e.g., "Machine Learning" before "Machine"
linkable_entities.sort(key=lambda x: len(x["name"]), reverse=True)
updated_content = content
links_added = 0
for entity in linkable_entities:
entity_name = entity["name"]
entity_path = entity["path"]
# Skip short entity names to avoid false positives
if len(entity_name) < 3:
continue
# Create markdown link with full path (including user namespace)
# Wiki.js expects full paths like /users/jpmschweitzer/...
markdown_link = f"[{entity_name}](/{entity_path})"
# Find all existing markdown links to protect them (recompute each iteration)
link_pattern = r'\[([^\]]+)\]\([^\)]+\)'
existing_links = list(re.finditer(link_pattern, updated_content))
# Create a list of protected ranges (start, end) for ENTIRE links (text + URL)
# This prevents linking entity names inside existing link URLs
protected_ranges = [(m.start(), m.end()) for m in existing_links]
# Find all potential matches
pattern = r'\b(' + re.escape(entity_name) + r')\b'
matches = list(re.finditer(pattern, updated_content, flags=re.IGNORECASE))
# Filter out matches that are inside existing links
valid_matches = []
for match in matches:
match_start = match.start()
match_end = match.end()
# Check if this match is inside any protected range
inside_link = False
for prot_start, prot_end in protected_ranges:
if prot_start <= match_start < prot_end or prot_start < match_end <= prot_end:
inside_link = True
break
# Also check if already a link (pattern like [entity_name](...))
if match_end < len(updated_content) - 1:
next_chars = updated_content[match_end:match_end+2]
if next_chars == '](':
inside_link = True
if not inside_link:
valid_matches.append(match)
if not valid_matches:
continue
# Replace valid matches in reverse order (to preserve positions)
for match in reversed(valid_matches):
updated_content = (
updated_content[:match.start()] +
markdown_link +
updated_content[match.end():]
)
links_added += 1
return updated_content, links_added
+253
View File
@@ -0,0 +1,253 @@
"""
Graph router for Library Desk API.
Endpoints for Neo4j knowledge graph operations.
"""
from fastapi import APIRouter, HTTPException, Depends, Query
from typing import Optional, List
import logging
from src.models.graph import (
CypherQueryRequest, CypherQueryResponse,
UpdateFromPageRequest, GraphUpdateSummary,
NodeListResponse, GraphNodeDetail,
MindMapResponse
)
from src.services.graph_service import GraphService
from src.clients.neo4j_client import Neo4jClient
from src.clients.wikijs_client import WikiJSClient
from src.core.dependencies import Neo4jDep, WikiJSDep, verify_api_key
from src.core.multi_tenancy import DEFAULT_USER
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/graph", tags=["Graph"])
# Dependency to get graph service
def get_graph_service(
neo4j_client: Neo4jDep,
wiki_client: WikiJSDep
) -> GraphService:
"""Get graph service instance."""
return GraphService(neo4j_client, wiki_client)
@router.post("/query", response_model=CypherQueryResponse)
async def execute_cypher_query(
request: CypherQueryRequest,
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Execute a user-scoped Cypher query.
The query is automatically scoped to the user's data for security.
This prevents users from accessing other users' graph data.
**Example Request:**
```json
{
"query": "MATCH (d:Document) RETURN d LIMIT 10",
"parameters": {},
"user": "jpmschweitzer"
}
```
**Security:** Query is automatically scoped with user label.
"""
try:
return await graph_service.execute_query(
query=request.query,
parameters=request.parameters,
user=request.user
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Cypher query failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Query execution failed")
@router.get("/nodes", response_model=NodeListResponse)
async def list_nodes(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
node_type: Optional[str] = Query(default=None, description="Node type filter"),
limit: int = Query(default=100, ge=1, le=500, description="Maximum nodes"),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
List graph nodes for a user.
Optionally filter by node type (Document, Person, Project, Concept, etc.).
**Example:** `/graph/nodes?user=jpmschweitzer&node_type=Document&limit=50`
"""
try:
return await graph_service.list_nodes(
user=user,
node_type=node_type,
limit=limit
)
except Exception as e:
logger.error(f"Failed to list nodes: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Failed to list nodes")
@router.get("/nodes/{node_id}", response_model=GraphNodeDetail)
async def get_node(
node_id: str,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Get detailed information about a graph node.
Returns the node, its relationships, and connected nodes.
**Example:** `/graph/nodes/4:abc123def:0?user=jpmschweitzer`
"""
try:
node = await graph_service.get_node(node_id, user)
if not node:
raise HTTPException(status_code=404, detail=f"Node {node_id} not found")
return node
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to get node {node_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Failed to get node")
@router.post("/update-from-page/{page_id}", response_model=GraphUpdateSummary)
async def update_graph_from_page(
page_id: int,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
force_refresh: bool = Query(default=False, description="Force re-extraction"),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Update knowledge graph from a wiki page.
This endpoint:
1. Fetches the page from Wiki.js
2. Extracts entities (people, projects, concepts, technologies)
3. Creates/updates Document node
4. Creates entity nodes and MENTIONS relationships
5. Returns summary of what was updated
**Use Cases:**
- Called automatically after page creation/update (via BackgroundTasks)
- Called manually by user/Librarian to refresh graph
- Called by Scheduler for batch processing
**Example:** `POST /graph/update-from-page/4?user=jpmschweitzer`
**Returns:** Summary with nodes/relationships created and entities extracted
"""
try:
summary = await graph_service.update_from_page(
page_id=page_id,
user=user,
force_refresh=force_refresh
)
if not summary.success:
raise HTTPException(
status_code=500,
detail=f"Graph update failed: {summary.error_message}"
)
return summary
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to update graph from page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Graph update failed")
@router.post("/mindmap", response_model=MindMapResponse)
async def generate_mindmap(
center_node_id: str = Query(..., description="Central node ID"),
user: str = Query(default=DEFAULT_USER, description="User identifier"),
depth: int = Query(default=2, ge=1, le=5, description="Traversal depth"),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Generate mind map data for visualization.
Traverses the graph from a center node and returns nodes/links
in a format suitable for D3.js or similar visualization libraries.
**Parameters:**
- `center_node_id`: The node to center the mind map on
- `depth`: How many hops away from center to include (1-5)
- `user`: User identifier for scoping
**Example:** `POST /graph/mindmap?center_node_id=4:abc:0&depth=2`
**Returns:** Nodes and links for visualization
"""
try:
return await graph_service.generate_mindmap(
center_node_id=center_node_id,
user=user,
depth=depth
)
except Exception as e:
logger.error(f"Failed to generate mindmap: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Mindmap generation failed")
@router.post("/generate-entity-pages")
async def generate_entity_pages(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
min_mentions: int = Query(default=5, ge=1, le=100, description="Minimum mentions threshold"),
entity_types: Optional[List[str]] = Query(default=None, description="Entity types to process"),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Generate wiki stub pages for graph entities.
Creates pages in `/entities/{type}/{name}` namespace for entities
that have been mentioned in multiple documents. This creates a
bidirectional knowledge graph ↔ wiki synchronization.
**Threshold:** Entities must be mentioned in at least `min_mentions` documents (default: 5)
**Auto-stub flow:**
1. Find entities with >= min_mentions
2. Check if entity already has a wiki page
3. If not, create stub page with:
- List of mentioning documents
- Related entities (co-occurring)
- Auto-generated tag to prevent feedback loops
**Feedback loop protection:** Pages tagged with `entity-stub` skip entity extraction
**Parameters:**
- `min_mentions`: Minimum number of document mentions required (default: 5)
- `entity_types`: List of types to process (default: Person, Technology, Concept, Project)
- `user`: User identifier for scoping
**Example:** `POST /graph/generate-entity-pages?min_mentions=3&user=jpmschweitzer`
**Returns:** Summary with list of pages created and skipped
"""
try:
result = await graph_service.generate_entity_stubs(
user=user,
min_mentions=min_mentions,
entity_types=entity_types
)
return result
except Exception as e:
logger.error(f"Failed to generate entity pages: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Entity page generation failed")
+117
View File
@@ -0,0 +1,117 @@
"""
HybridRAG router for multi-source search API.
Provides endpoint for combining vector, graph, and web search
with RRF fusion and LLM re-ranking.
"""
from fastapi import APIRouter, HTTPException, Depends, Query
import logging
from src.models.hybrid_rag import HybridRAGRequest, HybridRAGResponse
from src.services.hybrid_rag_service import HybridRAGService
from src.services.vector_service import VectorService
from src.services.graph_service import GraphService
from src.clients.searxng_client import SearXNGClient
from src.clients.ollama_client import OllamaClient
from src.core.dependencies import (
Neo4jDep, WikiJSDep, QdrantDep, OllamaDep,
SearXNGDep, verify_api_key, get_settings
)
from src.config import Settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/query", tags=["HybridRAG"])
# Dependency to get HybridRAG service
def get_hybrid_rag_service(
neo4j_client: Neo4jDep,
wiki_client: WikiJSDep,
qdrant_client: QdrantDep,
ollama_client: OllamaDep,
searxng_client: SearXNGDep,
settings: Settings = Depends(get_settings)
) -> HybridRAGService:
"""Get HybridRAG service instance with all dependencies."""
from src.services.vector_service import VectorService
from src.services.graph_service import GraphService
# Create component services
vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
graph_service = GraphService(neo4j_client, wiki_client)
# Create HybridRAG service
return HybridRAGService(
vector_service=vector_service,
graph_service=graph_service,
searxng_client=searxng_client,
ollama_client=ollama_client,
settings=settings
)
@router.post("/hybrid", response_model=HybridRAGResponse)
async def hybrid_search(
request: HybridRAGRequest,
user: str = Query(default="jpmschweitzer", description="User identifier for multi-tenancy"),
hybrid_rag_service: HybridRAGService = Depends(get_hybrid_rag_service),
api_key: str = Depends(verify_api_key)
):
"""
Execute HybridRAG query combining vector, graph, and web search.
**6-Phase Pipeline:**
1. **Query Enhancement**: Extract keywords/synonyms with LLM
2. **Parallel Retrieval**: Search vector (Qdrant), graph (Neo4j), web (SearXNG)
3. **RRF Fusion**: Merge results with Reciprocal Rank Fusion
4. **Enrichment**: Add related documents via shared entities
5. **LLM Re-ranking**: Re-rank with mistral-nemo for relevance
6. **Context Formatting**: Format for LLM consumption
7. **Persistence**: Store for Librarian knowledge consolidation
**Example Request:**
```json
{
"query": "How does Docker orchestration work with Kubernetes?",
"user": "jpmschweitzer",
"config": {
"vector_limit": 10,
"graph_limit": 10,
"web_limit": 5,
"enable_reranking": true,
"final_result_count": 10
}
}
```
**Returns:**
- Ranked results from all sources
- Extracted keywords/synonyms
- Related dossiers (via graph)
- Formatted context for LLM
- Performance timing breakdown
- Search ID for Librarian tracking
"""
try:
logger.info(f"HybridRAG request: '{request.query}' for user '{user}'")
response = await hybrid_rag_service.search(
query=request.query,
user=user,
config=request.config
)
logger.info(
f"HybridRAG completed: {response.total_results} results in {response.timing.total_ms:.0f}ms"
)
return response
except ValueError as e:
logger.error(f"Invalid request: {e}")
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"HybridRAG search failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Search failed")
+169
View File
@@ -0,0 +1,169 @@
"""
Document Ingestion API Router
Endpoints for ingesting wiki pages into the knowledge base (vectors + graph).
"""
from fastapi import APIRouter, Depends, HTTPException, Query
from typing import Optional
from src.services.ingestion_service import IngestionService
from src.models.ingestion import (
IngestionRequest,
IngestionResult,
BatchIngestionRequest,
BatchIngestionResult
)
from src.core.dependencies import get_ingestion_service, verify_api_key
router = APIRouter(prefix="/ingest", tags=["Document Ingestion"])
@router.post("/page", response_model=IngestionResult)
async def ingest_page(
request: IngestionRequest,
ingestion: IngestionService = Depends(get_ingestion_service),
api_key: str = Depends(verify_api_key)
):
"""
Ingest a single wiki page into the knowledge base.
This endpoint:
1. Fetches page content from Wiki.js
2. Chunks content and generates embeddings (Qdrant)
3. Extracts entities and updates knowledge graph (Neo4j)
## Use Cases
- **After page creation**: Automatically called by consolidation service
- **Manual re-indexing**: Force refresh a page after manual edits
- **Selective ingestion**: Skip vectors or graph if only one is needed
## Performance
- Typical page: 1-3 seconds
- Large page (>5000 words): 5-10 seconds
- Vector and graph ingestion run in parallel
## Example
```bash
curl -X POST http://192.168.86.149:8089/ingest/page \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d '{
"page_id": 19,
"user": "jpmschweitzer",
"force_refresh": false
}'
```
"""
result = await ingestion.ingest_page(
page_id=request.page_id,
user=request.user,
force_refresh=request.force_refresh,
skip_vectors=request.skip_vectors,
skip_graph=request.skip_graph
)
if not result.success:
raise HTTPException(
status_code=500,
detail=f"Ingestion failed: {result.error}"
)
return result
@router.post("/batch", response_model=BatchIngestionResult)
async def ingest_batch(
request: BatchIngestionRequest,
ingestion: IngestionService = Depends(get_ingestion_service),
api_key: str = Depends(verify_api_key)
):
"""
Ingest multiple wiki pages concurrently.
## Concurrency Control
The `max_concurrent` parameter controls how many pages are processed simultaneously:
- **Low (1-2)**: Safer for resource-constrained systems
- **Medium (3-5)**: Good balance of speed and stability
- **High (6-10)**: Maximum speed, requires good resources
## Batch Size Recommendations
- **Small batches (<10 pages)**: Use max_concurrent=3-5
- **Medium batches (10-50 pages)**: Use max_concurrent=3
- **Large batches (>50 pages)**: Use max_concurrent=2, consider splitting
## Example
```bash
curl -X POST http://192.168.86.149:8089/ingest/batch \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d '{
"page_ids": [19, 20, 21, 22],
"user": "jpmschweitzer",
"max_concurrent": 3
}'
```
"""
result = await ingestion.ingest_batch(
page_ids=request.page_ids,
user=request.user,
force_refresh=request.force_refresh,
skip_vectors=request.skip_vectors,
skip_graph=request.skip_graph,
max_concurrent=request.max_concurrent
)
return result
@router.post("/all", response_model=BatchIngestionResult)
async def ingest_all_pages(
user: str = Query(default="jpmschweitzer", description="User identifier"),
path_prefix: Optional[str] = Query(None, description="Path prefix filter (e.g., 'users/jpmschweitzer/tech')"),
force_refresh: bool = Query(False, description="Force re-ingestion of all pages"),
max_concurrent: int = Query(3, ge=1, le=10, description="Maximum concurrent ingestion tasks"),
ingestion: IngestionService = Depends(get_ingestion_service),
api_key: str = Depends(verify_api_key)
):
"""
Ingest all wiki pages for a user (bulk re-indexing).
## Use Cases
- **Initial setup**: Index all existing pages
- **Full re-index**: After major schema changes
- **Path-specific**: Re-index a specific section (e.g., all tech docs)
## Performance
- **Small wiki (<50 pages)**: 2-5 minutes
- **Medium wiki (50-200 pages)**: 5-20 minutes
- **Large wiki (>200 pages)**: 20+ minutes
**Recommendation**: Run as background job for large wikis
## Example
```bash
# Ingest all pages for user
curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer" \
-H "Authorization: Bearer $API_KEY"
# Ingest only tech docs
curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer&path_prefix=users/jpmschweitzer/tech" \
-H "Authorization: Bearer $API_KEY"
```
"""
result = await ingestion.ingest_all_pages(
user=user,
path_prefix=path_prefix,
force_refresh=force_refresh,
max_concurrent=max_concurrent
)
return result
+334
View File
@@ -0,0 +1,334 @@
"""
Tools router for Library Desk API.
Provides tool discovery endpoint for AI agents.
"""
from fastapi import APIRouter
from src.models.tools import (
ToolCatalog, ToolDefinition, ToolParameter, CategoryInfo, ParameterType
)
router = APIRouter(prefix="/tools", tags=["Tools"])
def get_wiki_tools() -> list[ToolDefinition]:
"""Get wiki tool definitions."""
return [
ToolDefinition(
name="wiki_list_pages",
category="wiki",
description="List wiki pages for a user with optional tag filtering",
method="GET",
endpoint="/wiki/pages",
parameters=[
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer",
example="jpmschweitzer"
),
ToolParameter(
name="tag",
type=ParameterType.STRING,
description="Filter by tag (dossier)",
required=False,
example="projects"
),
ToolParameter(
name="limit",
type=ParameterType.INTEGER,
description="Maximum pages to return",
required=False,
default=50,
example=20
),
],
returns="List of pages with summaries",
fast=True
),
ToolDefinition(
name="wiki_get_page",
category="wiki",
description="Get a single wiki page by ID with full content",
method="GET",
endpoint="/wiki/pages/{page_id}",
parameters=[
ToolParameter(
name="page_id",
type=ParameterType.INTEGER,
description="Page ID to retrieve",
required=True,
example=123
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier for access control",
required=False,
default="jpmschweitzer"
),
],
returns="Complete page object with content",
fast=True
),
ToolDefinition(
name="wiki_create_page",
category="wiki",
description="Create a new wiki page in user's namespace",
method="POST",
endpoint="/wiki/pages",
parameters=[
ToolParameter(
name="title",
type=ParameterType.STRING,
description="Page title",
required=True,
example="Project Documentation"
),
ToolParameter(
name="path",
type=ParameterType.STRING,
description="Page path (will be prefixed with user namespace)",
required=True,
example="/projects/my-project"
),
ToolParameter(
name="content",
type=ParameterType.STRING,
description="Page content in Markdown format",
required=True,
example="# Overview\n\nThis is the content."
),
ToolParameter(
name="description",
type=ParameterType.STRING,
description="Short page description",
required=False,
example="Documentation for my project"
),
ToolParameter(
name="tags",
type=ParameterType.ARRAY,
description="Tags for categorization (dossiers)",
required=False,
example=["projects", "documentation"]
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
],
returns="Created page object",
example={
"title": "My Project",
"path": "/projects/my-project",
"content": "# My Project\n\nProject description here.",
"tags": ["projects"],
"user": "jpmschweitzer"
},
fast=True
),
ToolDefinition(
name="wiki_update_page",
category="wiki",
description="Update an existing wiki page",
method="PUT",
endpoint="/wiki/pages/{page_id}",
parameters=[
ToolParameter(
name="page_id",
type=ParameterType.INTEGER,
description="Page ID to update",
required=True,
example=123
),
ToolParameter(
name="title",
type=ParameterType.STRING,
description="New page title",
required=False
),
ToolParameter(
name="content",
type=ParameterType.STRING,
description="New page content",
required=False
),
ToolParameter(
name="description",
type=ParameterType.STRING,
description="New description",
required=False
),
ToolParameter(
name="tags",
type=ParameterType.ARRAY,
description="New tags",
required=False
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
],
returns="Updated page object",
fast=True
),
ToolDefinition(
name="wiki_delete_page",
category="wiki",
description="Delete a wiki page",
method="DELETE",
endpoint="/wiki/pages/{page_id}",
parameters=[
ToolParameter(
name="page_id",
type=ParameterType.INTEGER,
description="Page ID to delete",
required=True,
example=123
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
],
returns="Success confirmation",
fast=True
),
ToolDefinition(
name="wiki_search_pages",
category="wiki",
description="Search wiki pages by content within user's namespace",
method="GET",
endpoint="/wiki/search",
parameters=[
ToolParameter(
name="q",
type=ParameterType.STRING,
description="Search query",
required=True,
example="docker configuration"
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
ToolParameter(
name="limit",
type=ParameterType.INTEGER,
description="Maximum results",
required=False,
default=20
),
],
returns="List of matching pages",
fast=True
),
ToolDefinition(
name="wiki_list_dossiers",
category="wiki",
description="List all dossiers (unique tags) for a user with page counts",
method="GET",
endpoint="/wiki/dossiers",
parameters=[
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
],
returns="List of dossiers with page counts",
fast=True
),
ToolDefinition(
name="wiki_get_dossier_pages",
category="wiki",
description="Get all pages in a specific dossier",
method="GET",
endpoint="/wiki/dossiers/{dossier_name}/pages",
parameters=[
ToolParameter(
name="dossier_name",
type=ParameterType.STRING,
description="Dossier name (tag)",
required=True,
example="projects"
),
ToolParameter(
name="user",
type=ParameterType.STRING,
description="User identifier",
required=False,
default="jpmschweitzer"
),
ToolParameter(
name="limit",
type=ParameterType.INTEGER,
description="Maximum pages",
required=False,
default=100
),
],
returns="List of pages in dossier",
fast=True
),
]
@router.get("", response_model=ToolCatalog)
async def get_tool_catalog() -> ToolCatalog:
"""
Get simplified tool catalog for AI agent consumption.
This endpoint provides a machine-readable catalog of all Library Desk tools,
optimized for discovery and use by AI agents like The Librarian.
Returns tool definitions with:
- Clear descriptions
- Parameter specifications
- Usage examples
- Performance characteristics
"""
wiki_tools = get_wiki_tools()
# Calculate category stats
categories = {}
for tool in wiki_tools:
if tool.category not in categories:
categories[tool.category] = 0
categories[tool.category] += 1
category_info = [
CategoryInfo(
name="wiki",
description="Wiki.js page and dossier management operations",
tool_count=categories.get("wiki", 0)
)
]
return ToolCatalog(
service="library-desk",
version="1.0.0",
base_url="http://library-desk:8089",
categories=category_info,
tools=wiki_tools,
authentication="Bearer token via Authorization header (LIBRARY_API_KEY)"
)
+173
View File
@@ -0,0 +1,173 @@
"""
Vector router for Library Desk API.
Endpoints for semantic search and vector operations.
"""
from fastapi import APIRouter, HTTPException, Depends, Query
from typing import Optional
import logging
from src.models.vector import (
SearchRequest, SearchResponse,
VectorUpdateRequest, VectorUpdateSummary,
CollectionListResponse,
DeletePageChunksRequest, DeletePageChunksResponse
)
from src.services.vector_service import VectorService
from src.clients.qdrant_client import QdrantClientWrapper
from src.clients.wikijs_client import WikiJSClient
from src.clients.ollama_client import OllamaClient
from src.core.dependencies import QdrantDep, WikiJSDep, OllamaDep, verify_api_key
from src.core.multi_tenancy import DEFAULT_USER
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/vector", tags=["Vector"])
# Dependency to get vector service
def get_vector_service(
qdrant_client: QdrantDep,
wiki_client: WikiJSDep,
ollama_client: OllamaDep
) -> VectorService:
"""Get vector service instance."""
return VectorService(qdrant_client, wiki_client, ollama_client)
@router.post("/search", response_model=SearchResponse)
async def semantic_search(
request: SearchRequest,
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
Perform semantic search across user's documents.
Uses Ollama to generate query embedding, then searches Qdrant
for similar document chunks.
**Example Request:**
```json
{
"query": "how to configure docker",
"user": "jpmschweitzer",
"limit": 10,
"score_threshold": 0.5
}
```
**Returns:** List of matching chunks with similarity scores
"""
try:
return await vector_service.search(
query=request.query,
user=request.user,
limit=request.limit,
score_threshold=request.score_threshold
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Semantic search failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Search failed")
@router.post("/update-from-page/{page_id}", response_model=VectorUpdateSummary)
async def update_vectors_from_page(
page_id: int,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
force_refresh: bool = Query(default=False, description="Force re-embedding"),
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
Update vector embeddings from a wiki page.
This endpoint:
1. Fetches the page from Wiki.js
2. Chunks the content (500 tokens with 50 token overlap)
3. Generates embeddings via Ollama
4. Upserts chunks to Qdrant with metadata
**Use Cases:**
- Called automatically after page creation/update (via BackgroundTasks)
- Called manually by user/Librarian to refresh vectors
- Called by Scheduler for batch processing
**Example:** `POST /vector/update-from-page/5?user=jpmschweitzer`
**Returns:** Summary with chunks created and processing time
"""
try:
summary = await vector_service.update_from_page(
page_id=page_id,
user=user,
force_refresh=force_refresh
)
if not summary.success:
raise HTTPException(
status_code=500,
detail=f"Vector update failed: {summary.error_message}"
)
return summary
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to update vectors from page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Vector update failed")
@router.delete("/pages/{page_id}", response_model=DeletePageChunksResponse)
async def delete_page_chunks(
page_id: int,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
Delete all vector chunks for a wiki page.
This is automatically called when a page is deleted from the wiki.
**Example:** `DELETE /vector/pages/5?user=jpmschweitzer`
"""
try:
deleted_count = await vector_service.delete_page_chunks(
page_id=page_id,
user=user
)
return DeletePageChunksResponse(
page_id=page_id,
chunks_deleted=deleted_count,
success=deleted_count > 0,
message=f"Deleted {deleted_count} chunks for page {page_id}"
)
except Exception as e:
logger.error(f"Failed to delete chunks for page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Failed to delete chunks")
@router.get("/collections", response_model=CollectionListResponse)
async def list_collections(
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
List all Qdrant collections with statistics.
Returns collection names, vector counts, and point counts.
**Example:** `GET /vector/collections`
"""
try:
return await vector_service.list_collections()
except Exception as e:
logger.error(f"Failed to list collections: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Failed to list collections")
+498
View File
@@ -0,0 +1,498 @@
"""
Wiki.js Webhook Handler
Receives webhook events from Wiki.js for page CRUD operations
and processes them identically to AI-generated content.
"""
import logging
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from pydantic import BaseModel
from typing import Optional, Literal
from src.core.dependencies import (
get_ingestion_service,
get_wiki_service,
get_graph_service,
verify_api_key
)
from src.services.ingestion_service import IngestionService
from src.services.consolidation_service import ConsolidationService
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/webhooks", tags=["Webhooks"])
class WikiJSWebhookPayload(BaseModel):
"""
Wiki.js webhook payload structure.
See: https://docs.requarks.io/webhooks
"""
event: Literal["page.create", "page.update", "page.delete", "page.rename"]
page: dict # Contains: id, title, path, content, etc.
user: dict # Contains: id, name, email
timestamp: str
class WebhookProcessingResult(BaseModel):
"""Result of webhook processing."""
success: bool
page_id: int
page_title: str
event: str
ingested: bool
entity_linking: Optional[dict] = None
error: Optional[str] = None
processing_time_ms: float
@router.post("/wikijs", response_model=WebhookProcessingResult)
async def handle_wikijs_webhook(
payload: WikiJSWebhookPayload,
background_tasks: BackgroundTasks,
ingestion_service: IngestionService = Depends(get_ingestion_service),
api_key: str = Depends(verify_api_key)
):
"""
Handle Wiki.js webhook events.
Processes page changes identically to AI-generated content:
1. Ingest page → Vector embeddings + Graph entity extraction
2. Apply bidirectional entity linking
This ensures user-edited pages have the same structure and
connectivity as AI-generated pages.
Configuration in Wiki.js:
- Administration → Webhooks
- Add new webhook:
- URL: http://library-desk:8089/webhooks/wikijs
- Events: page.create, page.update
- Headers: Authorization: Bearer <API_KEY>
"""
import time
start_time = time.time()
page_id = payload.page.get("id")
page_title = payload.page.get("title")
user_email = payload.user.get("email", "unknown")
# Extract user identifier from email (assumes format: user@domain)
# Adjust this based on your user mapping strategy
user = extract_user_from_email(user_email)
logger.info(
f"Received Wiki.js webhook: {payload.event} "
f"for page {page_id} ('{page_title}') by {user_email}"
)
try:
# Handle different events
if payload.event == "page.delete":
# For deletions, clean up vectors and graph
logger.info(f"Page {page_id} deleted, cleaning up knowledge base")
background_tasks.add_task(
cleanup_deleted_page,
page_id=page_id,
page_title=page_title,
user=user,
ingestion_service=ingestion_service
)
processing_time = (time.time() - start_time) * 1000
return WebhookProcessingResult(
success=True,
page_id=page_id,
page_title=page_title,
event=payload.event,
ingested=False,
processing_time_ms=processing_time
)
# For create/update events, process the page
if payload.event in ["page.create", "page.update"]:
# Run ingestion and entity linking in background
# to avoid blocking the webhook response
background_tasks.add_task(
process_wiki_page_change,
page_id=page_id,
page_title=page_title,
user=user,
event=payload.event,
ingestion_service=ingestion_service
)
processing_time = (time.time() - start_time) * 1000
return WebhookProcessingResult(
success=True,
page_id=page_id,
page_title=page_title,
event=payload.event,
ingested=True, # Will be processed in background
processing_time_ms=processing_time
)
# Handle page rename/move events
if payload.event == "page.rename":
# Update path and re-link if title changed
new_path = payload.page.get("path")
new_title = payload.page.get("title")
old_path = payload.page.get("oldPath", new_path)
old_title = payload.page.get("oldTitle", new_title)
background_tasks.add_task(
process_page_rename,
page_id=page_id,
old_path=old_path,
new_path=new_path,
old_title=old_title,
new_title=new_title,
user=user,
ingestion_service=ingestion_service
)
processing_time = (time.time() - start_time) * 1000
return WebhookProcessingResult(
success=True,
page_id=page_id,
page_title=page_title,
event=payload.event,
ingested=False, # Path update only, no re-embedding needed
processing_time_ms=processing_time
)
except Exception as e:
logger.error(f"Webhook processing failed: {e}", exc_info=True)
processing_time = (time.time() - start_time) * 1000
return WebhookProcessingResult(
success=False,
page_id=page_id,
page_title=page_title,
event=payload.event,
ingested=False,
error=str(e),
processing_time_ms=processing_time
)
async def process_wiki_page_change(
page_id: int,
page_title: str,
user: str,
event: str,
ingestion_service: IngestionService
):
"""
Process wiki page change identically to AI-generated content.
This ensures consistency between manual and AI workflows.
Steps:
1. Ingest page → Vector embeddings + Graph entities
2. Apply bidirectional entity linking
"""
from src.core.dependencies import get_neo4j_client, get_ollama_client, get_wikijs_client
from src.config import get_settings
try:
logger.info(f"Processing {event} for page {page_id} ('{page_title}')")
# STEP 1: Ingest page (vector + graph)
logger.info(f"Ingesting page {page_id} into knowledge base")
await ingestion_service.ingest_page(
page_id=page_id,
user=user,
force_refresh=(event == "page.update") # Force refresh on updates
)
logger.info(f"Ingestion complete for page {page_id}")
# STEP 2: Apply bidirectional entity linking
logger.info(f"Applying bidirectional entity linking for page {page_id}")
# Import consolidation service to use the entity linking method
settings = get_settings()
consolidation_service = ConsolidationService(
neo4j=get_neo4j_client(),
ollama=get_ollama_client(),
wiki=get_wikijs_client(),
settings=settings,
ingestion_service=ingestion_service
)
link_stats = await consolidation_service._apply_bidirectional_entity_linking(
page_id=page_id,
page_title=page_title,
user=user
)
logger.info(
f"Entity linking complete for page {page_id}: "
f"{link_stats['forward_links']} forward links, "
f"{link_stats['backward_links']} backward links "
f"({link_stats['pages_updated']} pages updated)"
)
logger.info(f"Successfully processed {event} for page {page_id}")
except Exception as e:
logger.error(f"Failed to process page {page_id}: {e}", exc_info=True)
def extract_user_from_email(email: str) -> str:
"""
Extract user identifier from email.
Customize this based on your user mapping strategy:
- Option 1: Use email prefix (user@domain → user)
- Option 2: Map email to Wiki.js username
- Option 3: Use email directly
Args:
email: User email from Wiki.js webhook
Returns:
User identifier for multi-tenancy
"""
# Option 1: Extract prefix from email
if "@" in email:
return email.split("@")[0]
# Fallback: use email as-is
return email
async def process_page_rename(
page_id: int,
old_path: str,
new_path: str,
old_title: str,
new_title: str,
user: str,
ingestion_service: IngestionService
):
"""
Process page rename/move events.
Handles two scenarios:
1. Path change only (move to different location) - update path in graph
2. Title change (rename) - update title, re-link entities, re-process
Args:
page_id: Wiki page ID
old_path: Previous page path
new_path: New page path
old_title: Previous page title
new_title: New page title
user: User identifier
ingestion_service: Ingestion service instance
"""
from src.core.dependencies import get_neo4j_client
from src.core.multi_tenancy import get_neo4j_user_label
try:
logger.info(
f"Processing rename for page {page_id}: "
f"'{old_title}' → '{new_title}', "
f"'{old_path}' → '{new_path}'"
)
neo4j = get_neo4j_client()
user_doc_label = get_neo4j_user_label(user)
# Check if title changed (rename) or just path changed (move)
title_changed = old_title != new_title
path_changed = old_path != new_path
if not title_changed and not path_changed:
logger.info("No changes detected, skipping processing")
return
# Update Document node in graph
update_query = f"""
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})
SET d.path = $new_path,
d.title = $new_title,
d.updated_at = datetime()
RETURN d
"""
try:
await neo4j.execute_query(update_query, {
"page_id": page_id,
"new_path": new_path,
"new_title": new_title
})
logger.info(f"Updated Document node for page {page_id}")
except Exception as e:
logger.error(f"Failed to update Document node: {e}")
# If title changed, need to re-process for entity linking
if title_changed:
logger.info(f"Title changed, re-processing page {page_id}")
# Re-ingest to update entities (title might be an entity)
try:
await ingestion_service.ingest_page(
page_id=page_id,
user=user,
force_refresh=True
)
logger.info(f"Re-ingested page {page_id} after title change")
except Exception as e:
logger.error(f"Failed to re-ingest page {page_id}: {e}")
# Re-apply bidirectional entity linking
from src.config import get_settings
from src.core.dependencies import get_ollama_client, get_wikijs_client
from src.services.consolidation_service import ConsolidationService
settings = get_settings()
consolidation_service = ConsolidationService(
neo4j=neo4j,
ollama=get_ollama_client(),
wiki=get_wikijs_client(),
settings=settings,
ingestion_service=ingestion_service
)
try:
link_stats = await consolidation_service._apply_bidirectional_entity_linking(
page_id=page_id,
page_title=new_title,
user=user
)
logger.info(
f"Entity linking complete for renamed page {page_id}: "
f"{link_stats['forward_links']} forward links, "
f"{link_stats['backward_links']} backward links"
)
except Exception as e:
logger.error(f"Failed to apply entity linking: {e}")
elif path_changed:
logger.info(f"Path changed only (move), no re-processing needed")
logger.info(f"Rename processing complete for page {page_id}")
except Exception as e:
logger.error(f"Failed to process rename for page {page_id}: {e}", exc_info=True)
async def cleanup_deleted_page(
page_id: int,
page_title: str,
user: str,
ingestion_service: IngestionService
):
"""
Clean up vectors and graph when a page is deleted.
Steps:
1. Remove vector embeddings from Qdrant
2. Remove Document node from Neo4j
3. Clean up orphaned entities (entities only connected to this document)
4. Remove broken MENTIONS relationships
"""
from src.core.dependencies import get_neo4j_client, get_vector_service
from src.core.multi_tenancy import get_neo4j_user_base_label, get_neo4j_user_label
try:
logger.info(f"Cleaning up deleted page {page_id} ('{page_title}')")
vector_service = get_vector_service()
neo4j = get_neo4j_client()
user_base_label = get_neo4j_user_base_label(user)
user_doc_label = get_neo4j_user_label(user)
# STEP 1: Remove vectors from Qdrant
logger.info(f"Removing vectors for page {page_id}")
try:
await vector_service.delete_page_chunks(page_id, user)
logger.info(f"Removed vectors for page {page_id}")
except Exception as e:
logger.error(f"Failed to remove vectors for page {page_id}: {e}")
# STEP 2: Find and store orphaned entities before deletion
# (entities that only have this document mentioning them)
orphaned_entities_query = f"""
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})-[:MENTIONS]->(e:{user_base_label})
WHERE NOT e:Document
WITH e, count{{(d2:Document)-[:MENTIONS]->(e)}} as mention_count
WHERE mention_count = 1
RETURN e.id as entity_id, e.name as entity_name, labels(e) as labels
"""
try:
orphaned = await neo4j.execute_query(orphaned_entities_query, {"page_id": page_id})
logger.info(f"Found {len(orphaned)} orphaned entities for page {page_id}")
except Exception as e:
logger.error(f"Failed to find orphaned entities: {e}")
orphaned = []
# STEP 3: Delete Document node (this will cascade delete MENTIONS relationships)
delete_doc_query = f"""
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})
DETACH DELETE d
RETURN count(d) as deleted_count
"""
try:
result = await neo4j.execute_query(delete_doc_query, {"page_id": page_id})
deleted_count = result[0]["deleted_count"] if result else 0
logger.info(f"Deleted {deleted_count} Document node(s) for page {page_id}")
except Exception as e:
logger.error(f"Failed to delete Document node: {e}")
# STEP 4: Delete orphaned entities
if orphaned:
for entity in orphaned:
entity_id = entity["entity_id"]
entity_name = entity["entity_name"]
delete_entity_query = f"""
MATCH (e:{user_base_label} {{id: $entity_id}})
WHERE NOT e:Document
AND NOT EXISTS {{(d:Document)-[:MENTIONS]->(e)}}
DETACH DELETE e
RETURN count(e) as deleted_count
"""
try:
result = await neo4j.execute_query(delete_entity_query, {"entity_id": entity_id})
deleted = result[0]["deleted_count"] if result else 0
if deleted > 0:
logger.info(f"Deleted orphaned entity: {entity_name}")
except Exception as e:
logger.error(f"Failed to delete orphaned entity {entity_name}: {e}")
# STEP 5: Clean up broken SearchQuery relationships
cleanup_search_query = f"""
MATCH (sq:SearchQuery)-[r:FOUND]->(d:Document)
WHERE NOT EXISTS {{(d)}}
DELETE r
RETURN count(r) as cleaned_count
"""
try:
result = await neo4j.execute_query(cleanup_search_query, {})
cleaned = result[0]["cleaned_count"] if result else 0
if cleaned > 0:
logger.info(f"Cleaned up {cleaned} broken SearchQuery relationships")
except Exception as e:
logger.error(f"Failed to clean SearchQuery relationships: {e}")
logger.info(f"Cleanup complete for deleted page {page_id}")
except Exception as e:
logger.error(f"Failed to cleanup deleted page {page_id}: {e}", exc_info=True)
# Health check endpoint
@router.get("/health")
async def webhook_health():
"""Health check for webhook endpoint."""
return {"status": "ok", "service": "webhooks"}
+409
View File
@@ -0,0 +1,409 @@
"""
Wiki router for Library Desk API.
Endpoints for wiki page and dossier management.
All operations are scoped to user namespaces for multi-tenancy.
"""
from fastapi import APIRouter, HTTPException, Depends, Query, Security, BackgroundTasks
from fastapi.security import HTTPAuthorizationCredentials
from typing import Optional
import logging
from src.models.wiki import (
WikiPage, WikiPageList, WikiPageCreate, WikiPageUpdate, WikiPageMove,
WikiOperationResponse, WikiSearchResponse,
DossierList, WikiSearchResult
)
from src.services.wiki_service import WikiService
from src.services.graph_service import GraphService
from src.services.vector_service import VectorService
from src.clients.wikijs_client import WikiJSClient
from src.clients.neo4j_client import Neo4jClient
from src.clients.qdrant_client import QdrantClientWrapper
from src.clients.ollama_client import OllamaClient
from src.core.dependencies import WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, verify_api_key
from src.core.multi_tenancy import DEFAULT_USER
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/wiki", tags=["Wiki"])
# Dependency to get wiki service
def get_wiki_service(wiki_client: WikiJSDep) -> WikiService:
"""Get wiki service instance."""
return WikiService(wiki_client)
# Dependency to get graph service
def get_graph_service(neo4j_client: Neo4jDep, wiki_client: WikiJSDep) -> GraphService:
"""Get graph service instance."""
return GraphService(neo4j_client, wiki_client)
# Dependency to get vector service
def get_vector_service(
qdrant_client: QdrantDep,
wiki_client: WikiJSDep,
ollama_client: OllamaDep
) -> VectorService:
"""Get vector service instance."""
return VectorService(qdrant_client, wiki_client, ollama_client)
# Page operations
@router.get("/pages", response_model=WikiPageList)
async def list_pages(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
tag: Optional[str] = Query(default=None, description="Filter by tag (dossier)"),
limit: int = Query(default=50, ge=1, le=200, description="Maximum pages to return"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
List wiki pages for a user.
Optionally filter by tag (dossier). Pages are scoped to user's namespace.
**Example:** `/wiki/pages?user=jpmschweitzer&tag=projects&limit=20`
"""
try:
return await wiki_service.list_pages(user=user, tag=tag, limit=limit)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to list pages: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.get("/pages/{page_id}", response_model=WikiPage)
async def get_page(
page_id: int,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
Get a single wiki page by ID.
Access is restricted to pages within the user's namespace.
**Example:** `/wiki/pages/123?user=jpmschweitzer`
"""
try:
page = await wiki_service.get_page(page_id=page_id, user=user)
if not page:
raise HTTPException(status_code=404, detail=f"Page {page_id} not found or access denied")
return page
except HTTPException:
raise
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to get page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/pages", response_model=WikiPage, status_code=201)
async def create_page(
page_data: WikiPageCreate,
background_tasks: BackgroundTasks,
wiki_service: WikiService = Depends(get_wiki_service),
graph_service: GraphService = Depends(get_graph_service),
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
Create a new wiki page.
The page will be created in the user's namespace. If path doesn't start
with namespace, it will be automatically prefixed.
**Auto-updates knowledge graph and vector embeddings**: After creating
the page, the graph and vectors are automatically updated in the
background to extract entities/relationships and generate semantic embeddings.
**Example Request:**
```json
{
"title": "Library Desk Architecture",
"path": "/projects/library-desk/architecture",
"content": "# Architecture\\n\\nThis describes...",
"description": "Architecture documentation",
"tags": ["projects", "architecture"],
"user": "jpmschweitzer"
}
```
"""
try:
page = await wiki_service.create_page(page_data)
user = page_data.user or DEFAULT_USER
# Schedule BOTH graph and vector updates in background (non-blocking)
background_tasks.add_task(
graph_service.update_from_page,
page_id=page.id,
user=user
)
background_tasks.add_task(
vector_service.update_from_page,
page_id=page.id,
user=user
)
logger.info(f"Page {page.id} created, graph and vector updates scheduled")
return page
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to create page: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.put("/pages/{page_id}", response_model=WikiPage)
async def update_page(
page_id: int,
page_data: WikiPageUpdate,
background_tasks: BackgroundTasks,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
wiki_service: WikiService = Depends(get_wiki_service),
graph_service: GraphService = Depends(get_graph_service),
vector_service: VectorService = Depends(get_vector_service),
api_key: str = Depends(verify_api_key)
):
"""
Update an existing wiki page.
Only pages within the user's namespace can be updated.
Partial updates are supported - only provided fields will be updated.
**Auto-updates knowledge graph and vector embeddings**: After updating
the page, the graph and vectors are automatically refreshed in the
background to reflect the changes.
**Example Request:**
```json
{
"title": "Updated Title",
"tags": ["projects", "updated"]
}
```
"""
try:
page = await wiki_service.update_page(
page_id=page_id,
page_data=page_data,
user=user
)
# Schedule BOTH graph and vector updates in background (non-blocking)
background_tasks.add_task(
graph_service.update_from_page,
page_id=page_id,
user=user,
force_refresh=True # Force refresh on updates
)
background_tasks.add_task(
vector_service.update_from_page,
page_id=page_id,
user=user,
force_refresh=True # Force refresh on updates
)
logger.info(f"Page {page_id} updated, graph and vector refresh scheduled")
return page
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to update page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.delete("/pages/{page_id}", response_model=WikiOperationResponse)
async def delete_page(
page_id: int,
background_tasks: BackgroundTasks,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
wiki_service: WikiService = Depends(get_wiki_service),
vector_service: VectorService = Depends(get_vector_service),
graph_service: GraphService = Depends(get_graph_service),
api_key: str = Depends(verify_api_key)
):
"""
Delete a wiki page.
Only pages within the user's namespace can be deleted.
This operation cannot be undone.
**Auto-cleanup**: Vector chunks and graph nodes for this page are
automatically deleted in the background.
"""
try:
success = await wiki_service.delete_page(page_id=page_id, user=user)
# Schedule vector cleanup in background
background_tasks.add_task(
vector_service.delete_page_chunks,
page_id=page_id,
user=user
)
# Schedule graph cleanup in background
background_tasks.add_task(
graph_service.delete_page,
page_id=page_id,
user=user
)
logger.info(f"Page {page_id} deleted, vector and graph cleanup scheduled")
return WikiOperationResponse(
success=success,
message=f"Page {page_id} deleted successfully",
page_id=page_id
)
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
except Exception as e:
logger.error(f"Failed to delete page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/pages/{page_id}/move", response_model=WikiOperationResponse)
async def move_page(
page_id: int,
move_data: WikiPageMove,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
Move or rename a wiki page.
The new path must be within the user's namespace.
**Example Request:**
```json
{
"new_path": "/projects/library-desk/docs/architecture"
}
```
"""
try:
success = await wiki_service.move_page(
page_id=page_id,
new_path=move_data.new_path,
user=user
)
if not success:
raise HTTPException(status_code=500, detail="Failed to move page")
return WikiOperationResponse(
success=True,
message=f"Page {page_id} moved successfully",
page_id=page_id,
page_path=move_data.new_path
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to move page {page_id}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
# Search operations
@router.get("/search", response_model=WikiSearchResponse)
async def search_pages(
q: str = Query(..., min_length=1, description="Search query"),
user: str = Query(default=DEFAULT_USER, description="User identifier"),
limit: int = Query(default=20, ge=1, le=100, description="Maximum results"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
Search wiki pages within user's namespace.
**Example:** `/wiki/search?q=architecture&user=jpmschweitzer&limit=10`
"""
try:
results = await wiki_service.search_pages(
query=q,
user=user,
limit=limit
)
return WikiSearchResponse(
results=[
WikiSearchResult(
id=r.id,
path=r.path,
title=r.title,
description=r.description,
relevance=None
)
for r in results
],
query=q,
total=len(results)
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Search failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
# Dossier operations
@router.get("/dossiers", response_model=DossierList)
async def list_dossiers(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
List all dossiers (unique tags) for a user.
Dossiers are tag-based collections of pages.
**Example:** `/wiki/dossiers?user=jpmschweitzer`
"""
try:
return await wiki_service.list_dossiers(user=user)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to list dossiers: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")
@router.get("/dossiers/{dossier_name}/pages", response_model=WikiPageList)
async def get_dossier_pages(
dossier_name: str,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
limit: int = Query(default=100, ge=1, le=500, description="Maximum pages"),
wiki_service: WikiService = Depends(get_wiki_service),
api_key: str = Depends(verify_api_key)
):
"""
Get all pages in a dossier.
Returns pages tagged with the dossier name.
**Example:** `/wiki/dossiers/projects/pages?user=jpmschweitzer`
"""
try:
return await wiki_service.get_dossier_pages(
dossier_name=dossier_name,
user=user,
limit=limit
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
logger.error(f"Failed to get dossier pages: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Internal server error")