Initial commit: library-desk service extraction from portainer-core
Build and Push / build (release) Successful in 36s
Build and Push / build (release) Successful in 36s
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Knowledge Consolidation router for Librarian processing.
|
||||
|
||||
Provides endpoints for the Scheduler to trigger knowledge consolidation
|
||||
from HybridRAG search results into wiki pages.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
import logging
|
||||
|
||||
from src.models.consolidation import ConsolidationRequest, ConsolidationResponse
|
||||
from src.services.consolidation_service import ConsolidationService
|
||||
from src.core.dependencies import (
|
||||
Neo4jDep, OllamaDep, WikiJSDep,
|
||||
verify_api_key, get_settings, get_ingestion_service
|
||||
)
|
||||
from src.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/consolidate", tags=["Consolidation"])
|
||||
|
||||
|
||||
# Dependency to get Consolidation service
|
||||
def get_consolidation_service(
|
||||
neo4j_client: Neo4jDep,
|
||||
ollama_client: OllamaDep,
|
||||
wiki_client: WikiJSDep,
|
||||
settings: Settings = Depends(get_settings)
|
||||
) -> ConsolidationService:
|
||||
"""Get ConsolidationService instance with all dependencies."""
|
||||
return ConsolidationService(
|
||||
neo4j=neo4j_client,
|
||||
ollama=ollama_client,
|
||||
wiki=wiki_client,
|
||||
settings=settings,
|
||||
ingestion_service=get_ingestion_service()
|
||||
)
|
||||
|
||||
|
||||
@router.post("/knowledge", response_model=ConsolidationResponse)
|
||||
async def consolidate_knowledge(
|
||||
request: ConsolidationRequest,
|
||||
consolidation_service: ConsolidationService = Depends(get_consolidation_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Consolidate knowledge from HybridRAG search results into wiki pages.
|
||||
|
||||
**Librarian Task** - Processes unprocessed SearchQuery nodes to:
|
||||
1. Find searches with web results from last N days
|
||||
2. Analyze web content with LLM for novel information
|
||||
3. Create new wiki pages for new concepts/technologies
|
||||
4. Update existing pages with new facts and citations
|
||||
5. Add new entities to knowledge graph
|
||||
6. Mark SearchQuery nodes as processed
|
||||
|
||||
**Typically called by The Scheduler** on a periodic basis (e.g., hourly).
|
||||
|
||||
**Parameters:**
|
||||
- `process_limit`: Maximum searches to process per run (default: 10)
|
||||
- `lookback_days`: Only process searches from last N days (default: 7)
|
||||
- `min_web_results`: Minimum web results required to consolidate (default: 2)
|
||||
- `dry_run`: If true, analyze but don't create pages (default: false)
|
||||
|
||||
**Returns:**
|
||||
- `total_found`: Number of unprocessed searches found
|
||||
- `processed_count`: Successfully processed searches
|
||||
- `pages_created`: New wiki pages created
|
||||
- `pages_updated`: Existing pages updated with new facts
|
||||
- `entities_added`: New entities added to knowledge graph
|
||||
- `errors`: List of error messages if any failed
|
||||
- `results`: Per-search processing results
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"process_limit": 10,
|
||||
"lookback_days": 7,
|
||||
"min_web_results": 2,
|
||||
"dry_run": false
|
||||
}
|
||||
```
|
||||
|
||||
**Example Response:**
|
||||
```json
|
||||
{
|
||||
"total_found": 5,
|
||||
"processed_count": 4,
|
||||
"pages_created": 2,
|
||||
"pages_updated": 3,
|
||||
"entities_added": 7,
|
||||
"errors": ["Search abc123: Failed to parse response"],
|
||||
"results": [
|
||||
{
|
||||
"search_id": "uuid-1",
|
||||
"query": "docker orchestration kubernetes",
|
||||
"pages_created": 1,
|
||||
"pages_updated": 1,
|
||||
"entities_added": 3
|
||||
}
|
||||
],
|
||||
"dry_run": false
|
||||
}
|
||||
```
|
||||
|
||||
**Scheduler Task Configuration:**
|
||||
```json
|
||||
{
|
||||
"task_name": "knowledge_consolidation",
|
||||
"service": "library-desk",
|
||||
"executor": "rest_api_executor",
|
||||
"priority": 50,
|
||||
"minute": 0,
|
||||
"hour": -1,
|
||||
"description": "Hourly knowledge consolidation from search results",
|
||||
"config": {
|
||||
"url": "http://library-desk:8089/consolidate/knowledge",
|
||||
"method": "POST",
|
||||
"payload": {
|
||||
"process_limit": 10,
|
||||
"lookback_days": 7,
|
||||
"min_web_results": 2,
|
||||
"dry_run": false
|
||||
},
|
||||
"auth": {
|
||||
"type": "bearer",
|
||||
"token": "${LIBRARY_DESK_API_KEY}"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
"""
|
||||
try:
|
||||
logger.info(
|
||||
f"Knowledge consolidation requested: "
|
||||
f"limit={request.process_limit}, lookback={request.lookback_days}d, "
|
||||
f"dry_run={request.dry_run}"
|
||||
)
|
||||
|
||||
response = await consolidation_service.consolidate_knowledge(
|
||||
process_limit=request.process_limit,
|
||||
lookback_days=request.lookback_days,
|
||||
min_web_results=request.min_web_results,
|
||||
dry_run=request.dry_run
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Consolidation completed: {response.processed_count}/{response.total_found} searches, "
|
||||
f"{response.pages_created} pages created, {response.pages_updated} updated"
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"Invalid request: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Knowledge consolidation failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Consolidation failed")
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
Entity Linking Router
|
||||
|
||||
Finds and links mentions of existing entities in wiki pages to the knowledge graph.
|
||||
Creates both:
|
||||
1. Graph relationships (MENTIONS) in Neo4j
|
||||
2. Hyperlinks in wiki page content (markdown links)
|
||||
"""
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
import re
|
||||
|
||||
from src.config import get_settings
|
||||
|
||||
from src.core.dependencies import (
|
||||
get_wiki_service,
|
||||
get_graph_service,
|
||||
get_ingestion_service,
|
||||
verify_api_key
|
||||
)
|
||||
from src.services.wiki_service import WikiService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.models.wiki import WikiPageUpdate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/entity-linking", tags=["Entity Linking"])
|
||||
|
||||
|
||||
class EntityLinkingResult(BaseModel):
|
||||
"""Result of entity linking operation."""
|
||||
page_id: int
|
||||
page_title: str
|
||||
entities_found: List[Dict[str, Any]]
|
||||
new_graph_links_created: int # New MENTIONS relationships in Neo4j
|
||||
total_graph_links: int
|
||||
content_links_added: int # New hyperlinks added to wiki page content
|
||||
content_updated: bool # Whether wiki page content was modified
|
||||
re_indexed: bool
|
||||
processing_time_ms: float
|
||||
|
||||
|
||||
class EntityLinkingRequest(BaseModel):
|
||||
"""Request for entity linking."""
|
||||
user: str
|
||||
page_id: int
|
||||
create_relationships: bool = True
|
||||
re_index_if_changed: bool = True
|
||||
|
||||
|
||||
@router.post("/link-page", response_model=EntityLinkingResult)
|
||||
async def link_entities_in_page(
|
||||
request: EntityLinkingRequest,
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
ingestion_service: IngestionService = Depends(get_ingestion_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
) -> EntityLinkingResult:
|
||||
"""
|
||||
Find and link entities mentioned in a wiki page.
|
||||
|
||||
This endpoint:
|
||||
1. Retrieves the page content from Wiki.js
|
||||
2. Fetches all existing entities from the knowledge graph
|
||||
3. Finds mentions of those entities in the page text
|
||||
4. Creates MENTIONS relationships in Neo4j
|
||||
5. Optionally re-indexes the page if new links were created
|
||||
|
||||
Args:
|
||||
request: EntityLinkingRequest with page_id and user
|
||||
|
||||
Returns:
|
||||
EntityLinkingResult with statistics about linked entities
|
||||
"""
|
||||
import time
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
logger.info(f"Entity linking for page {request.page_id} (user: {request.user})")
|
||||
|
||||
# Step 1: Get page content
|
||||
page = await wiki_service.get_page(request.page_id, request.user)
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail=f"Page {request.page_id} not found")
|
||||
|
||||
page_title = page.title
|
||||
page_content = page.content
|
||||
original_content = page_content # Keep original for comparison
|
||||
|
||||
logger.info(f"Processing page: {page_title}")
|
||||
|
||||
# Step 2: Get all entities from knowledge graph with their wiki page paths
|
||||
entities = await get_entities_with_paths(graph_service, request.user)
|
||||
logger.info(f"Found {len(entities)} existing entities in graph")
|
||||
|
||||
# Step 3: Find entity mentions in page content
|
||||
found_entities = find_entity_mentions(page_content, entities)
|
||||
logger.info(f"Found {len(found_entities)} entity mentions in page")
|
||||
|
||||
# Filter out self-referential links (entities linking to the current page)
|
||||
found_entities = [
|
||||
e for e in found_entities
|
||||
if e.get("doc_page_id") != request.page_id
|
||||
]
|
||||
logger.info(f"After filtering self-references: {len(found_entities)} entities to link")
|
||||
|
||||
# Step 4: Add hyperlinks to wiki content for entities with pages
|
||||
content_links_added = 0
|
||||
content_updated = False
|
||||
|
||||
if found_entities:
|
||||
updated_content, content_links_added = add_entity_links_to_content(
|
||||
page_content,
|
||||
found_entities
|
||||
)
|
||||
|
||||
if updated_content != original_content:
|
||||
content_updated = True
|
||||
logger.info(f"Added {content_links_added} hyperlinks to page content")
|
||||
|
||||
# Update the wiki page (preserve existing title, description, tags)
|
||||
try:
|
||||
update_data = WikiPageUpdate(
|
||||
content=updated_content,
|
||||
title=page.title,
|
||||
description=page.description if hasattr(page, 'description') else None,
|
||||
tags=page.tags if hasattr(page, 'tags') else None
|
||||
)
|
||||
await wiki_service.update_page(
|
||||
page_id=request.page_id,
|
||||
page_data=update_data,
|
||||
user=request.user
|
||||
)
|
||||
logger.info(f"Updated wiki page {request.page_id} with entity links")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update wiki page content: {e}")
|
||||
# Continue anyway - graph links can still be created
|
||||
|
||||
# Step 5: Create graph relationships if requested
|
||||
new_graph_links = 0
|
||||
total_graph_links = 0
|
||||
|
||||
if request.create_relationships and found_entities:
|
||||
new_graph_links = await graph_service.create_entity_mentions(
|
||||
page_id=request.page_id,
|
||||
user=request.user,
|
||||
entity_names=found_entities
|
||||
)
|
||||
total_graph_links = len(found_entities)
|
||||
logger.info(f"Created {new_graph_links} new MENTIONS relationships")
|
||||
|
||||
# Step 6: Re-index if changes were made (content or graph)
|
||||
re_indexed = False
|
||||
if request.re_index_if_changed and (content_updated or new_graph_links > 0):
|
||||
logger.info(f"Re-indexing page {request.page_id} due to entity linking changes")
|
||||
try:
|
||||
await ingestion_service.ingest_page(request.page_id, request.user)
|
||||
re_indexed = True
|
||||
except Exception as e:
|
||||
logger.error(f"Re-indexing failed: {e}")
|
||||
# Don't fail the whole operation if re-indexing fails
|
||||
|
||||
processing_time = (time.time() - start_time) * 1000
|
||||
|
||||
return EntityLinkingResult(
|
||||
page_id=request.page_id,
|
||||
page_title=page_title,
|
||||
entities_found=found_entities,
|
||||
new_graph_links_created=new_graph_links,
|
||||
total_graph_links=total_graph_links,
|
||||
content_links_added=content_links_added,
|
||||
content_updated=content_updated,
|
||||
re_indexed=re_indexed,
|
||||
processing_time_ms=processing_time
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Entity linking failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Entity linking failed: {str(e)}")
|
||||
|
||||
|
||||
def find_entity_mentions(content: str, entities: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find mentions of entities in the page content.
|
||||
|
||||
Uses case-insensitive regex matching to find entity names in the text.
|
||||
|
||||
Args:
|
||||
content: Page content to search
|
||||
entities: List of entities with 'name', 'type', and optionally 'path' fields
|
||||
|
||||
Returns:
|
||||
List of found entities with additional 'mentions' field
|
||||
"""
|
||||
found = []
|
||||
content_lower = content.lower()
|
||||
|
||||
for entity in entities:
|
||||
entity_name = entity.get("name", "")
|
||||
if not entity_name or len(entity_name) < 3:
|
||||
continue
|
||||
|
||||
# Create regex pattern for whole word matching
|
||||
# This avoids matching "John" in "Johnson"
|
||||
pattern = r'\b' + re.escape(entity_name.lower()) + r'\b'
|
||||
|
||||
# Find all matches
|
||||
matches = list(re.finditer(pattern, content_lower))
|
||||
|
||||
if matches:
|
||||
found.append({
|
||||
"name": entity_name,
|
||||
"type": entity.get("type", "unknown"),
|
||||
"mentions": len(matches),
|
||||
"entity_id": entity.get("id"),
|
||||
"path": entity.get("path"), # Include path if available
|
||||
"doc_page_id": entity.get("doc_page_id") # Include for self-reference filtering
|
||||
})
|
||||
|
||||
# Sort by number of mentions (descending)
|
||||
found.sort(key=lambda x: x["mentions"], reverse=True)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def fuzzy_match_entity_to_document(
|
||||
entity_name: str,
|
||||
doc_title: str,
|
||||
settings
|
||||
) -> Tuple[bool, float]:
|
||||
"""
|
||||
Match entity name to document title with tolerance for variations.
|
||||
|
||||
Uses multiple strategies with confidence scoring:
|
||||
1. Exact match (confidence: 1.0)
|
||||
2. Containment match (confidence: 0.85-0.80)
|
||||
3. Token overlap match (confidence: 0.60-0.80)
|
||||
|
||||
Args:
|
||||
entity_name: Entity name to match
|
||||
doc_title: Document title to match against
|
||||
settings: Application settings with matching thresholds
|
||||
|
||||
Returns:
|
||||
Tuple of (is_match, confidence_score)
|
||||
"""
|
||||
e_lower = entity_name.lower().strip()
|
||||
d_lower = doc_title.lower().strip()
|
||||
|
||||
# Strategy 1: Exact match (confidence: 1.0)
|
||||
if e_lower == d_lower:
|
||||
return True, 1.0
|
||||
|
||||
# Strategy 2: Containment match (confidence: 0.85-0.80)
|
||||
# Require minimum length to avoid false positives
|
||||
if len(entity_name) >= settings.entity_linking_min_entity_length:
|
||||
# Entity is substring of title: "lingecollege" in "RSG Lingecollege"
|
||||
if e_lower in d_lower:
|
||||
confidence = len(entity_name) / len(doc_title)
|
||||
if confidence >= settings.entity_linking_min_containment_ratio:
|
||||
return True, 0.85
|
||||
|
||||
# Title is substring of entity: "Google" in "Google Cloud Platform"
|
||||
if d_lower in e_lower:
|
||||
confidence = len(doc_title) / len(entity_name)
|
||||
if confidence >= settings.entity_linking_min_containment_ratio:
|
||||
return True, 0.80
|
||||
|
||||
# Strategy 3: Word-level overlap (confidence: 0.60-0.80)
|
||||
entity_tokens = set(e_lower.split())
|
||||
title_tokens = set(d_lower.split())
|
||||
|
||||
# Remove common stop words to reduce false positives
|
||||
stop_words = {'the', 'a', 'an', 'of', 'for', 'and', 'or', 'in', 'on', 'at', 'to'}
|
||||
entity_tokens -= stop_words
|
||||
title_tokens -= stop_words
|
||||
|
||||
if entity_tokens and title_tokens:
|
||||
overlap = len(entity_tokens & title_tokens)
|
||||
total = len(entity_tokens | title_tokens)
|
||||
overlap_ratio = overlap / total
|
||||
|
||||
# Require significant overlap to avoid weak matches
|
||||
if overlap_ratio >= settings.entity_linking_min_token_overlap:
|
||||
confidence = 0.75 * overlap_ratio # Scale: 0.45-0.75
|
||||
return True, confidence
|
||||
|
||||
return False, 0.0
|
||||
|
||||
|
||||
async def get_entities_with_paths(graph_service: GraphService, user: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all entities and check which ones have corresponding wiki pages.
|
||||
|
||||
Uses fuzzy matching to handle name variations, typos, and partial matches
|
||||
while minimizing false positives through confidence thresholds.
|
||||
|
||||
Returns entities with their names, types, paths, and match confidence.
|
||||
"""
|
||||
from src.core.multi_tenancy import get_neo4j_user_base_label
|
||||
|
||||
settings = get_settings()
|
||||
user_base_label = get_neo4j_user_base_label(user)
|
||||
|
||||
# Step 1: Get all entities
|
||||
entities_query = f"""
|
||||
MATCH (e:{user_base_label})
|
||||
WHERE NOT e:Document
|
||||
RETURN e.name as name,
|
||||
e.type as type,
|
||||
e.id as id
|
||||
"""
|
||||
|
||||
# Step 2: Get all documents
|
||||
documents_query = """
|
||||
MATCH (d:Document)
|
||||
RETURN d.title as title,
|
||||
d.path as path,
|
||||
d.page_id as page_id
|
||||
"""
|
||||
|
||||
try:
|
||||
entities = await graph_service.neo4j.execute_query(entities_query, {})
|
||||
documents = await graph_service.neo4j.execute_query(documents_query, {})
|
||||
|
||||
# Step 3: Fuzzy match entities to documents
|
||||
results = []
|
||||
for entity in entities:
|
||||
entity_name = entity["name"]
|
||||
if not entity_name:
|
||||
continue
|
||||
|
||||
best_match = None
|
||||
best_confidence = 0.0
|
||||
|
||||
# Try to match this entity to any document
|
||||
for doc in documents:
|
||||
doc_title = doc["title"]
|
||||
if not doc_title:
|
||||
continue
|
||||
|
||||
is_match, confidence = fuzzy_match_entity_to_document(
|
||||
entity_name,
|
||||
doc_title,
|
||||
settings
|
||||
)
|
||||
|
||||
if is_match and confidence > best_confidence:
|
||||
best_match = doc
|
||||
best_confidence = confidence
|
||||
|
||||
# Only include matches above minimum confidence threshold
|
||||
if best_match and best_confidence >= settings.entity_linking_min_confidence:
|
||||
results.append({
|
||||
"name": entity_name,
|
||||
"type": entity.get("type", "unknown"),
|
||||
"id": entity.get("id"),
|
||||
"path": best_match["path"],
|
||||
"doc_page_id": best_match["page_id"],
|
||||
"match_confidence": best_confidence # NEW: track confidence
|
||||
})
|
||||
else:
|
||||
# No matching document found (or below threshold)
|
||||
results.append({
|
||||
"name": entity_name,
|
||||
"type": entity.get("type", "unknown"),
|
||||
"id": entity.get("id"),
|
||||
"path": None,
|
||||
"doc_page_id": None,
|
||||
"match_confidence": 0.0
|
||||
})
|
||||
|
||||
logger.info(f"Matched {len([r for r in results if r['path']])} entities to documents (threshold: {settings.entity_linking_min_confidence})")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get entities with paths: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def add_entity_links_to_content(
|
||||
content: str,
|
||||
entities_with_paths: List[Dict[str, Any]]
|
||||
) -> Tuple[str, int]:
|
||||
"""
|
||||
Add markdown hyperlinks for entities in the content.
|
||||
|
||||
Only links entities that:
|
||||
1. Have a wiki page (path is not None)
|
||||
2. Are not already inside existing links
|
||||
3. Are not already linked in the content
|
||||
|
||||
Returns:
|
||||
Tuple of (updated_content, links_added_count)
|
||||
"""
|
||||
if not entities_with_paths:
|
||||
return content, 0
|
||||
|
||||
# Filter to only entities with paths
|
||||
linkable_entities = [e for e in entities_with_paths if e.get("path")]
|
||||
if not linkable_entities:
|
||||
return content, 0
|
||||
|
||||
# Sort by length (longest first) to avoid partial replacements
|
||||
# e.g., "Machine Learning" before "Machine"
|
||||
linkable_entities.sort(key=lambda x: len(x["name"]), reverse=True)
|
||||
|
||||
updated_content = content
|
||||
links_added = 0
|
||||
|
||||
for entity in linkable_entities:
|
||||
entity_name = entity["name"]
|
||||
entity_path = entity["path"]
|
||||
|
||||
# Skip short entity names to avoid false positives
|
||||
if len(entity_name) < 3:
|
||||
continue
|
||||
|
||||
# Create markdown link with full path (including user namespace)
|
||||
# Wiki.js expects full paths like /users/jpmschweitzer/...
|
||||
markdown_link = f"[{entity_name}](/{entity_path})"
|
||||
|
||||
# Find all existing markdown links to protect them (recompute each iteration)
|
||||
link_pattern = r'\[([^\]]+)\]\([^\)]+\)'
|
||||
existing_links = list(re.finditer(link_pattern, updated_content))
|
||||
|
||||
# Create a list of protected ranges (start, end) for ENTIRE links (text + URL)
|
||||
# This prevents linking entity names inside existing link URLs
|
||||
protected_ranges = [(m.start(), m.end()) for m in existing_links]
|
||||
|
||||
# Find all potential matches
|
||||
pattern = r'\b(' + re.escape(entity_name) + r')\b'
|
||||
matches = list(re.finditer(pattern, updated_content, flags=re.IGNORECASE))
|
||||
|
||||
# Filter out matches that are inside existing links
|
||||
valid_matches = []
|
||||
for match in matches:
|
||||
match_start = match.start()
|
||||
match_end = match.end()
|
||||
|
||||
# Check if this match is inside any protected range
|
||||
inside_link = False
|
||||
for prot_start, prot_end in protected_ranges:
|
||||
if prot_start <= match_start < prot_end or prot_start < match_end <= prot_end:
|
||||
inside_link = True
|
||||
break
|
||||
|
||||
# Also check if already a link (pattern like [entity_name](...))
|
||||
if match_end < len(updated_content) - 1:
|
||||
next_chars = updated_content[match_end:match_end+2]
|
||||
if next_chars == '](':
|
||||
inside_link = True
|
||||
|
||||
if not inside_link:
|
||||
valid_matches.append(match)
|
||||
|
||||
if not valid_matches:
|
||||
continue
|
||||
|
||||
# Replace valid matches in reverse order (to preserve positions)
|
||||
for match in reversed(valid_matches):
|
||||
updated_content = (
|
||||
updated_content[:match.start()] +
|
||||
markdown_link +
|
||||
updated_content[match.end():]
|
||||
)
|
||||
links_added += 1
|
||||
|
||||
return updated_content, links_added
|
||||
@@ -0,0 +1,253 @@
|
||||
"""
|
||||
Graph router for Library Desk API.
|
||||
|
||||
Endpoints for Neo4j knowledge graph operations.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query
|
||||
from typing import Optional, List
|
||||
import logging
|
||||
|
||||
from src.models.graph import (
|
||||
CypherQueryRequest, CypherQueryResponse,
|
||||
UpdateFromPageRequest, GraphUpdateSummary,
|
||||
NodeListResponse, GraphNodeDetail,
|
||||
MindMapResponse
|
||||
)
|
||||
from src.services.graph_service import GraphService
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.core.dependencies import Neo4jDep, WikiJSDep, verify_api_key
|
||||
from src.core.multi_tenancy import DEFAULT_USER
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/graph", tags=["Graph"])
|
||||
|
||||
|
||||
# Dependency to get graph service
|
||||
def get_graph_service(
|
||||
neo4j_client: Neo4jDep,
|
||||
wiki_client: WikiJSDep
|
||||
) -> GraphService:
|
||||
"""Get graph service instance."""
|
||||
return GraphService(neo4j_client, wiki_client)
|
||||
|
||||
|
||||
@router.post("/query", response_model=CypherQueryResponse)
|
||||
async def execute_cypher_query(
|
||||
request: CypherQueryRequest,
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Execute a user-scoped Cypher query.
|
||||
|
||||
The query is automatically scoped to the user's data for security.
|
||||
This prevents users from accessing other users' graph data.
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"query": "MATCH (d:Document) RETURN d LIMIT 10",
|
||||
"parameters": {},
|
||||
"user": "jpmschweitzer"
|
||||
}
|
||||
```
|
||||
|
||||
**Security:** Query is automatically scoped with user label.
|
||||
"""
|
||||
try:
|
||||
return await graph_service.execute_query(
|
||||
query=request.query,
|
||||
parameters=request.parameters,
|
||||
user=request.user
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Cypher query failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Query execution failed")
|
||||
|
||||
|
||||
@router.get("/nodes", response_model=NodeListResponse)
|
||||
async def list_nodes(
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
node_type: Optional[str] = Query(default=None, description="Node type filter"),
|
||||
limit: int = Query(default=100, ge=1, le=500, description="Maximum nodes"),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
List graph nodes for a user.
|
||||
|
||||
Optionally filter by node type (Document, Person, Project, Concept, etc.).
|
||||
|
||||
**Example:** `/graph/nodes?user=jpmschweitzer&node_type=Document&limit=50`
|
||||
"""
|
||||
try:
|
||||
return await graph_service.list_nodes(
|
||||
user=user,
|
||||
node_type=node_type,
|
||||
limit=limit
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list nodes: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to list nodes")
|
||||
|
||||
|
||||
@router.get("/nodes/{node_id}", response_model=GraphNodeDetail)
|
||||
async def get_node(
|
||||
node_id: str,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Get detailed information about a graph node.
|
||||
|
||||
Returns the node, its relationships, and connected nodes.
|
||||
|
||||
**Example:** `/graph/nodes/4:abc123def:0?user=jpmschweitzer`
|
||||
"""
|
||||
try:
|
||||
node = await graph_service.get_node(node_id, user)
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail=f"Node {node_id} not found")
|
||||
return node
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get node {node_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to get node")
|
||||
|
||||
|
||||
@router.post("/update-from-page/{page_id}", response_model=GraphUpdateSummary)
|
||||
async def update_graph_from_page(
|
||||
page_id: int,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
force_refresh: bool = Query(default=False, description="Force re-extraction"),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Update knowledge graph from a wiki page.
|
||||
|
||||
This endpoint:
|
||||
1. Fetches the page from Wiki.js
|
||||
2. Extracts entities (people, projects, concepts, technologies)
|
||||
3. Creates/updates Document node
|
||||
4. Creates entity nodes and MENTIONS relationships
|
||||
5. Returns summary of what was updated
|
||||
|
||||
**Use Cases:**
|
||||
- Called automatically after page creation/update (via BackgroundTasks)
|
||||
- Called manually by user/Librarian to refresh graph
|
||||
- Called by Scheduler for batch processing
|
||||
|
||||
**Example:** `POST /graph/update-from-page/4?user=jpmschweitzer`
|
||||
|
||||
**Returns:** Summary with nodes/relationships created and entities extracted
|
||||
"""
|
||||
try:
|
||||
summary = await graph_service.update_from_page(
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=force_refresh
|
||||
)
|
||||
|
||||
if not summary.success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Graph update failed: {summary.error_message}"
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update graph from page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Graph update failed")
|
||||
|
||||
|
||||
@router.post("/mindmap", response_model=MindMapResponse)
|
||||
async def generate_mindmap(
|
||||
center_node_id: str = Query(..., description="Central node ID"),
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
depth: int = Query(default=2, ge=1, le=5, description="Traversal depth"),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Generate mind map data for visualization.
|
||||
|
||||
Traverses the graph from a center node and returns nodes/links
|
||||
in a format suitable for D3.js or similar visualization libraries.
|
||||
|
||||
**Parameters:**
|
||||
- `center_node_id`: The node to center the mind map on
|
||||
- `depth`: How many hops away from center to include (1-5)
|
||||
- `user`: User identifier for scoping
|
||||
|
||||
**Example:** `POST /graph/mindmap?center_node_id=4:abc:0&depth=2`
|
||||
|
||||
**Returns:** Nodes and links for visualization
|
||||
"""
|
||||
try:
|
||||
return await graph_service.generate_mindmap(
|
||||
center_node_id=center_node_id,
|
||||
user=user,
|
||||
depth=depth
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate mindmap: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Mindmap generation failed")
|
||||
|
||||
|
||||
@router.post("/generate-entity-pages")
|
||||
async def generate_entity_pages(
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
min_mentions: int = Query(default=5, ge=1, le=100, description="Minimum mentions threshold"),
|
||||
entity_types: Optional[List[str]] = Query(default=None, description="Entity types to process"),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Generate wiki stub pages for graph entities.
|
||||
|
||||
Creates pages in `/entities/{type}/{name}` namespace for entities
|
||||
that have been mentioned in multiple documents. This creates a
|
||||
bidirectional knowledge graph ↔ wiki synchronization.
|
||||
|
||||
**Threshold:** Entities must be mentioned in at least `min_mentions` documents (default: 5)
|
||||
|
||||
**Auto-stub flow:**
|
||||
1. Find entities with >= min_mentions
|
||||
2. Check if entity already has a wiki page
|
||||
3. If not, create stub page with:
|
||||
- List of mentioning documents
|
||||
- Related entities (co-occurring)
|
||||
- Auto-generated tag to prevent feedback loops
|
||||
|
||||
**Feedback loop protection:** Pages tagged with `entity-stub` skip entity extraction
|
||||
|
||||
**Parameters:**
|
||||
- `min_mentions`: Minimum number of document mentions required (default: 5)
|
||||
- `entity_types`: List of types to process (default: Person, Technology, Concept, Project)
|
||||
- `user`: User identifier for scoping
|
||||
|
||||
**Example:** `POST /graph/generate-entity-pages?min_mentions=3&user=jpmschweitzer`
|
||||
|
||||
**Returns:** Summary with list of pages created and skipped
|
||||
"""
|
||||
try:
|
||||
result = await graph_service.generate_entity_stubs(
|
||||
user=user,
|
||||
min_mentions=min_mentions,
|
||||
entity_types=entity_types
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate entity pages: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Entity page generation failed")
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
HybridRAG router for multi-source search API.
|
||||
|
||||
Provides endpoint for combining vector, graph, and web search
|
||||
with RRF fusion and LLM re-ranking.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query
|
||||
import logging
|
||||
|
||||
from src.models.hybrid_rag import HybridRAGRequest, HybridRAGResponse
|
||||
from src.services.hybrid_rag_service import HybridRAGService
|
||||
from src.services.vector_service import VectorService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.clients.searxng_client import SearXNGClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.core.dependencies import (
|
||||
Neo4jDep, WikiJSDep, QdrantDep, OllamaDep,
|
||||
SearXNGDep, verify_api_key, get_settings
|
||||
)
|
||||
from src.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/query", tags=["HybridRAG"])
|
||||
|
||||
|
||||
# Dependency to get HybridRAG service
|
||||
def get_hybrid_rag_service(
|
||||
neo4j_client: Neo4jDep,
|
||||
wiki_client: WikiJSDep,
|
||||
qdrant_client: QdrantDep,
|
||||
ollama_client: OllamaDep,
|
||||
searxng_client: SearXNGDep,
|
||||
settings: Settings = Depends(get_settings)
|
||||
) -> HybridRAGService:
|
||||
"""Get HybridRAG service instance with all dependencies."""
|
||||
from src.services.vector_service import VectorService
|
||||
from src.services.graph_service import GraphService
|
||||
|
||||
# Create component services
|
||||
vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
|
||||
graph_service = GraphService(neo4j_client, wiki_client)
|
||||
|
||||
# Create HybridRAG service
|
||||
return HybridRAGService(
|
||||
vector_service=vector_service,
|
||||
graph_service=graph_service,
|
||||
searxng_client=searxng_client,
|
||||
ollama_client=ollama_client,
|
||||
settings=settings
|
||||
)
|
||||
|
||||
|
||||
@router.post("/hybrid", response_model=HybridRAGResponse)
|
||||
async def hybrid_search(
|
||||
request: HybridRAGRequest,
|
||||
user: str = Query(default="jpmschweitzer", description="User identifier for multi-tenancy"),
|
||||
hybrid_rag_service: HybridRAGService = Depends(get_hybrid_rag_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Execute HybridRAG query combining vector, graph, and web search.
|
||||
|
||||
**6-Phase Pipeline:**
|
||||
1. **Query Enhancement**: Extract keywords/synonyms with LLM
|
||||
2. **Parallel Retrieval**: Search vector (Qdrant), graph (Neo4j), web (SearXNG)
|
||||
3. **RRF Fusion**: Merge results with Reciprocal Rank Fusion
|
||||
4. **Enrichment**: Add related documents via shared entities
|
||||
5. **LLM Re-ranking**: Re-rank with mistral-nemo for relevance
|
||||
6. **Context Formatting**: Format for LLM consumption
|
||||
7. **Persistence**: Store for Librarian knowledge consolidation
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"query": "How does Docker orchestration work with Kubernetes?",
|
||||
"user": "jpmschweitzer",
|
||||
"config": {
|
||||
"vector_limit": 10,
|
||||
"graph_limit": 10,
|
||||
"web_limit": 5,
|
||||
"enable_reranking": true,
|
||||
"final_result_count": 10
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Returns:**
|
||||
- Ranked results from all sources
|
||||
- Extracted keywords/synonyms
|
||||
- Related dossiers (via graph)
|
||||
- Formatted context for LLM
|
||||
- Performance timing breakdown
|
||||
- Search ID for Librarian tracking
|
||||
"""
|
||||
try:
|
||||
logger.info(f"HybridRAG request: '{request.query}' for user '{user}'")
|
||||
|
||||
response = await hybrid_rag_service.search(
|
||||
query=request.query,
|
||||
user=user,
|
||||
config=request.config
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"HybridRAG completed: {response.total_results} results in {response.timing.total_ms:.0f}ms"
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(f"Invalid request: {e}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"HybridRAG search failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Search failed")
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
Document Ingestion API Router
|
||||
|
||||
Endpoints for ingesting wiki pages into the knowledge base (vectors + graph).
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from typing import Optional
|
||||
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.models.ingestion import (
|
||||
IngestionRequest,
|
||||
IngestionResult,
|
||||
BatchIngestionRequest,
|
||||
BatchIngestionResult
|
||||
)
|
||||
from src.core.dependencies import get_ingestion_service, verify_api_key
|
||||
|
||||
router = APIRouter(prefix="/ingest", tags=["Document Ingestion"])
|
||||
|
||||
|
||||
@router.post("/page", response_model=IngestionResult)
|
||||
async def ingest_page(
|
||||
request: IngestionRequest,
|
||||
ingestion: IngestionService = Depends(get_ingestion_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Ingest a single wiki page into the knowledge base.
|
||||
|
||||
This endpoint:
|
||||
1. Fetches page content from Wiki.js
|
||||
2. Chunks content and generates embeddings (Qdrant)
|
||||
3. Extracts entities and updates knowledge graph (Neo4j)
|
||||
|
||||
## Use Cases
|
||||
|
||||
- **After page creation**: Automatically called by consolidation service
|
||||
- **Manual re-indexing**: Force refresh a page after manual edits
|
||||
- **Selective ingestion**: Skip vectors or graph if only one is needed
|
||||
|
||||
## Performance
|
||||
|
||||
- Typical page: 1-3 seconds
|
||||
- Large page (>5000 words): 5-10 seconds
|
||||
- Vector and graph ingestion run in parallel
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
curl -X POST http://192.168.86.149:8089/ingest/page \
|
||||
-H "Authorization: Bearer $API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"page_id": 19,
|
||||
"user": "jpmschweitzer",
|
||||
"force_refresh": false
|
||||
}'
|
||||
```
|
||||
"""
|
||||
result = await ingestion.ingest_page(
|
||||
page_id=request.page_id,
|
||||
user=request.user,
|
||||
force_refresh=request.force_refresh,
|
||||
skip_vectors=request.skip_vectors,
|
||||
skip_graph=request.skip_graph
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Ingestion failed: {result.error}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/batch", response_model=BatchIngestionResult)
|
||||
async def ingest_batch(
|
||||
request: BatchIngestionRequest,
|
||||
ingestion: IngestionService = Depends(get_ingestion_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Ingest multiple wiki pages concurrently.
|
||||
|
||||
## Concurrency Control
|
||||
|
||||
The `max_concurrent` parameter controls how many pages are processed simultaneously:
|
||||
- **Low (1-2)**: Safer for resource-constrained systems
|
||||
- **Medium (3-5)**: Good balance of speed and stability
|
||||
- **High (6-10)**: Maximum speed, requires good resources
|
||||
|
||||
## Batch Size Recommendations
|
||||
|
||||
- **Small batches (<10 pages)**: Use max_concurrent=3-5
|
||||
- **Medium batches (10-50 pages)**: Use max_concurrent=3
|
||||
- **Large batches (>50 pages)**: Use max_concurrent=2, consider splitting
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
curl -X POST http://192.168.86.149:8089/ingest/batch \
|
||||
-H "Authorization: Bearer $API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"page_ids": [19, 20, 21, 22],
|
||||
"user": "jpmschweitzer",
|
||||
"max_concurrent": 3
|
||||
}'
|
||||
```
|
||||
"""
|
||||
result = await ingestion.ingest_batch(
|
||||
page_ids=request.page_ids,
|
||||
user=request.user,
|
||||
force_refresh=request.force_refresh,
|
||||
skip_vectors=request.skip_vectors,
|
||||
skip_graph=request.skip_graph,
|
||||
max_concurrent=request.max_concurrent
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/all", response_model=BatchIngestionResult)
|
||||
async def ingest_all_pages(
|
||||
user: str = Query(default="jpmschweitzer", description="User identifier"),
|
||||
path_prefix: Optional[str] = Query(None, description="Path prefix filter (e.g., 'users/jpmschweitzer/tech')"),
|
||||
force_refresh: bool = Query(False, description="Force re-ingestion of all pages"),
|
||||
max_concurrent: int = Query(3, ge=1, le=10, description="Maximum concurrent ingestion tasks"),
|
||||
ingestion: IngestionService = Depends(get_ingestion_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Ingest all wiki pages for a user (bulk re-indexing).
|
||||
|
||||
## Use Cases
|
||||
|
||||
- **Initial setup**: Index all existing pages
|
||||
- **Full re-index**: After major schema changes
|
||||
- **Path-specific**: Re-index a specific section (e.g., all tech docs)
|
||||
|
||||
## Performance
|
||||
|
||||
- **Small wiki (<50 pages)**: 2-5 minutes
|
||||
- **Medium wiki (50-200 pages)**: 5-20 minutes
|
||||
- **Large wiki (>200 pages)**: 20+ minutes
|
||||
|
||||
**Recommendation**: Run as background job for large wikis
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
# Ingest all pages for user
|
||||
curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer" \
|
||||
-H "Authorization: Bearer $API_KEY"
|
||||
|
||||
# Ingest only tech docs
|
||||
curl -X POST "http://192.168.86.149:8089/ingest/all?user=jpmschweitzer&path_prefix=users/jpmschweitzer/tech" \
|
||||
-H "Authorization: Bearer $API_KEY"
|
||||
```
|
||||
"""
|
||||
result = await ingestion.ingest_all_pages(
|
||||
user=user,
|
||||
path_prefix=path_prefix,
|
||||
force_refresh=force_refresh,
|
||||
max_concurrent=max_concurrent
|
||||
)
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,334 @@
|
||||
"""
|
||||
Tools router for Library Desk API.
|
||||
|
||||
Provides tool discovery endpoint for AI agents.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from src.models.tools import (
|
||||
ToolCatalog, ToolDefinition, ToolParameter, CategoryInfo, ParameterType
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/tools", tags=["Tools"])
|
||||
|
||||
|
||||
def get_wiki_tools() -> list[ToolDefinition]:
|
||||
"""Get wiki tool definitions."""
|
||||
return [
|
||||
ToolDefinition(
|
||||
name="wiki_list_pages",
|
||||
category="wiki",
|
||||
description="List wiki pages for a user with optional tag filtering",
|
||||
method="GET",
|
||||
endpoint="/wiki/pages",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer",
|
||||
example="jpmschweitzer"
|
||||
),
|
||||
ToolParameter(
|
||||
name="tag",
|
||||
type=ParameterType.STRING,
|
||||
description="Filter by tag (dossier)",
|
||||
required=False,
|
||||
example="projects"
|
||||
),
|
||||
ToolParameter(
|
||||
name="limit",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Maximum pages to return",
|
||||
required=False,
|
||||
default=50,
|
||||
example=20
|
||||
),
|
||||
],
|
||||
returns="List of pages with summaries",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_get_page",
|
||||
category="wiki",
|
||||
description="Get a single wiki page by ID with full content",
|
||||
method="GET",
|
||||
endpoint="/wiki/pages/{page_id}",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="page_id",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Page ID to retrieve",
|
||||
required=True,
|
||||
example=123
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier for access control",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
],
|
||||
returns="Complete page object with content",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_create_page",
|
||||
category="wiki",
|
||||
description="Create a new wiki page in user's namespace",
|
||||
method="POST",
|
||||
endpoint="/wiki/pages",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="title",
|
||||
type=ParameterType.STRING,
|
||||
description="Page title",
|
||||
required=True,
|
||||
example="Project Documentation"
|
||||
),
|
||||
ToolParameter(
|
||||
name="path",
|
||||
type=ParameterType.STRING,
|
||||
description="Page path (will be prefixed with user namespace)",
|
||||
required=True,
|
||||
example="/projects/my-project"
|
||||
),
|
||||
ToolParameter(
|
||||
name="content",
|
||||
type=ParameterType.STRING,
|
||||
description="Page content in Markdown format",
|
||||
required=True,
|
||||
example="# Overview\n\nThis is the content."
|
||||
),
|
||||
ToolParameter(
|
||||
name="description",
|
||||
type=ParameterType.STRING,
|
||||
description="Short page description",
|
||||
required=False,
|
||||
example="Documentation for my project"
|
||||
),
|
||||
ToolParameter(
|
||||
name="tags",
|
||||
type=ParameterType.ARRAY,
|
||||
description="Tags for categorization (dossiers)",
|
||||
required=False,
|
||||
example=["projects", "documentation"]
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
],
|
||||
returns="Created page object",
|
||||
example={
|
||||
"title": "My Project",
|
||||
"path": "/projects/my-project",
|
||||
"content": "# My Project\n\nProject description here.",
|
||||
"tags": ["projects"],
|
||||
"user": "jpmschweitzer"
|
||||
},
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_update_page",
|
||||
category="wiki",
|
||||
description="Update an existing wiki page",
|
||||
method="PUT",
|
||||
endpoint="/wiki/pages/{page_id}",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="page_id",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Page ID to update",
|
||||
required=True,
|
||||
example=123
|
||||
),
|
||||
ToolParameter(
|
||||
name="title",
|
||||
type=ParameterType.STRING,
|
||||
description="New page title",
|
||||
required=False
|
||||
),
|
||||
ToolParameter(
|
||||
name="content",
|
||||
type=ParameterType.STRING,
|
||||
description="New page content",
|
||||
required=False
|
||||
),
|
||||
ToolParameter(
|
||||
name="description",
|
||||
type=ParameterType.STRING,
|
||||
description="New description",
|
||||
required=False
|
||||
),
|
||||
ToolParameter(
|
||||
name="tags",
|
||||
type=ParameterType.ARRAY,
|
||||
description="New tags",
|
||||
required=False
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
],
|
||||
returns="Updated page object",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_delete_page",
|
||||
category="wiki",
|
||||
description="Delete a wiki page",
|
||||
method="DELETE",
|
||||
endpoint="/wiki/pages/{page_id}",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="page_id",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Page ID to delete",
|
||||
required=True,
|
||||
example=123
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
],
|
||||
returns="Success confirmation",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_search_pages",
|
||||
category="wiki",
|
||||
description="Search wiki pages by content within user's namespace",
|
||||
method="GET",
|
||||
endpoint="/wiki/search",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="q",
|
||||
type=ParameterType.STRING,
|
||||
description="Search query",
|
||||
required=True,
|
||||
example="docker configuration"
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
ToolParameter(
|
||||
name="limit",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Maximum results",
|
||||
required=False,
|
||||
default=20
|
||||
),
|
||||
],
|
||||
returns="List of matching pages",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_list_dossiers",
|
||||
category="wiki",
|
||||
description="List all dossiers (unique tags) for a user with page counts",
|
||||
method="GET",
|
||||
endpoint="/wiki/dossiers",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
],
|
||||
returns="List of dossiers with page counts",
|
||||
fast=True
|
||||
),
|
||||
ToolDefinition(
|
||||
name="wiki_get_dossier_pages",
|
||||
category="wiki",
|
||||
description="Get all pages in a specific dossier",
|
||||
method="GET",
|
||||
endpoint="/wiki/dossiers/{dossier_name}/pages",
|
||||
parameters=[
|
||||
ToolParameter(
|
||||
name="dossier_name",
|
||||
type=ParameterType.STRING,
|
||||
description="Dossier name (tag)",
|
||||
required=True,
|
||||
example="projects"
|
||||
),
|
||||
ToolParameter(
|
||||
name="user",
|
||||
type=ParameterType.STRING,
|
||||
description="User identifier",
|
||||
required=False,
|
||||
default="jpmschweitzer"
|
||||
),
|
||||
ToolParameter(
|
||||
name="limit",
|
||||
type=ParameterType.INTEGER,
|
||||
description="Maximum pages",
|
||||
required=False,
|
||||
default=100
|
||||
),
|
||||
],
|
||||
returns="List of pages in dossier",
|
||||
fast=True
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@router.get("", response_model=ToolCatalog)
|
||||
async def get_tool_catalog() -> ToolCatalog:
|
||||
"""
|
||||
Get simplified tool catalog for AI agent consumption.
|
||||
|
||||
This endpoint provides a machine-readable catalog of all Library Desk tools,
|
||||
optimized for discovery and use by AI agents like The Librarian.
|
||||
|
||||
Returns tool definitions with:
|
||||
- Clear descriptions
|
||||
- Parameter specifications
|
||||
- Usage examples
|
||||
- Performance characteristics
|
||||
"""
|
||||
wiki_tools = get_wiki_tools()
|
||||
|
||||
# Calculate category stats
|
||||
categories = {}
|
||||
for tool in wiki_tools:
|
||||
if tool.category not in categories:
|
||||
categories[tool.category] = 0
|
||||
categories[tool.category] += 1
|
||||
|
||||
category_info = [
|
||||
CategoryInfo(
|
||||
name="wiki",
|
||||
description="Wiki.js page and dossier management operations",
|
||||
tool_count=categories.get("wiki", 0)
|
||||
)
|
||||
]
|
||||
|
||||
return ToolCatalog(
|
||||
service="library-desk",
|
||||
version="1.0.0",
|
||||
base_url="http://library-desk:8089",
|
||||
categories=category_info,
|
||||
tools=wiki_tools,
|
||||
authentication="Bearer token via Authorization header (LIBRARY_API_KEY)"
|
||||
)
|
||||
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Vector router for Library Desk API.
|
||||
|
||||
Endpoints for semantic search and vector operations.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
from src.models.vector import (
|
||||
SearchRequest, SearchResponse,
|
||||
VectorUpdateRequest, VectorUpdateSummary,
|
||||
CollectionListResponse,
|
||||
DeletePageChunksRequest, DeletePageChunksResponse
|
||||
)
|
||||
from src.services.vector_service import VectorService
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.core.dependencies import QdrantDep, WikiJSDep, OllamaDep, verify_api_key
|
||||
from src.core.multi_tenancy import DEFAULT_USER
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/vector", tags=["Vector"])
|
||||
|
||||
|
||||
# Dependency to get vector service
|
||||
def get_vector_service(
|
||||
qdrant_client: QdrantDep,
|
||||
wiki_client: WikiJSDep,
|
||||
ollama_client: OllamaDep
|
||||
) -> VectorService:
|
||||
"""Get vector service instance."""
|
||||
return VectorService(qdrant_client, wiki_client, ollama_client)
|
||||
|
||||
|
||||
@router.post("/search", response_model=SearchResponse)
|
||||
async def semantic_search(
|
||||
request: SearchRequest,
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Perform semantic search across user's documents.
|
||||
|
||||
Uses Ollama to generate query embedding, then searches Qdrant
|
||||
for similar document chunks.
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"query": "how to configure docker",
|
||||
"user": "jpmschweitzer",
|
||||
"limit": 10,
|
||||
"score_threshold": 0.5
|
||||
}
|
||||
```
|
||||
|
||||
**Returns:** List of matching chunks with similarity scores
|
||||
"""
|
||||
try:
|
||||
return await vector_service.search(
|
||||
query=request.query,
|
||||
user=request.user,
|
||||
limit=request.limit,
|
||||
score_threshold=request.score_threshold
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Semantic search failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Search failed")
|
||||
|
||||
|
||||
@router.post("/update-from-page/{page_id}", response_model=VectorUpdateSummary)
|
||||
async def update_vectors_from_page(
|
||||
page_id: int,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
force_refresh: bool = Query(default=False, description="Force re-embedding"),
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Update vector embeddings from a wiki page.
|
||||
|
||||
This endpoint:
|
||||
1. Fetches the page from Wiki.js
|
||||
2. Chunks the content (500 tokens with 50 token overlap)
|
||||
3. Generates embeddings via Ollama
|
||||
4. Upserts chunks to Qdrant with metadata
|
||||
|
||||
**Use Cases:**
|
||||
- Called automatically after page creation/update (via BackgroundTasks)
|
||||
- Called manually by user/Librarian to refresh vectors
|
||||
- Called by Scheduler for batch processing
|
||||
|
||||
**Example:** `POST /vector/update-from-page/5?user=jpmschweitzer`
|
||||
|
||||
**Returns:** Summary with chunks created and processing time
|
||||
"""
|
||||
try:
|
||||
summary = await vector_service.update_from_page(
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=force_refresh
|
||||
)
|
||||
|
||||
if not summary.success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Vector update failed: {summary.error_message}"
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update vectors from page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Vector update failed")
|
||||
|
||||
|
||||
@router.delete("/pages/{page_id}", response_model=DeletePageChunksResponse)
|
||||
async def delete_page_chunks(
|
||||
page_id: int,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Delete all vector chunks for a wiki page.
|
||||
|
||||
This is automatically called when a page is deleted from the wiki.
|
||||
|
||||
**Example:** `DELETE /vector/pages/5?user=jpmschweitzer`
|
||||
"""
|
||||
try:
|
||||
deleted_count = await vector_service.delete_page_chunks(
|
||||
page_id=page_id,
|
||||
user=user
|
||||
)
|
||||
|
||||
return DeletePageChunksResponse(
|
||||
page_id=page_id,
|
||||
chunks_deleted=deleted_count,
|
||||
success=deleted_count > 0,
|
||||
message=f"Deleted {deleted_count} chunks for page {page_id}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete chunks for page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to delete chunks")
|
||||
|
||||
|
||||
@router.get("/collections", response_model=CollectionListResponse)
|
||||
async def list_collections(
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
List all Qdrant collections with statistics.
|
||||
|
||||
Returns collection names, vector counts, and point counts.
|
||||
|
||||
**Example:** `GET /vector/collections`
|
||||
"""
|
||||
try:
|
||||
return await vector_service.list_collections()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list collections: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Failed to list collections")
|
||||
@@ -0,0 +1,498 @@
|
||||
"""
|
||||
Wiki.js Webhook Handler
|
||||
|
||||
Receives webhook events from Wiki.js for page CRUD operations
|
||||
and processes them identically to AI-generated content.
|
||||
"""
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Literal
|
||||
|
||||
from src.core.dependencies import (
|
||||
get_ingestion_service,
|
||||
get_wiki_service,
|
||||
get_graph_service,
|
||||
verify_api_key
|
||||
)
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.services.consolidation_service import ConsolidationService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/webhooks", tags=["Webhooks"])
|
||||
|
||||
|
||||
class WikiJSWebhookPayload(BaseModel):
|
||||
"""
|
||||
Wiki.js webhook payload structure.
|
||||
|
||||
See: https://docs.requarks.io/webhooks
|
||||
"""
|
||||
event: Literal["page.create", "page.update", "page.delete", "page.rename"]
|
||||
page: dict # Contains: id, title, path, content, etc.
|
||||
user: dict # Contains: id, name, email
|
||||
timestamp: str
|
||||
|
||||
|
||||
class WebhookProcessingResult(BaseModel):
|
||||
"""Result of webhook processing."""
|
||||
success: bool
|
||||
page_id: int
|
||||
page_title: str
|
||||
event: str
|
||||
ingested: bool
|
||||
entity_linking: Optional[dict] = None
|
||||
error: Optional[str] = None
|
||||
processing_time_ms: float
|
||||
|
||||
|
||||
@router.post("/wikijs", response_model=WebhookProcessingResult)
|
||||
async def handle_wikijs_webhook(
|
||||
payload: WikiJSWebhookPayload,
|
||||
background_tasks: BackgroundTasks,
|
||||
ingestion_service: IngestionService = Depends(get_ingestion_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Handle Wiki.js webhook events.
|
||||
|
||||
Processes page changes identically to AI-generated content:
|
||||
1. Ingest page → Vector embeddings + Graph entity extraction
|
||||
2. Apply bidirectional entity linking
|
||||
|
||||
This ensures user-edited pages have the same structure and
|
||||
connectivity as AI-generated pages.
|
||||
|
||||
Configuration in Wiki.js:
|
||||
- Administration → Webhooks
|
||||
- Add new webhook:
|
||||
- URL: http://library-desk:8089/webhooks/wikijs
|
||||
- Events: page.create, page.update
|
||||
- Headers: Authorization: Bearer <API_KEY>
|
||||
"""
|
||||
import time
|
||||
start_time = time.time()
|
||||
|
||||
page_id = payload.page.get("id")
|
||||
page_title = payload.page.get("title")
|
||||
user_email = payload.user.get("email", "unknown")
|
||||
|
||||
# Extract user identifier from email (assumes format: user@domain)
|
||||
# Adjust this based on your user mapping strategy
|
||||
user = extract_user_from_email(user_email)
|
||||
|
||||
logger.info(
|
||||
f"Received Wiki.js webhook: {payload.event} "
|
||||
f"for page {page_id} ('{page_title}') by {user_email}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Handle different events
|
||||
if payload.event == "page.delete":
|
||||
# For deletions, clean up vectors and graph
|
||||
logger.info(f"Page {page_id} deleted, cleaning up knowledge base")
|
||||
background_tasks.add_task(
|
||||
cleanup_deleted_page,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
user=user,
|
||||
ingestion_service=ingestion_service
|
||||
)
|
||||
processing_time = (time.time() - start_time) * 1000
|
||||
return WebhookProcessingResult(
|
||||
success=True,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
event=payload.event,
|
||||
ingested=False,
|
||||
processing_time_ms=processing_time
|
||||
)
|
||||
|
||||
# For create/update events, process the page
|
||||
if payload.event in ["page.create", "page.update"]:
|
||||
# Run ingestion and entity linking in background
|
||||
# to avoid blocking the webhook response
|
||||
background_tasks.add_task(
|
||||
process_wiki_page_change,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
user=user,
|
||||
event=payload.event,
|
||||
ingestion_service=ingestion_service
|
||||
)
|
||||
|
||||
processing_time = (time.time() - start_time) * 1000
|
||||
|
||||
return WebhookProcessingResult(
|
||||
success=True,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
event=payload.event,
|
||||
ingested=True, # Will be processed in background
|
||||
processing_time_ms=processing_time
|
||||
)
|
||||
|
||||
# Handle page rename/move events
|
||||
if payload.event == "page.rename":
|
||||
# Update path and re-link if title changed
|
||||
new_path = payload.page.get("path")
|
||||
new_title = payload.page.get("title")
|
||||
old_path = payload.page.get("oldPath", new_path)
|
||||
old_title = payload.page.get("oldTitle", new_title)
|
||||
|
||||
background_tasks.add_task(
|
||||
process_page_rename,
|
||||
page_id=page_id,
|
||||
old_path=old_path,
|
||||
new_path=new_path,
|
||||
old_title=old_title,
|
||||
new_title=new_title,
|
||||
user=user,
|
||||
ingestion_service=ingestion_service
|
||||
)
|
||||
|
||||
processing_time = (time.time() - start_time) * 1000
|
||||
|
||||
return WebhookProcessingResult(
|
||||
success=True,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
event=payload.event,
|
||||
ingested=False, # Path update only, no re-embedding needed
|
||||
processing_time_ms=processing_time
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Webhook processing failed: {e}", exc_info=True)
|
||||
processing_time = (time.time() - start_time) * 1000
|
||||
|
||||
return WebhookProcessingResult(
|
||||
success=False,
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
event=payload.event,
|
||||
ingested=False,
|
||||
error=str(e),
|
||||
processing_time_ms=processing_time
|
||||
)
|
||||
|
||||
|
||||
async def process_wiki_page_change(
|
||||
page_id: int,
|
||||
page_title: str,
|
||||
user: str,
|
||||
event: str,
|
||||
ingestion_service: IngestionService
|
||||
):
|
||||
"""
|
||||
Process wiki page change identically to AI-generated content.
|
||||
|
||||
This ensures consistency between manual and AI workflows.
|
||||
|
||||
Steps:
|
||||
1. Ingest page → Vector embeddings + Graph entities
|
||||
2. Apply bidirectional entity linking
|
||||
"""
|
||||
from src.core.dependencies import get_neo4j_client, get_ollama_client, get_wikijs_client
|
||||
from src.config import get_settings
|
||||
|
||||
try:
|
||||
logger.info(f"Processing {event} for page {page_id} ('{page_title}')")
|
||||
|
||||
# STEP 1: Ingest page (vector + graph)
|
||||
logger.info(f"Ingesting page {page_id} into knowledge base")
|
||||
await ingestion_service.ingest_page(
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=(event == "page.update") # Force refresh on updates
|
||||
)
|
||||
logger.info(f"Ingestion complete for page {page_id}")
|
||||
|
||||
# STEP 2: Apply bidirectional entity linking
|
||||
logger.info(f"Applying bidirectional entity linking for page {page_id}")
|
||||
|
||||
# Import consolidation service to use the entity linking method
|
||||
settings = get_settings()
|
||||
consolidation_service = ConsolidationService(
|
||||
neo4j=get_neo4j_client(),
|
||||
ollama=get_ollama_client(),
|
||||
wiki=get_wikijs_client(),
|
||||
settings=settings,
|
||||
ingestion_service=ingestion_service
|
||||
)
|
||||
|
||||
link_stats = await consolidation_service._apply_bidirectional_entity_linking(
|
||||
page_id=page_id,
|
||||
page_title=page_title,
|
||||
user=user
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Entity linking complete for page {page_id}: "
|
||||
f"{link_stats['forward_links']} forward links, "
|
||||
f"{link_stats['backward_links']} backward links "
|
||||
f"({link_stats['pages_updated']} pages updated)"
|
||||
)
|
||||
|
||||
logger.info(f"Successfully processed {event} for page {page_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process page {page_id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
def extract_user_from_email(email: str) -> str:
|
||||
"""
|
||||
Extract user identifier from email.
|
||||
|
||||
Customize this based on your user mapping strategy:
|
||||
- Option 1: Use email prefix (user@domain → user)
|
||||
- Option 2: Map email to Wiki.js username
|
||||
- Option 3: Use email directly
|
||||
|
||||
Args:
|
||||
email: User email from Wiki.js webhook
|
||||
|
||||
Returns:
|
||||
User identifier for multi-tenancy
|
||||
"""
|
||||
# Option 1: Extract prefix from email
|
||||
if "@" in email:
|
||||
return email.split("@")[0]
|
||||
|
||||
# Fallback: use email as-is
|
||||
return email
|
||||
|
||||
|
||||
async def process_page_rename(
|
||||
page_id: int,
|
||||
old_path: str,
|
||||
new_path: str,
|
||||
old_title: str,
|
||||
new_title: str,
|
||||
user: str,
|
||||
ingestion_service: IngestionService
|
||||
):
|
||||
"""
|
||||
Process page rename/move events.
|
||||
|
||||
Handles two scenarios:
|
||||
1. Path change only (move to different location) - update path in graph
|
||||
2. Title change (rename) - update title, re-link entities, re-process
|
||||
|
||||
Args:
|
||||
page_id: Wiki page ID
|
||||
old_path: Previous page path
|
||||
new_path: New page path
|
||||
old_title: Previous page title
|
||||
new_title: New page title
|
||||
user: User identifier
|
||||
ingestion_service: Ingestion service instance
|
||||
"""
|
||||
from src.core.dependencies import get_neo4j_client
|
||||
from src.core.multi_tenancy import get_neo4j_user_label
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Processing rename for page {page_id}: "
|
||||
f"'{old_title}' → '{new_title}', "
|
||||
f"'{old_path}' → '{new_path}'"
|
||||
)
|
||||
|
||||
neo4j = get_neo4j_client()
|
||||
user_doc_label = get_neo4j_user_label(user)
|
||||
|
||||
# Check if title changed (rename) or just path changed (move)
|
||||
title_changed = old_title != new_title
|
||||
path_changed = old_path != new_path
|
||||
|
||||
if not title_changed and not path_changed:
|
||||
logger.info("No changes detected, skipping processing")
|
||||
return
|
||||
|
||||
# Update Document node in graph
|
||||
update_query = f"""
|
||||
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})
|
||||
SET d.path = $new_path,
|
||||
d.title = $new_title,
|
||||
d.updated_at = datetime()
|
||||
RETURN d
|
||||
"""
|
||||
|
||||
try:
|
||||
await neo4j.execute_query(update_query, {
|
||||
"page_id": page_id,
|
||||
"new_path": new_path,
|
||||
"new_title": new_title
|
||||
})
|
||||
logger.info(f"Updated Document node for page {page_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update Document node: {e}")
|
||||
|
||||
# If title changed, need to re-process for entity linking
|
||||
if title_changed:
|
||||
logger.info(f"Title changed, re-processing page {page_id}")
|
||||
|
||||
# Re-ingest to update entities (title might be an entity)
|
||||
try:
|
||||
await ingestion_service.ingest_page(
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=True
|
||||
)
|
||||
logger.info(f"Re-ingested page {page_id} after title change")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to re-ingest page {page_id}: {e}")
|
||||
|
||||
# Re-apply bidirectional entity linking
|
||||
from src.config import get_settings
|
||||
from src.core.dependencies import get_ollama_client, get_wikijs_client
|
||||
from src.services.consolidation_service import ConsolidationService
|
||||
|
||||
settings = get_settings()
|
||||
consolidation_service = ConsolidationService(
|
||||
neo4j=neo4j,
|
||||
ollama=get_ollama_client(),
|
||||
wiki=get_wikijs_client(),
|
||||
settings=settings,
|
||||
ingestion_service=ingestion_service
|
||||
)
|
||||
|
||||
try:
|
||||
link_stats = await consolidation_service._apply_bidirectional_entity_linking(
|
||||
page_id=page_id,
|
||||
page_title=new_title,
|
||||
user=user
|
||||
)
|
||||
logger.info(
|
||||
f"Entity linking complete for renamed page {page_id}: "
|
||||
f"{link_stats['forward_links']} forward links, "
|
||||
f"{link_stats['backward_links']} backward links"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to apply entity linking: {e}")
|
||||
|
||||
elif path_changed:
|
||||
logger.info(f"Path changed only (move), no re-processing needed")
|
||||
|
||||
logger.info(f"Rename processing complete for page {page_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to process rename for page {page_id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
async def cleanup_deleted_page(
|
||||
page_id: int,
|
||||
page_title: str,
|
||||
user: str,
|
||||
ingestion_service: IngestionService
|
||||
):
|
||||
"""
|
||||
Clean up vectors and graph when a page is deleted.
|
||||
|
||||
Steps:
|
||||
1. Remove vector embeddings from Qdrant
|
||||
2. Remove Document node from Neo4j
|
||||
3. Clean up orphaned entities (entities only connected to this document)
|
||||
4. Remove broken MENTIONS relationships
|
||||
"""
|
||||
from src.core.dependencies import get_neo4j_client, get_vector_service
|
||||
from src.core.multi_tenancy import get_neo4j_user_base_label, get_neo4j_user_label
|
||||
|
||||
try:
|
||||
logger.info(f"Cleaning up deleted page {page_id} ('{page_title}')")
|
||||
|
||||
vector_service = get_vector_service()
|
||||
neo4j = get_neo4j_client()
|
||||
|
||||
user_base_label = get_neo4j_user_base_label(user)
|
||||
user_doc_label = get_neo4j_user_label(user)
|
||||
|
||||
# STEP 1: Remove vectors from Qdrant
|
||||
logger.info(f"Removing vectors for page {page_id}")
|
||||
try:
|
||||
await vector_service.delete_page_chunks(page_id, user)
|
||||
logger.info(f"Removed vectors for page {page_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to remove vectors for page {page_id}: {e}")
|
||||
|
||||
# STEP 2: Find and store orphaned entities before deletion
|
||||
# (entities that only have this document mentioning them)
|
||||
orphaned_entities_query = f"""
|
||||
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})-[:MENTIONS]->(e:{user_base_label})
|
||||
WHERE NOT e:Document
|
||||
WITH e, count{{(d2:Document)-[:MENTIONS]->(e)}} as mention_count
|
||||
WHERE mention_count = 1
|
||||
RETURN e.id as entity_id, e.name as entity_name, labels(e) as labels
|
||||
"""
|
||||
|
||||
try:
|
||||
orphaned = await neo4j.execute_query(orphaned_entities_query, {"page_id": page_id})
|
||||
logger.info(f"Found {len(orphaned)} orphaned entities for page {page_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to find orphaned entities: {e}")
|
||||
orphaned = []
|
||||
|
||||
# STEP 3: Delete Document node (this will cascade delete MENTIONS relationships)
|
||||
delete_doc_query = f"""
|
||||
MATCH (d:{user_doc_label}:Document {{page_id: $page_id}})
|
||||
DETACH DELETE d
|
||||
RETURN count(d) as deleted_count
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await neo4j.execute_query(delete_doc_query, {"page_id": page_id})
|
||||
deleted_count = result[0]["deleted_count"] if result else 0
|
||||
logger.info(f"Deleted {deleted_count} Document node(s) for page {page_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete Document node: {e}")
|
||||
|
||||
# STEP 4: Delete orphaned entities
|
||||
if orphaned:
|
||||
for entity in orphaned:
|
||||
entity_id = entity["entity_id"]
|
||||
entity_name = entity["entity_name"]
|
||||
|
||||
delete_entity_query = f"""
|
||||
MATCH (e:{user_base_label} {{id: $entity_id}})
|
||||
WHERE NOT e:Document
|
||||
AND NOT EXISTS {{(d:Document)-[:MENTIONS]->(e)}}
|
||||
DETACH DELETE e
|
||||
RETURN count(e) as deleted_count
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await neo4j.execute_query(delete_entity_query, {"entity_id": entity_id})
|
||||
deleted = result[0]["deleted_count"] if result else 0
|
||||
if deleted > 0:
|
||||
logger.info(f"Deleted orphaned entity: {entity_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete orphaned entity {entity_name}: {e}")
|
||||
|
||||
# STEP 5: Clean up broken SearchQuery relationships
|
||||
cleanup_search_query = f"""
|
||||
MATCH (sq:SearchQuery)-[r:FOUND]->(d:Document)
|
||||
WHERE NOT EXISTS {{(d)}}
|
||||
DELETE r
|
||||
RETURN count(r) as cleaned_count
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await neo4j.execute_query(cleanup_search_query, {})
|
||||
cleaned = result[0]["cleaned_count"] if result else 0
|
||||
if cleaned > 0:
|
||||
logger.info(f"Cleaned up {cleaned} broken SearchQuery relationships")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clean SearchQuery relationships: {e}")
|
||||
|
||||
logger.info(f"Cleanup complete for deleted page {page_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup deleted page {page_id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
# Health check endpoint
|
||||
@router.get("/health")
|
||||
async def webhook_health():
|
||||
"""Health check for webhook endpoint."""
|
||||
return {"status": "ok", "service": "webhooks"}
|
||||
@@ -0,0 +1,409 @@
|
||||
"""
|
||||
Wiki router for Library Desk API.
|
||||
|
||||
Endpoints for wiki page and dossier management.
|
||||
All operations are scoped to user namespaces for multi-tenancy.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query, Security, BackgroundTasks
|
||||
from fastapi.security import HTTPAuthorizationCredentials
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
from src.models.wiki import (
|
||||
WikiPage, WikiPageList, WikiPageCreate, WikiPageUpdate, WikiPageMove,
|
||||
WikiOperationResponse, WikiSearchResponse,
|
||||
DossierList, WikiSearchResult
|
||||
)
|
||||
from src.services.wiki_service import WikiService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.services.vector_service import VectorService
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.core.dependencies import WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, verify_api_key
|
||||
from src.core.multi_tenancy import DEFAULT_USER
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/wiki", tags=["Wiki"])
|
||||
|
||||
|
||||
# Dependency to get wiki service
|
||||
def get_wiki_service(wiki_client: WikiJSDep) -> WikiService:
|
||||
"""Get wiki service instance."""
|
||||
return WikiService(wiki_client)
|
||||
|
||||
|
||||
# Dependency to get graph service
|
||||
def get_graph_service(neo4j_client: Neo4jDep, wiki_client: WikiJSDep) -> GraphService:
|
||||
"""Get graph service instance."""
|
||||
return GraphService(neo4j_client, wiki_client)
|
||||
|
||||
|
||||
# Dependency to get vector service
|
||||
def get_vector_service(
|
||||
qdrant_client: QdrantDep,
|
||||
wiki_client: WikiJSDep,
|
||||
ollama_client: OllamaDep
|
||||
) -> VectorService:
|
||||
"""Get vector service instance."""
|
||||
return VectorService(qdrant_client, wiki_client, ollama_client)
|
||||
|
||||
|
||||
# Page operations
|
||||
@router.get("/pages", response_model=WikiPageList)
|
||||
async def list_pages(
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
tag: Optional[str] = Query(default=None, description="Filter by tag (dossier)"),
|
||||
limit: int = Query(default=50, ge=1, le=200, description="Maximum pages to return"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
List wiki pages for a user.
|
||||
|
||||
Optionally filter by tag (dossier). Pages are scoped to user's namespace.
|
||||
|
||||
**Example:** `/wiki/pages?user=jpmschweitzer&tag=projects&limit=20`
|
||||
"""
|
||||
try:
|
||||
return await wiki_service.list_pages(user=user, tag=tag, limit=limit)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list pages: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.get("/pages/{page_id}", response_model=WikiPage)
|
||||
async def get_page(
|
||||
page_id: int,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Get a single wiki page by ID.
|
||||
|
||||
Access is restricted to pages within the user's namespace.
|
||||
|
||||
**Example:** `/wiki/pages/123?user=jpmschweitzer`
|
||||
"""
|
||||
try:
|
||||
page = await wiki_service.get_page(page_id=page_id, user=user)
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail=f"Page {page_id} not found or access denied")
|
||||
return page
|
||||
except HTTPException:
|
||||
raise
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.post("/pages", response_model=WikiPage, status_code=201)
|
||||
async def create_page(
|
||||
page_data: WikiPageCreate,
|
||||
background_tasks: BackgroundTasks,
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Create a new wiki page.
|
||||
|
||||
The page will be created in the user's namespace. If path doesn't start
|
||||
with namespace, it will be automatically prefixed.
|
||||
|
||||
**Auto-updates knowledge graph and vector embeddings**: After creating
|
||||
the page, the graph and vectors are automatically updated in the
|
||||
background to extract entities/relationships and generate semantic embeddings.
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"title": "Library Desk Architecture",
|
||||
"path": "/projects/library-desk/architecture",
|
||||
"content": "# Architecture\\n\\nThis describes...",
|
||||
"description": "Architecture documentation",
|
||||
"tags": ["projects", "architecture"],
|
||||
"user": "jpmschweitzer"
|
||||
}
|
||||
```
|
||||
"""
|
||||
try:
|
||||
page = await wiki_service.create_page(page_data)
|
||||
|
||||
user = page_data.user or DEFAULT_USER
|
||||
|
||||
# Schedule BOTH graph and vector updates in background (non-blocking)
|
||||
background_tasks.add_task(
|
||||
graph_service.update_from_page,
|
||||
page_id=page.id,
|
||||
user=user
|
||||
)
|
||||
background_tasks.add_task(
|
||||
vector_service.update_from_page,
|
||||
page_id=page.id,
|
||||
user=user
|
||||
)
|
||||
|
||||
logger.info(f"Page {page.id} created, graph and vector updates scheduled")
|
||||
return page
|
||||
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create page: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.put("/pages/{page_id}", response_model=WikiPage)
|
||||
async def update_page(
|
||||
page_id: int,
|
||||
page_data: WikiPageUpdate,
|
||||
background_tasks: BackgroundTasks,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Update an existing wiki page.
|
||||
|
||||
Only pages within the user's namespace can be updated.
|
||||
Partial updates are supported - only provided fields will be updated.
|
||||
|
||||
**Auto-updates knowledge graph and vector embeddings**: After updating
|
||||
the page, the graph and vectors are automatically refreshed in the
|
||||
background to reflect the changes.
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"title": "Updated Title",
|
||||
"tags": ["projects", "updated"]
|
||||
}
|
||||
```
|
||||
"""
|
||||
try:
|
||||
page = await wiki_service.update_page(
|
||||
page_id=page_id,
|
||||
page_data=page_data,
|
||||
user=user
|
||||
)
|
||||
|
||||
# Schedule BOTH graph and vector updates in background (non-blocking)
|
||||
background_tasks.add_task(
|
||||
graph_service.update_from_page,
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=True # Force refresh on updates
|
||||
)
|
||||
background_tasks.add_task(
|
||||
vector_service.update_from_page,
|
||||
page_id=page_id,
|
||||
user=user,
|
||||
force_refresh=True # Force refresh on updates
|
||||
)
|
||||
|
||||
logger.info(f"Page {page_id} updated, graph and vector refresh scheduled")
|
||||
return page
|
||||
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.delete("/pages/{page_id}", response_model=WikiOperationResponse)
|
||||
async def delete_page(
|
||||
page_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
vector_service: VectorService = Depends(get_vector_service),
|
||||
graph_service: GraphService = Depends(get_graph_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Delete a wiki page.
|
||||
|
||||
Only pages within the user's namespace can be deleted.
|
||||
This operation cannot be undone.
|
||||
|
||||
**Auto-cleanup**: Vector chunks and graph nodes for this page are
|
||||
automatically deleted in the background.
|
||||
"""
|
||||
try:
|
||||
success = await wiki_service.delete_page(page_id=page_id, user=user)
|
||||
|
||||
# Schedule vector cleanup in background
|
||||
background_tasks.add_task(
|
||||
vector_service.delete_page_chunks,
|
||||
page_id=page_id,
|
||||
user=user
|
||||
)
|
||||
|
||||
# Schedule graph cleanup in background
|
||||
background_tasks.add_task(
|
||||
graph_service.delete_page,
|
||||
page_id=page_id,
|
||||
user=user
|
||||
)
|
||||
|
||||
logger.info(f"Page {page_id} deleted, vector and graph cleanup scheduled")
|
||||
return WikiOperationResponse(
|
||||
success=success,
|
||||
message=f"Page {page_id} deleted successfully",
|
||||
page_id=page_id
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.post("/pages/{page_id}/move", response_model=WikiOperationResponse)
|
||||
async def move_page(
|
||||
page_id: int,
|
||||
move_data: WikiPageMove,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Move or rename a wiki page.
|
||||
|
||||
The new path must be within the user's namespace.
|
||||
|
||||
**Example Request:**
|
||||
```json
|
||||
{
|
||||
"new_path": "/projects/library-desk/docs/architecture"
|
||||
}
|
||||
```
|
||||
"""
|
||||
try:
|
||||
success = await wiki_service.move_page(
|
||||
page_id=page_id,
|
||||
new_path=move_data.new_path,
|
||||
user=user
|
||||
)
|
||||
if not success:
|
||||
raise HTTPException(status_code=500, detail="Failed to move page")
|
||||
|
||||
return WikiOperationResponse(
|
||||
success=True,
|
||||
message=f"Page {page_id} moved successfully",
|
||||
page_id=page_id,
|
||||
page_path=move_data.new_path
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to move page {page_id}: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
# Search operations
|
||||
@router.get("/search", response_model=WikiSearchResponse)
|
||||
async def search_pages(
|
||||
q: str = Query(..., min_length=1, description="Search query"),
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
limit: int = Query(default=20, ge=1, le=100, description="Maximum results"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Search wiki pages within user's namespace.
|
||||
|
||||
**Example:** `/wiki/search?q=architecture&user=jpmschweitzer&limit=10`
|
||||
"""
|
||||
try:
|
||||
results = await wiki_service.search_pages(
|
||||
query=q,
|
||||
user=user,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
return WikiSearchResponse(
|
||||
results=[
|
||||
WikiSearchResult(
|
||||
id=r.id,
|
||||
path=r.path,
|
||||
title=r.title,
|
||||
description=r.description,
|
||||
relevance=None
|
||||
)
|
||||
for r in results
|
||||
],
|
||||
query=q,
|
||||
total=len(results)
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
# Dossier operations
|
||||
@router.get("/dossiers", response_model=DossierList)
|
||||
async def list_dossiers(
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
List all dossiers (unique tags) for a user.
|
||||
|
||||
Dossiers are tag-based collections of pages.
|
||||
|
||||
**Example:** `/wiki/dossiers?user=jpmschweitzer`
|
||||
"""
|
||||
try:
|
||||
return await wiki_service.list_dossiers(user=user)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list dossiers: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.get("/dossiers/{dossier_name}/pages", response_model=WikiPageList)
|
||||
async def get_dossier_pages(
|
||||
dossier_name: str,
|
||||
user: str = Query(default=DEFAULT_USER, description="User identifier"),
|
||||
limit: int = Query(default=100, ge=1, le=500, description="Maximum pages"),
|
||||
wiki_service: WikiService = Depends(get_wiki_service),
|
||||
api_key: str = Depends(verify_api_key)
|
||||
):
|
||||
"""
|
||||
Get all pages in a dossier.
|
||||
|
||||
Returns pages tagged with the dossier name.
|
||||
|
||||
**Example:** `/wiki/dossiers/projects/pages?user=jpmschweitzer`
|
||||
"""
|
||||
try:
|
||||
return await wiki_service.get_dossier_pages(
|
||||
dossier_name=dossier_name,
|
||||
user=user,
|
||||
limit=limit
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get dossier pages: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
Reference in New Issue
Block a user