feat: add smart page creation endpoint with HybridRAG research
Add POST /wiki/pages/smart-create endpoint that combines research with content generation for the librarian agent: - Run HybridRAG search on topic (wiki + graph + web) - Use LLM (WikiPageWriter) to synthesize findings into wiki content - Create page with proper attribution and sources - Schedule background tasks for vector/graph indexing - Apply bidirectional entity linking (forward + backward links) New files: - src/services/entity_linking_utils.py - shared entity linking helper Modified: - src/models/wiki.py - WikiSmartCreateRequest/Response models - src/services/wiki_service.py - smart_create_page() method - src/routers/wiki.py - /pages/smart-create endpoint 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Shared entity linking utilities for Library Desk.
|
||||
|
||||
Provides bidirectional entity linking functionality that can be used by:
|
||||
- Consolidation service (knowledge consolidation)
|
||||
- Wiki router (smart page creation)
|
||||
- Any other service that creates wiki pages
|
||||
"""
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from src.core.multi_tenancy import get_neo4j_user_base_label
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def apply_bidirectional_entity_linking(
|
||||
page_id: int,
|
||||
page_title: str,
|
||||
user: str,
|
||||
neo4j_client: "Neo4jClient",
|
||||
wiki_service: "WikiService",
|
||||
ingestion_service: Optional["IngestionService"] = None
|
||||
) -> Dict[str, int]:
|
||||
"""
|
||||
Apply bidirectional entity linking after page creation/update.
|
||||
|
||||
This runs AFTER ingestion so entities are extracted and in the graph.
|
||||
|
||||
Steps:
|
||||
1. Link entities in the new page (forward links to existing entities)
|
||||
2. Find pages that mention the new entity (reverse references)
|
||||
3. Link entities in those pages (backward links to the new entity)
|
||||
|
||||
Args:
|
||||
page_id: Wiki page ID
|
||||
page_title: Page title (used to find reverse references)
|
||||
user: User identifier
|
||||
neo4j_client: Neo4j client for graph queries
|
||||
wiki_service: Wiki service for page operations
|
||||
ingestion_service: Optional ingestion service for re-indexing
|
||||
|
||||
Returns:
|
||||
Dict with link counts: {
|
||||
"forward_links": int, # Links added to the new page
|
||||
"backward_links": int, # Links added to other pages pointing to new page
|
||||
"pages_updated": int # Number of other pages updated
|
||||
}
|
||||
"""
|
||||
from src.routers.entity_linking import (
|
||||
link_entities_in_page,
|
||||
EntityLinkingRequest,
|
||||
get_entities_with_paths,
|
||||
add_entity_links_to_content
|
||||
)
|
||||
from src.core.dependencies import get_graph_service, get_wiki_service, get_ingestion_service
|
||||
from src.models.wiki import WikiPageUpdate
|
||||
|
||||
forward_links = 0
|
||||
backward_links = 0
|
||||
pages_updated = 0
|
||||
|
||||
try:
|
||||
graph_service = get_graph_service()
|
||||
|
||||
# Use provided services or get defaults
|
||||
wiki_svc = wiki_service
|
||||
ingestion_svc = ingestion_service or get_ingestion_service()
|
||||
|
||||
# STEP 1: Forward linking - link entities in the new page
|
||||
logger.info(f"Step 1/3: Linking entities in page {page_id} ('{page_title}')")
|
||||
try:
|
||||
forward_result = await link_entities_in_page(
|
||||
request=EntityLinkingRequest(
|
||||
user=user,
|
||||
page_id=page_id,
|
||||
create_relationships=True,
|
||||
re_index_if_changed=False # Already indexed, no need to re-index
|
||||
),
|
||||
wiki_service=wiki_svc,
|
||||
graph_service=graph_service,
|
||||
ingestion_service=ingestion_svc,
|
||||
api_key="" # Internal call, no auth needed
|
||||
)
|
||||
forward_links = forward_result.content_links_added
|
||||
logger.info(f"Added {forward_links} forward links in page {page_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to add forward links: {e}")
|
||||
|
||||
# STEP 2: Find reverse references - which pages mention this new entity?
|
||||
logger.info(f"Step 2/3: Finding pages that mention '{page_title}'")
|
||||
user_base_label = get_neo4j_user_base_label(user)
|
||||
|
||||
# Query to find documents that mention entities with this page's title
|
||||
reverse_query = f"""
|
||||
// Find entities with the same name as the page title
|
||||
MATCH (e:{user_base_label})
|
||||
WHERE toLower(e.name) = toLower($title)
|
||||
AND NOT e:Document
|
||||
|
||||
// Find documents that mention those entities
|
||||
MATCH (d:Document)-[r:MENTIONS]->(e)
|
||||
WHERE d.page_id <> $page_id // Exclude the page itself
|
||||
|
||||
RETURN DISTINCT d.page_id as page_id, d.title as title
|
||||
LIMIT 50
|
||||
"""
|
||||
|
||||
try:
|
||||
reverse_refs = await neo4j_client.execute_query(
|
||||
reverse_query,
|
||||
{"title": page_title, "page_id": page_id}
|
||||
)
|
||||
logger.info(f"Found {len(reverse_refs)} pages that mention '{page_title}'")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to find reverse references: {e}")
|
||||
reverse_refs = []
|
||||
|
||||
# STEP 3: Backward linking - add links in those pages to the new entity
|
||||
if reverse_refs:
|
||||
logger.info(f"Step 3/3: Adding backward links in {len(reverse_refs)} pages")
|
||||
for ref in reverse_refs:
|
||||
try:
|
||||
backward_result = await link_entities_in_page(
|
||||
request=EntityLinkingRequest(
|
||||
user=user,
|
||||
page_id=ref['page_id'],
|
||||
create_relationships=False, # Relationships already exist
|
||||
re_index_if_changed=False # Don't re-index for link updates
|
||||
),
|
||||
wiki_service=wiki_svc,
|
||||
graph_service=graph_service,
|
||||
ingestion_service=ingestion_svc,
|
||||
api_key=""
|
||||
)
|
||||
if backward_result.content_links_added > 0:
|
||||
backward_links += backward_result.content_links_added
|
||||
pages_updated += 1
|
||||
logger.info(
|
||||
f"Added {backward_result.content_links_added} links "
|
||||
f"in page {ref['page_id']} ('{ref['title']}')"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to add backward links in page {ref['page_id']}: {e}")
|
||||
else:
|
||||
logger.info("Step 3/3: No reverse references found, skipping backward linking")
|
||||
|
||||
return {
|
||||
"forward_links": forward_links,
|
||||
"backward_links": backward_links,
|
||||
"pages_updated": pages_updated
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Bidirectional entity linking failed: {e}", exc_info=True)
|
||||
return {
|
||||
"forward_links": 0,
|
||||
"backward_links": 0,
|
||||
"pages_updated": 0
|
||||
}
|
||||
@@ -431,3 +431,166 @@ class WikiService:
|
||||
WikiPageList filtered by dossier tag
|
||||
"""
|
||||
return await self.list_pages(user, tag=dossier_name, limit=limit)
|
||||
|
||||
async def smart_create_page(
|
||||
self,
|
||||
topic: str,
|
||||
user: str,
|
||||
path: Optional[str],
|
||||
tags: List[str],
|
||||
hybrid_rag_service: "HybridRAGService",
|
||||
wiki_page_writer: "WikiPageWriter",
|
||||
include_web: bool = True,
|
||||
include_wiki: bool = True
|
||||
) -> tuple["WikiPage", Dict[str, Any]]:
|
||||
"""
|
||||
Create wiki page with research from HybridRAG.
|
||||
|
||||
This method combines research + content generation + page creation:
|
||||
1. Run HybridRAG search on topic
|
||||
2. Format results for WikiPageWriter
|
||||
3. Generate page content with LLM
|
||||
4. Create page in Wiki.js
|
||||
5. Return page + research summary
|
||||
|
||||
Args:
|
||||
topic: Topic to research and create page about
|
||||
user: User identifier
|
||||
path: Optional page path (auto-generated from topic if not provided)
|
||||
tags: Tags for the page
|
||||
hybrid_rag_service: HybridRAG service for multi-source search
|
||||
wiki_page_writer: WikiPageWriter for LLM content generation
|
||||
include_web: Include web search results
|
||||
include_wiki: Include existing wiki knowledge
|
||||
|
||||
Returns:
|
||||
Tuple of (created WikiPage, research summary dict)
|
||||
"""
|
||||
from src.models.hybrid_rag import HybridRAGConfig
|
||||
|
||||
logger.info(f"Smart create page: topic='{topic}', user='{user}'")
|
||||
|
||||
# Step 1: Run HybridRAG search on the topic
|
||||
config = HybridRAGConfig(
|
||||
enable_vector=include_wiki,
|
||||
enable_graph=include_wiki,
|
||||
enable_web=include_web,
|
||||
enable_reranking=True,
|
||||
enable_enrichment=True,
|
||||
final_result_count=15 # Get more results for rich content
|
||||
)
|
||||
|
||||
search_response = await hybrid_rag_service.search(
|
||||
query=topic,
|
||||
user=user,
|
||||
config=config
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"HybridRAG search completed: {search_response.total_results} results, "
|
||||
f"search_id={search_response.search_id}"
|
||||
)
|
||||
|
||||
# Step 2: Format results for WikiPageWriter
|
||||
source_information = []
|
||||
wiki_results_count = 0
|
||||
web_results_count = 0
|
||||
graph_entities_count = 0
|
||||
|
||||
for result in search_response.results:
|
||||
source_type = result.source_type
|
||||
|
||||
if "web" in source_type:
|
||||
web_results_count += 1
|
||||
source_information.append({
|
||||
"title": result.title,
|
||||
"url": result.url or "",
|
||||
"content": result.content[:500] if result.content else ""
|
||||
})
|
||||
elif "vector" in source_type or "graph" in source_type:
|
||||
wiki_results_count += 1
|
||||
# For wiki results, use page path as URL
|
||||
source_information.append({
|
||||
"title": result.title,
|
||||
"url": f"/{result.page_path}" if result.page_path else "",
|
||||
"content": result.content[:500] if result.content else ""
|
||||
})
|
||||
|
||||
# Count entities from related dossiers
|
||||
if result.related_dossiers:
|
||||
graph_entities_count += len(result.related_dossiers)
|
||||
|
||||
# Step 3: Generate page content with LLM
|
||||
# Use topic as summary and let WikiPageWriter create structured content
|
||||
topic_summary = f"Research findings about: {topic}"
|
||||
if search_response.keywords:
|
||||
topic_summary += f"\n\nKey concepts: {', '.join(search_response.keywords.core_keywords)}"
|
||||
|
||||
# Extract entities from search results for knowledge graph linking
|
||||
entities = []
|
||||
if search_response.keywords and search_response.keywords.core_keywords:
|
||||
entities = search_response.keywords.core_keywords[:10]
|
||||
|
||||
# Get related documents for cross-linking
|
||||
related_docs = []
|
||||
for result in search_response.results[:5]:
|
||||
if result.page_path:
|
||||
related_docs.append(f"[{result.title}](/{result.page_path})")
|
||||
|
||||
content = await wiki_page_writer.create_page(
|
||||
title=topic,
|
||||
topic_summary=topic_summary,
|
||||
source_information=source_information[:10], # Limit sources
|
||||
entities=entities,
|
||||
related_docs=related_docs
|
||||
)
|
||||
|
||||
logger.info(f"Generated page content: {len(content)} characters")
|
||||
|
||||
# Step 4: Auto-generate path from topic if not provided
|
||||
if not path:
|
||||
# Convert topic to kebab-case path
|
||||
import re
|
||||
path_slug = topic.lower()
|
||||
path_slug = re.sub(r'[^\w\s-]', '', path_slug) # Remove special chars
|
||||
path_slug = re.sub(r'\s+', '-', path_slug) # Spaces to hyphens
|
||||
path_slug = re.sub(r'-+', '-', path_slug) # Multiple hyphens to single
|
||||
path_slug = path_slug.strip('-')
|
||||
|
||||
# Infer category from tags or use reference
|
||||
category = "reference"
|
||||
if tags:
|
||||
category = tags[0].lower()
|
||||
|
||||
path = f"/{category}/{path_slug}"
|
||||
|
||||
# Step 5: Create page using existing create_page method
|
||||
from src.models.wiki import WikiPageCreate
|
||||
|
||||
page_data = WikiPageCreate(
|
||||
title=topic,
|
||||
path=path,
|
||||
content=content,
|
||||
description=f"Research summary about {topic}",
|
||||
tags=tags,
|
||||
user=user
|
||||
)
|
||||
|
||||
page = await self.create_page(page_data)
|
||||
|
||||
logger.info(f"Created page: id={page.id}, path={page.path}")
|
||||
|
||||
# Build research summary
|
||||
research_summary = {
|
||||
"wiki_results": wiki_results_count,
|
||||
"web_results": web_results_count,
|
||||
"graph_entities": graph_entities_count,
|
||||
"keywords_extracted": len(search_response.keywords.core_keywords) if search_response.keywords else 0,
|
||||
"timing_ms": search_response.timing.total_ms if search_response.timing else 0
|
||||
}
|
||||
|
||||
return page, {
|
||||
"research_summary": research_summary,
|
||||
"sources_used": len(source_information),
|
||||
"search_id": search_response.search_id
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user