feat: add smart page creation endpoint with HybridRAG research

Add POST /wiki/pages/smart-create endpoint that combines research with
content generation for the librarian agent:

- Run HybridRAG search on topic (wiki + graph + web)
- Use LLM (WikiPageWriter) to synthesize findings into wiki content
- Create page with proper attribution and sources
- Schedule background tasks for vector/graph indexing
- Apply bidirectional entity linking (forward + backward links)

New files:
- src/services/entity_linking_utils.py - shared entity linking helper

Modified:
- src/models/wiki.py - WikiSmartCreateRequest/Response models
- src/services/wiki_service.py - smart_create_page() method
- src/routers/wiki.py - /pages/smart-create endpoint

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/models/wiki.py

@@ -8,7 +8,7 @@ Models for:
 """
 
 from pydantic import BaseModel, Field, field_validator
-from typing import Optional, List
+from typing import Optional, List, Dict, Any
 from datetime import datetime
 
 
@@ -190,3 +190,44 @@ class DossierOperationResponse(BaseModel):
     dossier_name: str = Field(..., description="Dossier name")
     index_page_id: Optional[int] = Field(None, description="Index page ID (if created)")
     index_page_path: Optional[str] = Field(None, description="Index page path (if created)")
+
+
+# Smart create models (HybridRAG-powered page creation)
+class WikiSmartCreateRequest(BaseModel):
+    """Request model for smart page creation with research."""
+    topic: str = Field(..., min_length=1, max_length=500, description="Topic to research and create page about")
+    path: Optional[str] = Field(None, description="Page path (auto-generated from topic if not provided)")
+    tags: List[str] = Field(default_factory=list, description="Tags for the page")
+    user: Optional[str] = Field(None, description="User identifier")
+    include_web_research: bool = Field(default=True, description="Include web search results")
+    include_wiki_search: bool = Field(default=True, description="Include existing wiki knowledge")
+
+    @field_validator("tags")
+    @classmethod
+    def validate_tags(cls, v: List[str]) -> List[str]:
+        """Validate and clean tags."""
+        cleaned = [tag.strip() for tag in v if tag.strip()]
+        return list(set(cleaned))
+
+    @field_validator("path")
+    @classmethod
+    def validate_path(cls, v: Optional[str]) -> Optional[str]:
+        """Validate page path if provided."""
+        if v is None:
+            return None
+        # Ensure path starts with /
+        if not v.startswith("/"):
+            v = f"/{v}"
+        # Remove trailing slash
+        if v.endswith("/") and v != "/":
+            v = v.rstrip("/")
+        return v
+
+
+class WikiSmartCreateResponse(BaseModel):
+    """Response model for smart page creation."""
+    page: WikiPage = Field(..., description="Created wiki page")
+    research_summary: Dict[str, Any] = Field(..., description="Summary of research used")
+    sources_used: int = Field(..., description="Number of sources incorporated")
+    search_id: Optional[str] = Field(None, description="HybridRAG search ID for reference")
+    entity_linking: Dict[str, int] = Field(default_factory=dict, description="Entity linking statistics")

src/routers/wiki.py

@@ -13,7 +13,8 @@ import logging
 from src.models.wiki import (
     WikiPage, WikiPageList, WikiPageCreate, WikiPageUpdate, WikiPageMove,
     WikiOperationResponse, WikiSearchResponse,
-    DossierList, WikiSearchResult
+    DossierList, WikiSearchResult,
+    WikiSmartCreateRequest, WikiSmartCreateResponse
 )
 from src.services.wiki_service import WikiService
 from src.services.graph_service import GraphService
@@ -22,8 +23,15 @@ from src.clients.wikijs_client import WikiJSClient
 from src.clients.neo4j_client import Neo4jClient
 from src.clients.qdrant_client import QdrantClientWrapper
 from src.clients.ollama_client import OllamaClient
-from src.core.dependencies import WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, verify_api_key
+from src.core.dependencies import (
+    WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, SearXNGDep,
+    verify_api_key, get_settings, get_hybrid_rag_service, get_ingestion_service
+)
 from src.core.multi_tenancy import DEFAULT_USER
+from src.services.hybrid_rag_service import HybridRAGService
+from src.services.wiki_page_writer import WikiPageWriter
+from src.services.entity_linking_utils import apply_bidirectional_entity_linking
+from src.config import Settings
 
 logger = logging.getLogger(__name__)
 
@@ -163,6 +171,123 @@ async def create_page(
         raise HTTPException(status_code=500, detail="Internal server error")
 
 
+@router.post("/pages/smart-create", response_model=WikiSmartCreateResponse, status_code=201)
+async def smart_create_page(
+    request: WikiSmartCreateRequest,
+    background_tasks: BackgroundTasks,
+    wiki_client: WikiJSDep,
+    neo4j_client: Neo4jDep,
+    qdrant_client: QdrantDep,
+    ollama_client: OllamaDep,
+    searxng_client: SearXNGDep,
+    settings: Settings = Depends(get_settings),
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Create wiki page with intelligent research.
+
+    Combines HybridRAG search with LLM content generation to create
+    rich, well-researched wiki pages in a single API call.
+
+    **Process:**
+    1. Runs HybridRAG search on the topic (wiki + graph + web)
+    2. Uses LLM to synthesize findings into structured wiki content
+    3. Creates the page with proper attribution/sources
+    4. Indexes into vectors + knowledge graph (background)
+    5. Applies bidirectional entity linking (background)
+
+    **Example Request:**
+    ```json
+    {
+        "topic": "Docker orchestration patterns",
+        "path": "/technology/containers/docker-orchestration",
+        "tags": ["technology", "devops", "containers"],
+        "user": "jpmschweitzer",
+        "include_web_research": true,
+        "include_wiki_search": true
+    }
+    ```
+
+    **Returns:**
+    - Created page with ID, path, content
+    - Research summary (wiki/web/graph result counts)
+    - Entity linking statistics (forward/backward links)
+    """
+    try:
+        user = request.user or DEFAULT_USER
+
+        # Build services
+        wiki_service = WikiService(wiki_client)
+        vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
+        graph_service = GraphService(neo4j_client, wiki_client)
+        hybrid_rag_service = HybridRAGService(
+            vector_service=vector_service,
+            graph_service=graph_service,
+            searxng_client=searxng_client,
+            ollama_client=ollama_client,
+            settings=settings
+        )
+        wiki_page_writer = WikiPageWriter(ollama_client=ollama_client)
+
+        # Step 1-5: Research + Generate + Create page
+        page, research_data = await wiki_service.smart_create_page(
+            topic=request.topic,
+            user=user,
+            path=request.path,
+            tags=request.tags,
+            hybrid_rag_service=hybrid_rag_service,
+            wiki_page_writer=wiki_page_writer,
+            include_web=request.include_web_research,
+            include_wiki=request.include_wiki_search
+        )
+
+        # Schedule graph and vector updates in background
+        background_tasks.add_task(
+            graph_service.update_from_page,
+            page_id=page.id,
+            user=user
+        )
+        background_tasks.add_task(
+            vector_service.update_from_page,
+            page_id=page.id,
+            user=user
+        )
+
+        # Schedule bidirectional entity linking in background
+        async def run_entity_linking():
+            ingestion_service = get_ingestion_service()
+            return await apply_bidirectional_entity_linking(
+                page_id=page.id,
+                page_title=page.title,
+                user=user,
+                neo4j_client=neo4j_client,
+                wiki_service=wiki_service,
+                ingestion_service=ingestion_service
+            )
+
+        background_tasks.add_task(run_entity_linking)
+
+        logger.info(
+            f"Smart page created: id={page.id}, path={page.path}, "
+            f"sources={research_data['sources_used']}"
+        )
+
+        return WikiSmartCreateResponse(
+            page=page,
+            research_summary=research_data["research_summary"],
+            sources_used=research_data["sources_used"],
+            search_id=research_data["search_id"],
+            entity_linking={"forward_links": 0, "backward_links": 0, "pages_updated": 0}
+            # Note: entity_linking stats are 0 here as it runs in background
+        )
+
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        logger.error(f"Failed to smart create page: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+
 @router.put("/pages/{page_id}", response_model=WikiPage)
 async def update_page(
     page_id: int,

src/services/entity_linking_utils.py

@@ -0,0 +1,160 @@
+"""
+Shared entity linking utilities for Library Desk.
+
+Provides bidirectional entity linking functionality that can be used by:
+- Consolidation service (knowledge consolidation)
+- Wiki router (smart page creation)
+- Any other service that creates wiki pages
+"""
+import logging
+from typing import Dict, Any, Optional
+
+from src.core.multi_tenancy import get_neo4j_user_base_label
+
+logger = logging.getLogger(__name__)
+
+
+async def apply_bidirectional_entity_linking(
+    page_id: int,
+    page_title: str,
+    user: str,
+    neo4j_client: "Neo4jClient",
+    wiki_service: "WikiService",
+    ingestion_service: Optional["IngestionService"] = None
+) -> Dict[str, int]:
+    """
+    Apply bidirectional entity linking after page creation/update.
+
+    This runs AFTER ingestion so entities are extracted and in the graph.
+
+    Steps:
+    1. Link entities in the new page (forward links to existing entities)
+    2. Find pages that mention the new entity (reverse references)
+    3. Link entities in those pages (backward links to the new entity)
+
+    Args:
+        page_id: Wiki page ID
+        page_title: Page title (used to find reverse references)
+        user: User identifier
+        neo4j_client: Neo4j client for graph queries
+        wiki_service: Wiki service for page operations
+        ingestion_service: Optional ingestion service for re-indexing
+
+    Returns:
+        Dict with link counts: {
+            "forward_links": int,  # Links added to the new page
+            "backward_links": int,  # Links added to other pages pointing to new page
+            "pages_updated": int   # Number of other pages updated
+        }
+    """
+    from src.routers.entity_linking import (
+        link_entities_in_page,
+        EntityLinkingRequest,
+        get_entities_with_paths,
+        add_entity_links_to_content
+    )
+    from src.core.dependencies import get_graph_service, get_wiki_service, get_ingestion_service
+    from src.models.wiki import WikiPageUpdate
+
+    forward_links = 0
+    backward_links = 0
+    pages_updated = 0
+
+    try:
+        graph_service = get_graph_service()
+
+        # Use provided services or get defaults
+        wiki_svc = wiki_service
+        ingestion_svc = ingestion_service or get_ingestion_service()
+
+        # STEP 1: Forward linking - link entities in the new page
+        logger.info(f"Step 1/3: Linking entities in page {page_id} ('{page_title}')")
+        try:
+            forward_result = await link_entities_in_page(
+                request=EntityLinkingRequest(
+                    user=user,
+                    page_id=page_id,
+                    create_relationships=True,
+                    re_index_if_changed=False  # Already indexed, no need to re-index
+                ),
+                wiki_service=wiki_svc,
+                graph_service=graph_service,
+                ingestion_service=ingestion_svc,
+                api_key=""  # Internal call, no auth needed
+            )
+            forward_links = forward_result.content_links_added
+            logger.info(f"Added {forward_links} forward links in page {page_id}")
+        except Exception as e:
+            logger.error(f"Failed to add forward links: {e}")
+
+        # STEP 2: Find reverse references - which pages mention this new entity?
+        logger.info(f"Step 2/3: Finding pages that mention '{page_title}'")
+        user_base_label = get_neo4j_user_base_label(user)
+
+        # Query to find documents that mention entities with this page's title
+        reverse_query = f"""
+        // Find entities with the same name as the page title
+        MATCH (e:{user_base_label})
+        WHERE toLower(e.name) = toLower($title)
+        AND NOT e:Document
+
+        // Find documents that mention those entities
+        MATCH (d:Document)-[r:MENTIONS]->(e)
+        WHERE d.page_id <> $page_id  // Exclude the page itself
+
+        RETURN DISTINCT d.page_id as page_id, d.title as title
+        LIMIT 50
+        """
+
+        try:
+            reverse_refs = await neo4j_client.execute_query(
+                reverse_query,
+                {"title": page_title, "page_id": page_id}
+            )
+            logger.info(f"Found {len(reverse_refs)} pages that mention '{page_title}'")
+        except Exception as e:
+            logger.error(f"Failed to find reverse references: {e}")
+            reverse_refs = []
+
+        # STEP 3: Backward linking - add links in those pages to the new entity
+        if reverse_refs:
+            logger.info(f"Step 3/3: Adding backward links in {len(reverse_refs)} pages")
+            for ref in reverse_refs:
+                try:
+                    backward_result = await link_entities_in_page(
+                        request=EntityLinkingRequest(
+                            user=user,
+                            page_id=ref['page_id'],
+                            create_relationships=False,  # Relationships already exist
+                            re_index_if_changed=False  # Don't re-index for link updates
+                        ),
+                        wiki_service=wiki_svc,
+                        graph_service=graph_service,
+                        ingestion_service=ingestion_svc,
+                        api_key=""
+                    )
+                    if backward_result.content_links_added > 0:
+                        backward_links += backward_result.content_links_added
+                        pages_updated += 1
+                        logger.info(
+                            f"Added {backward_result.content_links_added} links "
+                            f"in page {ref['page_id']} ('{ref['title']}')"
+                        )
+                except Exception as e:
+                    logger.error(f"Failed to add backward links in page {ref['page_id']}: {e}")
+        else:
+            logger.info("Step 3/3: No reverse references found, skipping backward linking")
+
+        return {
+            "forward_links": forward_links,
+            "backward_links": backward_links,
+            "pages_updated": pages_updated
+        }
+
+    except Exception as e:
+        logger.error(f"Bidirectional entity linking failed: {e}", exc_info=True)
+        return {
+            "forward_links": 0,
+            "backward_links": 0,
+            "pages_updated": 0
+        }

src/services/wiki_service.py

@@ -431,3 +431,166 @@ class WikiService:
             WikiPageList filtered by dossier tag
         """
         return await self.list_pages(user, tag=dossier_name, limit=limit)
+
+    async def smart_create_page(
+        self,
+        topic: str,
+        user: str,
+        path: Optional[str],
+        tags: List[str],
+        hybrid_rag_service: "HybridRAGService",
+        wiki_page_writer: "WikiPageWriter",
+        include_web: bool = True,
+        include_wiki: bool = True
+    ) -> tuple["WikiPage", Dict[str, Any]]:
+        """
+        Create wiki page with research from HybridRAG.
+
+        This method combines research + content generation + page creation:
+        1. Run HybridRAG search on topic
+        2. Format results for WikiPageWriter
+        3. Generate page content with LLM
+        4. Create page in Wiki.js
+        5. Return page + research summary
+
+        Args:
+            topic: Topic to research and create page about
+            user: User identifier
+            path: Optional page path (auto-generated from topic if not provided)
+            tags: Tags for the page
+            hybrid_rag_service: HybridRAG service for multi-source search
+            wiki_page_writer: WikiPageWriter for LLM content generation
+            include_web: Include web search results
+            include_wiki: Include existing wiki knowledge
+
+        Returns:
+            Tuple of (created WikiPage, research summary dict)
+        """
+        from src.models.hybrid_rag import HybridRAGConfig
+
+        logger.info(f"Smart create page: topic='{topic}', user='{user}'")
+
+        # Step 1: Run HybridRAG search on the topic
+        config = HybridRAGConfig(
+            enable_vector=include_wiki,
+            enable_graph=include_wiki,
+            enable_web=include_web,
+            enable_reranking=True,
+            enable_enrichment=True,
+            final_result_count=15  # Get more results for rich content
+        )
+
+        search_response = await hybrid_rag_service.search(
+            query=topic,
+            user=user,
+            config=config
+        )
+
+        logger.info(
+            f"HybridRAG search completed: {search_response.total_results} results, "
+            f"search_id={search_response.search_id}"
+        )
+
+        # Step 2: Format results for WikiPageWriter
+        source_information = []
+        wiki_results_count = 0
+        web_results_count = 0
+        graph_entities_count = 0
+
+        for result in search_response.results:
+            source_type = result.source_type
+
+            if "web" in source_type:
+                web_results_count += 1
+                source_information.append({
+                    "title": result.title,
+                    "url": result.url or "",
+                    "content": result.content[:500] if result.content else ""
+                })
+            elif "vector" in source_type or "graph" in source_type:
+                wiki_results_count += 1
+                # For wiki results, use page path as URL
+                source_information.append({
+                    "title": result.title,
+                    "url": f"/{result.page_path}" if result.page_path else "",
+                    "content": result.content[:500] if result.content else ""
+                })
+
+            # Count entities from related dossiers
+            if result.related_dossiers:
+                graph_entities_count += len(result.related_dossiers)
+
+        # Step 3: Generate page content with LLM
+        # Use topic as summary and let WikiPageWriter create structured content
+        topic_summary = f"Research findings about: {topic}"
+        if search_response.keywords:
+            topic_summary += f"\n\nKey concepts: {', '.join(search_response.keywords.core_keywords)}"
+
+        # Extract entities from search results for knowledge graph linking
+        entities = []
+        if search_response.keywords and search_response.keywords.core_keywords:
+            entities = search_response.keywords.core_keywords[:10]
+
+        # Get related documents for cross-linking
+        related_docs = []
+        for result in search_response.results[:5]:
+            if result.page_path:
+                related_docs.append(f"[{result.title}](/{result.page_path})")
+
+        content = await wiki_page_writer.create_page(
+            title=topic,
+            topic_summary=topic_summary,
+            source_information=source_information[:10],  # Limit sources
+            entities=entities,
+            related_docs=related_docs
+        )
+
+        logger.info(f"Generated page content: {len(content)} characters")
+
+        # Step 4: Auto-generate path from topic if not provided
+        if not path:
+            # Convert topic to kebab-case path
+            import re
+            path_slug = topic.lower()
+            path_slug = re.sub(r'[^\w\s-]', '', path_slug)  # Remove special chars
+            path_slug = re.sub(r'\s+', '-', path_slug)  # Spaces to hyphens
+            path_slug = re.sub(r'-+', '-', path_slug)  # Multiple hyphens to single
+            path_slug = path_slug.strip('-')
+
+            # Infer category from tags or use reference
+            category = "reference"
+            if tags:
+                category = tags[0].lower()
+
+            path = f"/{category}/{path_slug}"
+
+        # Step 5: Create page using existing create_page method
+        from src.models.wiki import WikiPageCreate
+
+        page_data = WikiPageCreate(
+            title=topic,
+            path=path,
+            content=content,
+            description=f"Research summary about {topic}",
+            tags=tags,
+            user=user
+        )
+
+        page = await self.create_page(page_data)
+
+        logger.info(f"Created page: id={page.id}, path={page.path}")
+
+        # Build research summary
+        research_summary = {
+            "wiki_results": wiki_results_count,
+            "web_results": web_results_count,
+            "graph_entities": graph_entities_count,
+            "keywords_extracted": len(search_response.keywords.core_keywords) if search_response.keywords else 0,
+            "timing_ms": search_response.timing.total_ms if search_response.timing else 0
+        }
+
+        return page, {
+            "research_summary": research_summary,
+            "sources_used": len(source_information),
+            "search_id": search_response.search_id
+        }