diff --git a/src/models/wiki.py b/src/models/wiki.py index dca7127..3b71bda 100644 --- a/src/models/wiki.py +++ b/src/models/wiki.py @@ -8,7 +8,7 @@ Models for: """ from pydantic import BaseModel, Field, field_validator -from typing import Optional, List +from typing import Optional, List, Dict, Any from datetime import datetime @@ -190,3 +190,44 @@ class DossierOperationResponse(BaseModel): dossier_name: str = Field(..., description="Dossier name") index_page_id: Optional[int] = Field(None, description="Index page ID (if created)") index_page_path: Optional[str] = Field(None, description="Index page path (if created)") + + +# Smart create models (HybridRAG-powered page creation) +class WikiSmartCreateRequest(BaseModel): + """Request model for smart page creation with research.""" + topic: str = Field(..., min_length=1, max_length=500, description="Topic to research and create page about") + path: Optional[str] = Field(None, description="Page path (auto-generated from topic if not provided)") + tags: List[str] = Field(default_factory=list, description="Tags for the page") + user: Optional[str] = Field(None, description="User identifier") + include_web_research: bool = Field(default=True, description="Include web search results") + include_wiki_search: bool = Field(default=True, description="Include existing wiki knowledge") + + @field_validator("tags") + @classmethod + def validate_tags(cls, v: List[str]) -> List[str]: + """Validate and clean tags.""" + cleaned = [tag.strip() for tag in v if tag.strip()] + return list(set(cleaned)) + + @field_validator("path") + @classmethod + def validate_path(cls, v: Optional[str]) -> Optional[str]: + """Validate page path if provided.""" + if v is None: + return None + # Ensure path starts with / + if not v.startswith("/"): + v = f"/{v}" + # Remove trailing slash + if v.endswith("/") and v != "/": + v = v.rstrip("/") + return v + + +class WikiSmartCreateResponse(BaseModel): + """Response model for smart page creation.""" + page: WikiPage = Field(..., description="Created wiki page") + research_summary: Dict[str, Any] = Field(..., description="Summary of research used") + sources_used: int = Field(..., description="Number of sources incorporated") + search_id: Optional[str] = Field(None, description="HybridRAG search ID for reference") + entity_linking: Dict[str, int] = Field(default_factory=dict, description="Entity linking statistics") diff --git a/src/routers/wiki.py b/src/routers/wiki.py index c164e12..b225c67 100644 --- a/src/routers/wiki.py +++ b/src/routers/wiki.py @@ -13,7 +13,8 @@ import logging from src.models.wiki import ( WikiPage, WikiPageList, WikiPageCreate, WikiPageUpdate, WikiPageMove, WikiOperationResponse, WikiSearchResponse, - DossierList, WikiSearchResult + DossierList, WikiSearchResult, + WikiSmartCreateRequest, WikiSmartCreateResponse ) from src.services.wiki_service import WikiService from src.services.graph_service import GraphService @@ -22,8 +23,15 @@ from src.clients.wikijs_client import WikiJSClient from src.clients.neo4j_client import Neo4jClient from src.clients.qdrant_client import QdrantClientWrapper from src.clients.ollama_client import OllamaClient -from src.core.dependencies import WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, verify_api_key +from src.core.dependencies import ( + WikiJSDep, Neo4jDep, QdrantDep, OllamaDep, SearXNGDep, + verify_api_key, get_settings, get_hybrid_rag_service, get_ingestion_service +) from src.core.multi_tenancy import DEFAULT_USER +from src.services.hybrid_rag_service import HybridRAGService +from src.services.wiki_page_writer import WikiPageWriter +from src.services.entity_linking_utils import apply_bidirectional_entity_linking +from src.config import Settings logger = logging.getLogger(__name__) @@ -163,6 +171,123 @@ async def create_page( raise HTTPException(status_code=500, detail="Internal server error") +@router.post("/pages/smart-create", response_model=WikiSmartCreateResponse, status_code=201) +async def smart_create_page( + request: WikiSmartCreateRequest, + background_tasks: BackgroundTasks, + wiki_client: WikiJSDep, + neo4j_client: Neo4jDep, + qdrant_client: QdrantDep, + ollama_client: OllamaDep, + searxng_client: SearXNGDep, + settings: Settings = Depends(get_settings), + api_key: str = Depends(verify_api_key) +): + """ + Create wiki page with intelligent research. + + Combines HybridRAG search with LLM content generation to create + rich, well-researched wiki pages in a single API call. + + **Process:** + 1. Runs HybridRAG search on the topic (wiki + graph + web) + 2. Uses LLM to synthesize findings into structured wiki content + 3. Creates the page with proper attribution/sources + 4. Indexes into vectors + knowledge graph (background) + 5. Applies bidirectional entity linking (background) + + **Example Request:** + ```json + { + "topic": "Docker orchestration patterns", + "path": "/technology/containers/docker-orchestration", + "tags": ["technology", "devops", "containers"], + "user": "jpmschweitzer", + "include_web_research": true, + "include_wiki_search": true + } + ``` + + **Returns:** + - Created page with ID, path, content + - Research summary (wiki/web/graph result counts) + - Entity linking statistics (forward/backward links) + """ + try: + user = request.user or DEFAULT_USER + + # Build services + wiki_service = WikiService(wiki_client) + vector_service = VectorService(qdrant_client, wiki_client, ollama_client) + graph_service = GraphService(neo4j_client, wiki_client) + hybrid_rag_service = HybridRAGService( + vector_service=vector_service, + graph_service=graph_service, + searxng_client=searxng_client, + ollama_client=ollama_client, + settings=settings + ) + wiki_page_writer = WikiPageWriter(ollama_client=ollama_client) + + # Step 1-5: Research + Generate + Create page + page, research_data = await wiki_service.smart_create_page( + topic=request.topic, + user=user, + path=request.path, + tags=request.tags, + hybrid_rag_service=hybrid_rag_service, + wiki_page_writer=wiki_page_writer, + include_web=request.include_web_research, + include_wiki=request.include_wiki_search + ) + + # Schedule graph and vector updates in background + background_tasks.add_task( + graph_service.update_from_page, + page_id=page.id, + user=user + ) + background_tasks.add_task( + vector_service.update_from_page, + page_id=page.id, + user=user + ) + + # Schedule bidirectional entity linking in background + async def run_entity_linking(): + ingestion_service = get_ingestion_service() + return await apply_bidirectional_entity_linking( + page_id=page.id, + page_title=page.title, + user=user, + neo4j_client=neo4j_client, + wiki_service=wiki_service, + ingestion_service=ingestion_service + ) + + background_tasks.add_task(run_entity_linking) + + logger.info( + f"Smart page created: id={page.id}, path={page.path}, " + f"sources={research_data['sources_used']}" + ) + + return WikiSmartCreateResponse( + page=page, + research_summary=research_data["research_summary"], + sources_used=research_data["sources_used"], + search_id=research_data["search_id"], + entity_linking={"forward_links": 0, "backward_links": 0, "pages_updated": 0} + # Note: entity_linking stats are 0 here as it runs in background + ) + + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + logger.error(f"Failed to smart create page: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Internal server error") + + @router.put("/pages/{page_id}", response_model=WikiPage) async def update_page( page_id: int, diff --git a/src/services/entity_linking_utils.py b/src/services/entity_linking_utils.py new file mode 100644 index 0000000..b29c149 --- /dev/null +++ b/src/services/entity_linking_utils.py @@ -0,0 +1,160 @@ +""" +Shared entity linking utilities for Library Desk. + +Provides bidirectional entity linking functionality that can be used by: +- Consolidation service (knowledge consolidation) +- Wiki router (smart page creation) +- Any other service that creates wiki pages +""" +import logging +from typing import Dict, Any, Optional + +from src.core.multi_tenancy import get_neo4j_user_base_label + +logger = logging.getLogger(__name__) + + +async def apply_bidirectional_entity_linking( + page_id: int, + page_title: str, + user: str, + neo4j_client: "Neo4jClient", + wiki_service: "WikiService", + ingestion_service: Optional["IngestionService"] = None +) -> Dict[str, int]: + """ + Apply bidirectional entity linking after page creation/update. + + This runs AFTER ingestion so entities are extracted and in the graph. + + Steps: + 1. Link entities in the new page (forward links to existing entities) + 2. Find pages that mention the new entity (reverse references) + 3. Link entities in those pages (backward links to the new entity) + + Args: + page_id: Wiki page ID + page_title: Page title (used to find reverse references) + user: User identifier + neo4j_client: Neo4j client for graph queries + wiki_service: Wiki service for page operations + ingestion_service: Optional ingestion service for re-indexing + + Returns: + Dict with link counts: { + "forward_links": int, # Links added to the new page + "backward_links": int, # Links added to other pages pointing to new page + "pages_updated": int # Number of other pages updated + } + """ + from src.routers.entity_linking import ( + link_entities_in_page, + EntityLinkingRequest, + get_entities_with_paths, + add_entity_links_to_content + ) + from src.core.dependencies import get_graph_service, get_wiki_service, get_ingestion_service + from src.models.wiki import WikiPageUpdate + + forward_links = 0 + backward_links = 0 + pages_updated = 0 + + try: + graph_service = get_graph_service() + + # Use provided services or get defaults + wiki_svc = wiki_service + ingestion_svc = ingestion_service or get_ingestion_service() + + # STEP 1: Forward linking - link entities in the new page + logger.info(f"Step 1/3: Linking entities in page {page_id} ('{page_title}')") + try: + forward_result = await link_entities_in_page( + request=EntityLinkingRequest( + user=user, + page_id=page_id, + create_relationships=True, + re_index_if_changed=False # Already indexed, no need to re-index + ), + wiki_service=wiki_svc, + graph_service=graph_service, + ingestion_service=ingestion_svc, + api_key="" # Internal call, no auth needed + ) + forward_links = forward_result.content_links_added + logger.info(f"Added {forward_links} forward links in page {page_id}") + except Exception as e: + logger.error(f"Failed to add forward links: {e}") + + # STEP 2: Find reverse references - which pages mention this new entity? + logger.info(f"Step 2/3: Finding pages that mention '{page_title}'") + user_base_label = get_neo4j_user_base_label(user) + + # Query to find documents that mention entities with this page's title + reverse_query = f""" + // Find entities with the same name as the page title + MATCH (e:{user_base_label}) + WHERE toLower(e.name) = toLower($title) + AND NOT e:Document + + // Find documents that mention those entities + MATCH (d:Document)-[r:MENTIONS]->(e) + WHERE d.page_id <> $page_id // Exclude the page itself + + RETURN DISTINCT d.page_id as page_id, d.title as title + LIMIT 50 + """ + + try: + reverse_refs = await neo4j_client.execute_query( + reverse_query, + {"title": page_title, "page_id": page_id} + ) + logger.info(f"Found {len(reverse_refs)} pages that mention '{page_title}'") + except Exception as e: + logger.error(f"Failed to find reverse references: {e}") + reverse_refs = [] + + # STEP 3: Backward linking - add links in those pages to the new entity + if reverse_refs: + logger.info(f"Step 3/3: Adding backward links in {len(reverse_refs)} pages") + for ref in reverse_refs: + try: + backward_result = await link_entities_in_page( + request=EntityLinkingRequest( + user=user, + page_id=ref['page_id'], + create_relationships=False, # Relationships already exist + re_index_if_changed=False # Don't re-index for link updates + ), + wiki_service=wiki_svc, + graph_service=graph_service, + ingestion_service=ingestion_svc, + api_key="" + ) + if backward_result.content_links_added > 0: + backward_links += backward_result.content_links_added + pages_updated += 1 + logger.info( + f"Added {backward_result.content_links_added} links " + f"in page {ref['page_id']} ('{ref['title']}')" + ) + except Exception as e: + logger.error(f"Failed to add backward links in page {ref['page_id']}: {e}") + else: + logger.info("Step 3/3: No reverse references found, skipping backward linking") + + return { + "forward_links": forward_links, + "backward_links": backward_links, + "pages_updated": pages_updated + } + + except Exception as e: + logger.error(f"Bidirectional entity linking failed: {e}", exc_info=True) + return { + "forward_links": 0, + "backward_links": 0, + "pages_updated": 0 + } diff --git a/src/services/wiki_service.py b/src/services/wiki_service.py index 4a8c5ca..43f454a 100644 --- a/src/services/wiki_service.py +++ b/src/services/wiki_service.py @@ -431,3 +431,166 @@ class WikiService: WikiPageList filtered by dossier tag """ return await self.list_pages(user, tag=dossier_name, limit=limit) + + async def smart_create_page( + self, + topic: str, + user: str, + path: Optional[str], + tags: List[str], + hybrid_rag_service: "HybridRAGService", + wiki_page_writer: "WikiPageWriter", + include_web: bool = True, + include_wiki: bool = True + ) -> tuple["WikiPage", Dict[str, Any]]: + """ + Create wiki page with research from HybridRAG. + + This method combines research + content generation + page creation: + 1. Run HybridRAG search on topic + 2. Format results for WikiPageWriter + 3. Generate page content with LLM + 4. Create page in Wiki.js + 5. Return page + research summary + + Args: + topic: Topic to research and create page about + user: User identifier + path: Optional page path (auto-generated from topic if not provided) + tags: Tags for the page + hybrid_rag_service: HybridRAG service for multi-source search + wiki_page_writer: WikiPageWriter for LLM content generation + include_web: Include web search results + include_wiki: Include existing wiki knowledge + + Returns: + Tuple of (created WikiPage, research summary dict) + """ + from src.models.hybrid_rag import HybridRAGConfig + + logger.info(f"Smart create page: topic='{topic}', user='{user}'") + + # Step 1: Run HybridRAG search on the topic + config = HybridRAGConfig( + enable_vector=include_wiki, + enable_graph=include_wiki, + enable_web=include_web, + enable_reranking=True, + enable_enrichment=True, + final_result_count=15 # Get more results for rich content + ) + + search_response = await hybrid_rag_service.search( + query=topic, + user=user, + config=config + ) + + logger.info( + f"HybridRAG search completed: {search_response.total_results} results, " + f"search_id={search_response.search_id}" + ) + + # Step 2: Format results for WikiPageWriter + source_information = [] + wiki_results_count = 0 + web_results_count = 0 + graph_entities_count = 0 + + for result in search_response.results: + source_type = result.source_type + + if "web" in source_type: + web_results_count += 1 + source_information.append({ + "title": result.title, + "url": result.url or "", + "content": result.content[:500] if result.content else "" + }) + elif "vector" in source_type or "graph" in source_type: + wiki_results_count += 1 + # For wiki results, use page path as URL + source_information.append({ + "title": result.title, + "url": f"/{result.page_path}" if result.page_path else "", + "content": result.content[:500] if result.content else "" + }) + + # Count entities from related dossiers + if result.related_dossiers: + graph_entities_count += len(result.related_dossiers) + + # Step 3: Generate page content with LLM + # Use topic as summary and let WikiPageWriter create structured content + topic_summary = f"Research findings about: {topic}" + if search_response.keywords: + topic_summary += f"\n\nKey concepts: {', '.join(search_response.keywords.core_keywords)}" + + # Extract entities from search results for knowledge graph linking + entities = [] + if search_response.keywords and search_response.keywords.core_keywords: + entities = search_response.keywords.core_keywords[:10] + + # Get related documents for cross-linking + related_docs = [] + for result in search_response.results[:5]: + if result.page_path: + related_docs.append(f"[{result.title}](/{result.page_path})") + + content = await wiki_page_writer.create_page( + title=topic, + topic_summary=topic_summary, + source_information=source_information[:10], # Limit sources + entities=entities, + related_docs=related_docs + ) + + logger.info(f"Generated page content: {len(content)} characters") + + # Step 4: Auto-generate path from topic if not provided + if not path: + # Convert topic to kebab-case path + import re + path_slug = topic.lower() + path_slug = re.sub(r'[^\w\s-]', '', path_slug) # Remove special chars + path_slug = re.sub(r'\s+', '-', path_slug) # Spaces to hyphens + path_slug = re.sub(r'-+', '-', path_slug) # Multiple hyphens to single + path_slug = path_slug.strip('-') + + # Infer category from tags or use reference + category = "reference" + if tags: + category = tags[0].lower() + + path = f"/{category}/{path_slug}" + + # Step 5: Create page using existing create_page method + from src.models.wiki import WikiPageCreate + + page_data = WikiPageCreate( + title=topic, + path=path, + content=content, + description=f"Research summary about {topic}", + tags=tags, + user=user + ) + + page = await self.create_page(page_data) + + logger.info(f"Created page: id={page.id}, path={page.path}") + + # Build research summary + research_summary = { + "wiki_results": wiki_results_count, + "web_results": web_results_count, + "graph_entities": graph_entities_count, + "keywords_extracted": len(search_response.keywords.core_keywords) if search_response.keywords else 0, + "timing_ms": search_response.timing.total_ms if search_response.timing else 0 + } + + return page, { + "research_summary": research_summary, + "sources_used": len(source_information), + "search_id": search_response.search_id + }