""" Graph service for Library Desk Neo4j operations. Handles knowledge graph management with user-scoped operations. """ import re import time from typing import List, Dict, Any, Optional from datetime import datetime import logging from src.clients.neo4j_client import Neo4jClient from src.clients.wikijs_client import WikiJSClient from src.core.multi_tenancy import get_neo4j_user_label, is_path_in_user_namespace from src.models.graph import ( GraphNode, GraphRelationship, GraphNodeDetail, CypherQueryResponse, GraphUpdateSummary, EntityMention, NodeListResponse, MindMapNode, MindMapLink, MindMapResponse ) logger = logging.getLogger(__name__) def _serialize_neo4j_types(obj: Any) -> Any: """ Convert Neo4j types to JSON-serializable types. Args: obj: Object that may contain Neo4j types Returns: JSON-serializable object """ if obj is None: return None # Handle Neo4j DateTime if hasattr(obj, 'to_native'): # Neo4j temporal types have to_native() return obj.to_native().isoformat() # Handle dict recursively if isinstance(obj, dict): return {k: _serialize_neo4j_types(v) for k, v in obj.items()} # Handle list recursively if isinstance(obj, list): return [_serialize_neo4j_types(item) for item in obj] # Return as-is for basic types return obj class GraphService: """ Service for Neo4j knowledge graph operations. Responsibilities: - User-scoped graph queries - Entity extraction from wiki pages - Graph updates from page content - Mind map generation """ def __init__(self, neo4j_client: Neo4jClient, wikijs_client: WikiJSClient): """ Initialize graph service. Args: neo4j_client: Neo4j database client wikijs_client: Wiki.js client for fetching pages """ self.neo4j = neo4j_client self.wiki = wikijs_client # Conservative denylist of Cypher write clauses / procedure calls. # Matched as whole words against the uppercased query. CALL is rejected # entirely (covers db.*/apoc.* write procedures and CALL {} subqueries) # because reliably distinguishing read from write procedures would # require a real Cypher parser. _WRITE_CLAUSE_PATTERN = re.compile( r"\b(CREATE|MERGE|DELETE|DETACH|SET|REMOVE|DROP|FOREACH|LOAD|CALL)\b" ) async def execute_query( self, query: str, parameters: Dict[str, Any], user: str ) -> CypherQueryResponse: """ Execute a raw Cypher query — READ-ONLY, NOT tenant-scoped. Security model (admin/debug endpoint): - Queries containing write clauses (CREATE/MERGE/DELETE/SET/REMOVE/ DROP/DETACH/FOREACH/LOAD CSV) or any CALL are rejected up front. - The query is executed through a session opened with default_access_mode=READ_ACCESS, so the database itself refuses writes even if the denylist is bypassed. - Results are NOT automatically restricted to the user's tenant labels: an arbitrary Cypher query can read any tenant's nodes. Callers must scope patterns themselves (e.g. match on `User_`/`User__Document` labels). Args: query: Cypher query (read-only) parameters: Query parameters user: Requesting user (audit logging only — does NOT scope the query) Returns: Query results with metadata Raises: ValueError: If the query contains write clauses or fails """ start_time = time.time() self._reject_write_clauses(query) logger.info(f"Read-only Cypher query for user '{user}' (unscoped): {query[:200]}") try: results = await self.neo4j.execute_read(query, parameters) query_time_ms = (time.time() - start_time) * 1000 return CypherQueryResponse( results=results, count=len(results), query_time_ms=query_time_ms ) except ValueError: raise except Exception as e: logger.error(f"Cypher query failed: {e}", exc_info=True) raise ValueError(f"Query execution failed: {str(e)}") def _reject_write_clauses(self, query: str) -> None: """ Reject Cypher queries containing write clauses or procedure calls. Conservative denylist on the uppercased query: false positives are acceptable (e.g. the word SET in a string literal), false negatives are not. The read-only session is the hard backstop. Args: query: Raw Cypher query Raises: ValueError: If a denylisted clause is found """ match = self._WRITE_CLAUSE_PATTERN.search(query.upper()) if match: raise ValueError( f"Query rejected: '{match.group(1)}' is not allowed — " "/query/graph and /graph/query are read-only (no write " "clauses or CALL procedures)" ) async def list_nodes( self, user: str, node_type: Optional[str] = None, limit: int = 100 ) -> NodeListResponse: """ List nodes for a user. Args: user: User identifier node_type: Optional node type filter (Document, Person, etc.) limit: Maximum nodes to return Returns: List of nodes """ user_label = get_neo4j_user_label(user) # Build query based on filters # Note: We need to explicitly return elementId and labels since result.data() converts nodes to dicts if node_type: query = f""" MATCH (n:{user_label}:{node_type}) RETURN elementId(n) as id, labels(n) as labels, properties(n) as props LIMIT $limit """ else: query = f""" MATCH (n:{user_label}) RETURN elementId(n) as id, labels(n) as labels, properties(n) as props LIMIT $limit """ results = await self.neo4j.execute_query(query, {"limit": limit}) nodes = [ GraphNode( id=str(record["id"]), labels=record["labels"], properties=_serialize_neo4j_types(record["props"]) ) for record in results ] return NodeListResponse( nodes=nodes, total=len(nodes), user=user ) async def get_node(self, node_id: str, user: str) -> Optional[GraphNodeDetail]: """ Get node details with relationships. Args: node_id: Node element ID user: User identifier Returns: Node details or None if not found """ user_label = get_neo4j_user_label(user) query = f""" MATCH (n:{user_label}) WHERE elementId(n) = $node_id OPTIONAL MATCH (n)-[r]-(m) RETURN elementId(n) as node_id, labels(n) as node_labels, properties(n) as node_props, collect({{ id: elementId(r), type: type(r), start_node: elementId(startNode(r)), end_node: elementId(endNode(r)), props: properties(r) }}) as rels, collect({{ id: elementId(m), labels: labels(m), props: properties(m) }}) as related """ results = await self.neo4j.execute_query(query, {"node_id": node_id}) if not results: return None record = results[0] node = GraphNode( id=node_id, labels=record["node_labels"], properties=_serialize_neo4j_types(record["node_props"]) ) # Parse relationships relationships = [] related_nodes = [] for rel in record["rels"]: if rel and rel["id"]: # Check if relationship exists (not null from OPTIONAL MATCH) relationships.append(GraphRelationship( id=rel["id"], type=rel["type"], start_node=rel["start_node"], end_node=rel["end_node"], properties=_serialize_neo4j_types(rel["props"] or {}) )) for rel_node in record["related"]: if rel_node and rel_node["id"]: # Check if node exists related_nodes.append(GraphNode( id=rel_node["id"], labels=rel_node["labels"], properties=_serialize_neo4j_types(rel_node["props"] or {}) )) return GraphNodeDetail( node=node, relationships=relationships, related_nodes=related_nodes ) def _extract_entities(self, content: str) -> List[EntityMention]: """ Extract entities from page content. Uses multiple strategies: 1. Markdown links (explicit entity references) 2. @mentions (person references) 3. [[WikiLinks]] (concept references) 4. Capitalized multi-word phrases (proper nouns) 5. Hardcoded technology keywords Args: content: Markdown content Returns: List of extracted entities with deduplication """ entities = [] seen_entities = set() # Track unique entities (text, type) pairs def add_entity(text: str, entity_type: str, confidence: float): """Helper to add entity with deduplication.""" # Normalize text text = text.strip() if not text or len(text) < 2: return # Create unique key key = (text.lower(), entity_type) if key not in seen_entities: seen_entities.add(key) entities.append(EntityMention( text=text, type=entity_type, confidence=confidence )) # Strategy 1: Markdown links - explicit entity references # Examples: [Docker](/technology/docker), [John Doe](/people/john-doe) markdown_link_pattern = r'\[([^\]]+)\]\(([^\)]+)\)' for match in re.finditer(markdown_link_pattern, content): link_text = match.group(1) link_path = match.group(2) # Skip external links (http/https) if link_path.startswith(('http://', 'https://')): continue # Infer entity type from path entity_type = "Entity" # Default if '/people/' in link_path or '/person/' in link_path: entity_type = "Person" elif '/companies/' in link_path or '/company/' in link_path or '/organizations/' in link_path: entity_type = "Organization" elif '/places/' in link_path or '/locations/' in link_path: entity_type = "Place" elif '/technology/' in link_path or '/tech/' in link_path: entity_type = "Technology" elif '/products/' in link_path or '/product/' in link_path: entity_type = "Product" elif '/projects/' in link_path or '/project/' in link_path: entity_type = "Project" elif '/events/' in link_path or '/event/' in link_path: entity_type = "Event" add_entity(link_text, entity_type, confidence=0.95) # Strategy 2: Person mentions: @username person_pattern = r'@([a-zA-Z0-9_-]+)' for match in re.finditer(person_pattern, content): add_entity(match.group(1), "Person", confidence=0.8) # Strategy 3: WikiLink mentions: [[WikiLink]] wikilink_pattern = r'\[\[([^\]]+)\]\]' for match in re.finditer(wikilink_pattern, content): add_entity(match.group(1), "Concept", confidence=0.9) # Strategy 4: Capitalized multi-word phrases (proper nouns) # Matches phrases like "RSG Lingecollege", "John Cabot University", "Google Cloud Platform" # Pattern: Word starting with capital, followed by 1-4 more capitalized words proper_noun_pattern = r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b' for match in re.finditer(proper_noun_pattern, content): phrase = match.group(1) # Filter out common false positives # Skip if starts with common sentence starters first_word = phrase.split()[0] if first_word in {'The', 'This', 'That', 'These', 'Those', 'A', 'An', 'My', 'Your', 'His', 'Her', 'Our', 'Their', 'Some', 'Many', 'Few', 'Several', 'All', 'Most'}: continue # Skip if all words are common words (likely not a proper noun) common_words = {'And', 'Or', 'But', 'For', 'With', 'From', 'About', 'After', 'Before', 'During', 'Until', 'Since'} if all(word in common_words for word in phrase.split()): continue # Guess entity type based on context or use generic add_entity(phrase, "Entity", confidence=0.6) # Strategy 5: Technology keywords (fallback for common tech) tech_keywords = ['docker', 'kubernetes', 'python', 'neo4j', 'qdrant', 'wikijs', 'fastapi', 'ollama', 'redis', 'postgresql', 'react', 'nodejs', 'typescript', 'javascript'] content_lower = content.lower() for tech in tech_keywords: if tech in content_lower: add_entity(tech.capitalize(), "Technology", confidence=0.7) logger.debug(f"Extracted {len(entities)} unique entities from content") return entities async def update_from_page( self, page_id: int, user: str, force_refresh: bool = False ) -> GraphUpdateSummary: """ Update knowledge graph from a wiki page. Extracts entities and creates/updates graph nodes and relationships. Args: page_id: Wiki page ID user: User identifier force_refresh: Force re-extraction even if unchanged Returns: Summary of update operation """ start_time = time.time() try: # Fetch page from Wiki.js page = await self.wiki.get_page(page_id) if not page: raise ValueError(f"Page {page_id} not found") # TENANT ISOLATION: only pages inside the user's own wiki # namespace may be written into that user's graph labels. if not is_path_in_user_namespace(page.get("path", ""), user): raise ValueError( f"Page {page_id} (path: {page.get('path')!r}) is outside " f"user '{user}' namespace - refusing cross-tenant ingestion" ) # PROTECTION: Skip entity extraction on auto-generated entity stub pages tags = page.get("tags", []) if "entity-stub" in tags or "auto-generated" in tags: logger.info(f"Skipping entity extraction for auto-generated page {page_id}") return GraphUpdateSummary( page_id=page_id, page_title=page.get("title", ""), processing_time_ms=(time.time() - start_time) * 1000, success=True ) # Extract entities from content content = page.get("content", "") entities = self._extract_entities(content) # Get user labels for namespacing from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) # For entities user_doc_label = get_neo4j_user_label(user) # For documents # Create/update Document node. # content_hash records the fingerprint of the ingested content so # /ingest/check-updates can detect changed pages without re-reading # the graph's source content. from src.core.hashing import compute_content_hash doc_query = f""" MERGE (d:{user_doc_label}:Document {{page_id: $page_id}}) SET d.title = $title, d.path = $path, d.tags = $tags, d.updated_at = datetime(), d.content_length = $content_length, d.content_hash = $content_hash RETURN d """ await self.neo4j.execute_write(doc_query, { "page_id": page_id, "title": page.get("title"), "path": page.get("path"), "tags": tags, "content_length": len(content), "content_hash": compute_content_hash(content) }) nodes_created = 1 # Document node nodes_updated = 0 relationships_created = 0 # Create entity nodes and relationships for entity in entities: entity_query = f""" MERGE (e:{user_base_label}:{entity.type} {{name: $name}}) WITH e MATCH (d:{user_doc_label}:Document {{page_id: $page_id}}) MERGE (d)-[r:MENTIONS]->(e) SET r.confidence = $confidence RETURN e, r """ result = await self.neo4j.execute_write(entity_query, { "name": entity.text, "page_id": page_id, "confidence": entity.confidence }) if result: relationships_created += 1 processing_time_ms = (time.time() - start_time) * 1000 logger.info(f"Updated graph from page {page_id}: {len(entities)} entities") return GraphUpdateSummary( page_id=page_id, page_title=page.get("title", ""), nodes_created=nodes_created, nodes_updated=nodes_updated, relationships_created=relationships_created, entities_extracted=entities, processing_time_ms=processing_time_ms, success=True ) except Exception as e: processing_time_ms = (time.time() - start_time) * 1000 logger.error(f"Failed to update graph from page {page_id}: {e}", exc_info=True) return GraphUpdateSummary( page_id=page_id, page_title="Unknown", processing_time_ms=processing_time_ms, success=False, error_message=str(e) ) async def delete_page( self, page_id: int, user: str ) -> int: """ Delete a page's Document node and all its relationships from the graph. Args: page_id: Wiki page ID to delete user: User identifier Returns: Number of nodes deleted (should be 1 if successful, 0 if not found) """ from src.core.multi_tenancy import get_neo4j_user_label user_doc_label = get_neo4j_user_label(user) # Delete Document node and all its relationships # DETACH DELETE removes the node and all relationships connected to it delete_query = f""" MATCH (d:{user_doc_label}:Document {{page_id: $page_id}}) DETACH DELETE d RETURN count(d) as deleted_count """ try: result = await self.neo4j.execute_write( delete_query, {"page_id": page_id} ) deleted_count = result[0]["deleted_count"] if result else 0 if deleted_count > 0: logger.info(f"Deleted Document node for page {page_id} from graph") else: logger.warning(f"No Document node found for page {page_id}") return deleted_count except Exception as e: logger.error(f"Failed to delete page {page_id} from graph: {e}", exc_info=True) return 0 async def generate_mindmap( self, center_node_id: str, user: str, depth: int = 2 ) -> MindMapResponse: """ Generate mind map data for visualization. Args: center_node_id: Central node ID user: User identifier depth: Traversal depth Returns: Mind map nodes and links """ user_label = get_neo4j_user_label(user) # Traverse graph from center node query = f""" MATCH path = (center:{user_label})-[*1..{depth}]-(related:{user_label}) WHERE elementId(center) = $center_id WITH center, collect(distinct related) as nodes, collect(relationships(path)) as rels_list UNWIND rels_list as rels_in_path UNWIND rels_in_path as rel WITH center, nodes, collect(distinct rel) as all_rels RETURN {{ id: elementId(center), labels: labels(center), props: properties(center) }} as center, [node in nodes | {{ id: elementId(node), labels: labels(node), props: properties(node) }}] as nodes, [r in all_rels | {{ id: elementId(r), type: type(r), start_node: elementId(startNode(r)), end_node: elementId(endNode(r)), props: properties(r) }}] as all_rels """ results = await self.neo4j.execute_query(query, {"center_id": center_node_id}) if not results: return MindMapResponse(nodes=[], links=[], center_node=center_node_id, depth=depth) record = results[0] # Build mind map nodes mind_nodes = [] # Center node center = record["center"] center_props = _serialize_neo4j_types(center["props"]) mind_nodes.append(MindMapNode( id=center["id"], label=center_props.get("title") or center_props.get("name", "Unknown"), type=center["labels"][0] if center["labels"] else "Node", size=20, color="#FF6B6B" )) # Related nodes for node in record["nodes"]: node_props = _serialize_neo4j_types(node["props"]) mind_nodes.append(MindMapNode( id=node["id"], label=node_props.get("title") or node_props.get("name", "Unknown"), type=node["labels"][0] if node["labels"] else "Node", size=10 )) # Build links links = [] for rel in record["all_rels"]: rel_props = _serialize_neo4j_types(rel["props"]) if rel["props"] else {} links.append(MindMapLink( source=rel["start_node"], target=rel["end_node"], type=rel["type"], strength=rel_props.get("confidence", 1.0) )) return MindMapResponse( nodes=mind_nodes, links=links, center_node=center_node_id, depth=depth ) async def _get_entity_mention_count( self, entity_name: str, entity_type: str, user: str ) -> int: """ Count how many documents mention an entity. Args: entity_name: Entity name entity_type: Entity type (Person, Technology, etc.) user: User identifier Returns: Number of documents mentioning this entity """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) query = f""" MATCH (e:{user_base_label}:{entity_type} {{name: $name}}) MATCH (d:{user_doc_label}:Document)-[:MENTIONS]->(e) RETURN count(distinct d) as mention_count """ try: results = await self.neo4j.execute_query(query, {"name": entity_name}) if results: return results[0]["mention_count"] return 0 except Exception as e: logger.error(f"Failed to get mention count: {e}") return 0 async def _entity_has_wiki_page( self, entity_name: str, entity_type: str, user: str ) -> bool: """ Check if entity already has a wiki page. Args: entity_name: Entity name entity_type: Entity type user: User identifier Returns: True if page exists """ # Construct entity page path (within user's namespace for multi-tenancy) from src.core.multi_tenancy import get_wikijs_namespace user_namespace = get_wikijs_namespace(user) entity_path = f"{user_namespace}/entities/{entity_type.lower()}/{entity_name.lower().replace(' ', '-')}" try: # Search for page by path (scoped to the user's namespace) pages = await self.wiki.list_pages(path_prefix=user_namespace, limit=1000) # list_pages returns a list directly, not a dict for page in pages: if page.get("path", "") == entity_path: return True return False except Exception as e: logger.error(f"Failed to check for entity page: {e}") return False def _generate_entity_stub_content( self, entity_name: str, entity_type: str, mentioning_pages: List[Dict[str, Any]], related_entities: List[Dict[str, Any]] ) -> str: """ Generate markdown content for entity stub page. Args: entity_name: Entity name entity_type: Entity type mentioning_pages: Pages that mention this entity related_entities: Related entities from graph Returns: Markdown content string """ # Build mentions section with Wiki.js links (with locale prefix) mentions_md = "\n".join([ f"- [{page['title']}](/en/{page['path']})" for page in mentioning_pages[:10] # Limit to first 10 ]) if len(mentioning_pages) > 10: mentions_md += f"\n\n*...and {len(mentioning_pages) - 10} more*" # Build related entities section with Wiki.js links if related_entities: related_items = [] for e in related_entities[:10]: name = e['name'] entity_type = e.get('type', 'Entity') path = e.get('path') # If entity has a stub page, link to it; otherwise just show name if path: related_items.append(f"- [{name}](/en/{path}) ({entity_type})") else: related_items.append(f"- {name} ({entity_type})") related_md = "\n".join(related_items) else: related_md = "*No related entities found yet*" # Generate content content = f"""# {entity_name} **Type:** {entity_type} **Mentioned in:** {len(mentioning_pages)} page(s) ## Overview *This entity has been detected in the knowledge graph. Add a description here to expand this page.* ## Documents Mentioning This Entity {mentions_md} ## Related Entities {related_md} ## Graph Visualization To see how this entity connects to others in the knowledge graph, use the mind map endpoint: ``` GET /graph/mindmap?center_node=[node_id]&user=[user] ``` --- 🤖 **This page was auto-generated from the knowledge graph** on {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}. Feel free to expand it with more details! """ return content async def _create_entity_stub_page( self, entity_name: str, entity_type: str, user: str ) -> Optional[int]: """ Create wiki stub page for an entity. Args: entity_name: Entity name entity_type: Entity type user: User identifier Returns: Page ID if created, None if failed """ try: from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) # Get mentioning documents (scoped to this tenant's documents) mention_query = f""" MATCH (e:{user_base_label}:{entity_type} {{name: $name}}) MATCH (d:{user_doc_label}:Document)-[:MENTIONS]->(e) RETURN d.title as title, d.path as path, d.page_id as page_id """ mention_results = await self.neo4j.execute_query( mention_query, {"name": entity_name} ) mentioning_pages = [ {"title": r["title"], "path": r["path"], "page_id": r["page_id"]} for r in mention_results ] # Get related entities (entities that co-occur in same documents) # Only match entity nodes (not Document nodes) related_query = f""" MATCH (e1:{user_base_label}:{entity_type} {{name: $name}}) MATCH (d:{user_doc_label}:Document)-[:MENTIONS]->(e1) MATCH (d)-[:MENTIONS]->(e2:{user_base_label}) WHERE e2 <> e1 AND NOT (e2:Document) RETURN DISTINCT e2.name as name, labels(e2) as labels, properties(e2).path as path, count(d) as co_occurrence ORDER BY co_occurrence DESC LIMIT 10 """ related_results = await self.neo4j.execute_query( related_query, {"name": entity_name} ) related_entities = [ { "name": r["name"], # Get the entity type label (not the user label) "type": [l for l in r["labels"] if l not in [user_base_label, "Document"]][0] if r["labels"] else "Entity", "path": r.get("path") # Include path if it exists (for entity stub pages) } for r in related_results ] # Generate content content = self._generate_entity_stub_content( entity_name=entity_name, entity_type=entity_type, mentioning_pages=mentioning_pages, related_entities=related_entities ) # Create page (within user's namespace for multi-tenancy) from src.core.multi_tenancy import get_wikijs_namespace user_namespace = get_wikijs_namespace(user) entity_path = f"{user_namespace}/entities/{entity_type.lower()}/{entity_name.lower().replace(' ', '-')}" from src.models.wiki import WikiPageCreate page_data = WikiPageCreate( title=entity_name, path=entity_path, content=content, description=f"Auto-generated entity page for {entity_type}: {entity_name}", tags=["entity-stub", "auto-generated", entity_type.lower()], user=user ) # Create the page (use wiki client directly) page = await self.wiki.create_page( path=page_data.path, title=page_data.title, content=page_data.content, description=page_data.description, tags=page_data.tags ) if page: logger.info(f"Created entity stub page for {entity_type} '{entity_name}': page {page['id']}") return page["id"] return None except Exception as e: logger.error(f"Failed to create entity stub page: {e}", exc_info=True) return None async def generate_entity_stubs( self, user: str, min_mentions: int = 5, entity_types: Optional[List[str]] = None ) -> Dict[str, Any]: """ Generate wiki stub pages for entities mentioned multiple times. Args: user: User identifier min_mentions: Minimum number of mentions required entity_types: List of entity types to process (default: all) Returns: Summary with counts of pages created """ if entity_types is None: entity_types = ["Person", "Technology", "Concept", "Project"] from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) pages_created = [] pages_skipped = [] try: # Query for entities with sufficient mentions (tenant-scoped) for entity_type in entity_types: query = f""" MATCH (e:{user_base_label}:{entity_type}) MATCH (d:{user_doc_label}:Document)-[:MENTIONS]->(e) WITH e, count(distinct d) as mention_count WHERE mention_count >= $min_mentions RETURN e.name as name, mention_count ORDER BY mention_count DESC """ results = await self.neo4j.execute_query( query, {"min_mentions": min_mentions} ) logger.info(f"Found {len(results)} {entity_type} entities with >= {min_mentions} mentions") for result in results: entity_name = result["name"] mention_count = result["mention_count"] # Check if page already exists has_page = await self._entity_has_wiki_page( entity_name=entity_name, entity_type=entity_type, user=user ) if has_page: logger.debug(f"Entity '{entity_name}' already has a page, skipping") pages_skipped.append({ "name": entity_name, "type": entity_type, "mentions": mention_count, "reason": "page_exists" }) continue # Create stub page page_id = await self._create_entity_stub_page( entity_name=entity_name, entity_type=entity_type, user=user ) if page_id: pages_created.append({ "name": entity_name, "type": entity_type, "mentions": mention_count, "page_id": page_id }) else: pages_skipped.append({ "name": entity_name, "type": entity_type, "mentions": mention_count, "reason": "creation_failed" }) return { "success": True, "pages_created": len(pages_created), "pages_skipped": len(pages_skipped), "created": pages_created, "skipped": pages_skipped, "min_mentions": min_mentions, "entity_types": entity_types } except Exception as e: logger.error(f"Failed to generate entity stubs: {e}", exc_info=True) return { "success": False, "error": str(e), "pages_created": len(pages_created), "created": pages_created } async def search_documents( self, query: str, user: str, limit: int = 10, keywords_data: Optional[Dict[str, Any]] = None ) -> List[Dict[str, Any]]: """ Search documents via entity matches in knowledge graph. Uses extracted keywords and synonyms from LLM query enhancement to find entities, then returns documents mentioning those entities. Args: query: Original search query user: User identifier limit: Maximum documents to return keywords_data: Extracted keywords/synonyms from Phase 0 (optional) Returns: List of documents with entity match counts """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) # Build search terms from keywords_data or fallback to simple extraction if keywords_data: # Use Phase 0 extracted keywords keywords = keywords_data.get("core_keywords", []) entities = keywords_data.get("entities", []) # Flatten synonyms dict into list synonyms = [] for word, syns in keywords_data.get("synonyms", {}).items(): synonyms.extend(syns) for word, expansions in keywords_data.get("expansions", {}).items(): synonyms.extend(expansions) # Combine all search terms all_terms = list(set(keywords + entities + synonyms)) else: # Fallback: Simple keyword extraction stop_words = {"the", "a", "an", "in", "on", "at", "for", "to", "of", "with", "by"} all_terms = [ w.strip().lower() for w in query.split() if w.strip().lower() not in stop_words and len(w) > 2 ] if not all_terms: logger.warning("No search terms extracted from query") return [] logger.info(f"Graph search using terms: {all_terms[:10]}") # Log first 10 terms # Find documents via entity matches (using keywords OR synonyms) search_query = f""" // Find entities matching query keywords OR synonyms MATCH (e:{user_base_label}) WHERE NOT e:Document AND any(term IN $terms WHERE toLower(e.name) CONTAINS toLower(term)) // Find documents mentioning those entities WITH e MATCH (d:{user_doc_label}:Document)-[:MENTIONS]->(e) // Aggregate results WITH d, count(DISTINCT e) as entity_matches, collect(DISTINCT e.name)[0..5] as matched_entities RETURN d.page_id as page_id, d.title as title, d.path as path, entity_matches, matched_entities ORDER BY entity_matches DESC LIMIT $limit """ try: results = await self.neo4j.execute_query( search_query, {"terms": all_terms, "limit": limit} ) # Deduplicate by page_id (safety net for any edge cases) seen_page_ids = set() unique_results = [] for r in results: page_id = r.get("page_id") if page_id and page_id not in seen_page_ids: seen_page_ids.add(page_id) unique_results.append(r) logger.info(f"Graph search found {len(unique_results)} unique documents (raw: {len(results)})") return unique_results except Exception as e: logger.error(f"Graph document search failed: {e}", exc_info=True) return [] async def get_related_documents( self, page_id: int, user: str, limit: int = 5 ) -> List[Dict[str, Any]]: """ Get documents related to a page via shared entities. Finds documents that mention the same entities as the given page. Returns related docs with their tags (dossiers) and shared entity counts. Args: page_id: Page ID to find related documents for user: User identifier limit: Maximum related documents to return Returns: List of related documents with dossier tags """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) query = f""" MATCH (d1:{user_doc_label}:Document {{page_id: $page_id}}) MATCH (d1)-[:MENTIONS]->(e:{user_base_label})<-[:MENTIONS]-(d2:{user_doc_label}:Document) WHERE d1 <> d2 AND NOT e:Document WITH d2, d2.tags as tags, count(DISTINCT e) as shared_entities WHERE tags IS NOT NULL AND size(tags) > 0 RETURN d2.page_id as page_id, d2.title as title, d2.path as path, tags, shared_entities ORDER BY shared_entities DESC LIMIT $limit """ try: results = await self.neo4j.execute_query( query, {"page_id": page_id, "limit": limit} ) return results except Exception as e: logger.error(f"Failed to get related documents for page {page_id}: {e}", exc_info=True) return [] async def get_related_documents_batch( self, page_ids: List[int], user: str, limit_per_page: int = 5 ) -> Dict[int, List[Dict[str, Any]]]: """ Batch variant of get_related_documents: ONE UNWIND query for all pages instead of one round-trip per page. Args: page_ids: Page IDs to find related documents for user: User identifier limit_per_page: Maximum related documents per page Returns: Mapping of page_id -> related-document rows (same shape as get_related_documents). Pages with no related documents are absent from the mapping. """ from src.core.multi_tenancy import get_neo4j_user_base_label if not page_ids: return {} user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) # ORDER BY runs before collect() so each page's list is sorted by # shared_entities descending; [..$limit] trims per page. query = f""" UNWIND $page_ids AS pid MATCH (d1:{user_doc_label}:Document {{page_id: pid}}) MATCH (d1)-[:MENTIONS]->(e:{user_base_label})<-[:MENTIONS]-(d2:{user_doc_label}:Document) WHERE d1 <> d2 AND NOT e:Document WITH pid, d2, d2.tags as tags, count(DISTINCT e) as shared_entities WHERE tags IS NOT NULL AND size(tags) > 0 WITH pid, d2, tags, shared_entities ORDER BY shared_entities DESC WITH pid, collect({{ page_id: d2.page_id, title: d2.title, path: d2.path, tags: tags, shared_entities: shared_entities }})[..$limit] AS related RETURN pid AS page_id, related """ try: rows = await self.neo4j.execute_query( query, {"page_ids": page_ids, "limit": limit_per_page} ) return {row["page_id"]: row["related"] for row in rows} except Exception as e: logger.error( f"Failed to get related documents for {len(page_ids)} pages: {e}", exc_info=True ) return {} async def get_all_entities(self, user: str) -> List[Dict[str, Any]]: """ Get all entities from the knowledge graph for a user. Returns entities that are not Document nodes (people, places, companies, etc.) Args: user: User identifier Returns: List of entities with name, type, and id """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) query = f""" MATCH (e:{user_base_label}) WHERE NOT e:Document RETURN e.name as name, labels(e) as labels, elementId(e) as id ORDER BY e.name """ try: results = await self.neo4j.execute_query(query, {}) # Format results entities = [] for result in results: # Extract entity type from labels (skip user base label) entity_labels = result.get("labels", []) entity_type = next( (label for label in entity_labels if label != user_base_label), "unknown" ) entities.append({ "name": result.get("name"), "type": entity_type, "id": result.get("id") }) logger.info(f"Retrieved {len(entities)} entities for user {user}") return entities except Exception as e: logger.error(f"Failed to get entities for user {user}: {e}", exc_info=True) return [] async def create_entity_mentions( self, page_id: int, user: str, entity_names: List[Dict[str, Any]] ) -> int: """ Create MENTIONS relationships between a page and entities. Only creates relationships that don't already exist. Args: page_id: Wiki page ID user: User identifier entity_names: List of entities with 'name' and 'mentions' fields Returns: Number of new relationships created """ from src.core.multi_tenancy import get_neo4j_user_label, get_neo4j_user_base_label user_doc_label = get_neo4j_user_label(user) user_base_label = get_neo4j_user_base_label(user) if not entity_names: return 0 # Extract just the entity names names = [e["name"] for e in entity_names] query = f""" // Find the document MATCH (d:{user_doc_label}:Document {{page_id: $page_id}}) // Find entities by name MATCH (e:{user_base_label}) WHERE e.name IN $entity_names AND NOT e:Document // Create MENTIONS relationship if it doesn't exist MERGE (d)-[r:MENTIONS]->(e) ON CREATE SET r.just_created = true, r.created_at = datetime(), r.mention_count = 1 ON MATCH SET r.updated_at = datetime(), r.mention_count = COALESCE(r.mention_count, 0) + 1 // Count only newly created relationships WITH r WHERE r.just_created = true REMOVE r.just_created RETURN count(r) as relationships_created """ try: results = await self.neo4j.execute_write( query, {"page_id": page_id, "entity_names": names} ) count = results[0]["relationships_created"] if results else 0 logger.info(f"Created/updated {count} MENTIONS relationships for page {page_id}") return count except Exception as e: logger.error(f"Failed to create entity mentions: {e}", exc_info=True) return 0 # ========== Cleanup Methods ========== async def delete_document_node( self, document_id: str, user: str ) -> int: """ Delete a Document Store document node and all its relationships. Args: document_id: Document UUID (Document Store) user: User identifier Returns: Number of nodes deleted (1 if successful, 0 if not found) """ user_doc_label = get_neo4j_user_label(user) delete_query = f""" MATCH (d:{user_doc_label}:Document {{document_id: $document_id}}) DETACH DELETE d RETURN count(d) as deleted_count """ try: result = await self.neo4j.execute_write( delete_query, {"document_id": document_id} ) deleted_count = result[0]["deleted_count"] if result else 0 if deleted_count > 0: logger.info(f"Deleted Document node for document {document_id}") else: logger.warning(f"No Document node found for document {document_id}") return deleted_count except Exception as e: logger.error(f"Failed to delete document {document_id} from graph: {e}", exc_info=True) return 0 async def delete_paperless_document( self, paperless_id: int, user: str ) -> int: """ Delete a Paperless document node and all its relationships. Args: paperless_id: Paperless-ngx document ID user: User identifier Returns: Number of nodes deleted (1 if successful, 0 if not found) """ user_doc_label = get_neo4j_user_label(user) delete_query = f""" MATCH (d:{user_doc_label}:Document {{paperless_id: $paperless_id}}) DETACH DELETE d RETURN count(d) as deleted_count """ try: result = await self.neo4j.execute_write( delete_query, {"paperless_id": paperless_id} ) deleted_count = result[0]["deleted_count"] if result else 0 if deleted_count > 0: logger.info(f"Deleted Document node for Paperless document {paperless_id}") else: logger.debug(f"No Document node found for Paperless document {paperless_id}") return deleted_count except Exception as e: logger.error(f"Failed to delete Paperless document {paperless_id} from graph: {e}", exc_info=True) return 0 async def delete_collection_node( self, collection_id: str, user: str ) -> int: """ Delete a DocumentCollection node and all contained documents. Args: collection_id: Collection UUID user: User identifier Returns: Number of nodes deleted (collection + documents) """ user_doc_label = get_neo4j_user_label(user) # Delete collection and all documents it contains delete_query = f""" MATCH (c:{user_doc_label}:DocumentCollection {{id: $collection_id}}) OPTIONAL MATCH (c)-[:CONTAINS]->(d:Document) DETACH DELETE c, d RETURN count(c) + count(d) as deleted_count """ try: result = await self.neo4j.execute_write( delete_query, {"collection_id": collection_id} ) deleted_count = result[0]["deleted_count"] if result else 0 logger.info(f"Deleted collection {collection_id} with {deleted_count} total nodes") return deleted_count except Exception as e: logger.error(f"Failed to delete collection {collection_id}: {e}", exc_info=True) return 0 async def find_orphan_entities( self, user: str ) -> List[Dict[str, Any]]: """ Find entities with no MENTIONS relationships (orphaned). Args: user: User identifier Returns: List of orphaned entities {id, name, type} """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) query = f""" MATCH (e:{user_base_label}) WHERE NOT e:Document AND NOT e:DocumentCollection AND NOT EXISTS {{ (d:{user_doc_label}:Document)-[:MENTIONS]->(e) }} RETURN elementId(e) as id, e.name as name, labels(e) as labels """ try: results = await self.neo4j.execute_query(query, {}) orphans = [] for r in results: labels = r.get("labels", []) entity_type = next( (l for l in labels if l != user_base_label), "Unknown" ) orphans.append({ "id": r["id"], "name": r["name"], "type": entity_type }) logger.info(f"Found {len(orphans)} orphan entities for user {user}") return orphans except Exception as e: logger.error(f"Failed to find orphan entities: {e}", exc_info=True) return [] async def purge_orphan_entities( self, user: str ) -> int: """ Delete all orphaned entities (entities with no MENTIONS relationships). Args: user: User identifier Returns: Number of entities purged """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) user_doc_label = get_neo4j_user_label(user) query = f""" MATCH (e:{user_base_label}) WHERE NOT e:Document AND NOT e:DocumentCollection AND NOT EXISTS {{ (d:{user_doc_label}:Document)-[:MENTIONS]->(e) }} DETACH DELETE e RETURN count(e) as purged_count """ try: results = await self.neo4j.execute_write(query, {}) purged_count = results[0]["purged_count"] if results else 0 logger.info(f"Purged {purged_count} orphan entities for user {user}") return purged_count except Exception as e: logger.error(f"Failed to purge orphan entities: {e}", exc_info=True) return 0 async def get_all_document_references( self, user: str ) -> List[Dict[str, Any]]: """ Get all Document node references for orphan detection. Returns page_id for wiki docs and document_id for Document Store docs. Args: user: User identifier Returns: List of document references {page_id, document_id, doc_type, title} """ user_doc_label = get_neo4j_user_label(user) query = f""" MATCH (d:{user_doc_label}:Document) RETURN d.page_id as page_id, d.document_id as document_id, COALESCE(d.doc_type, 'wiki') as doc_type, d.title as title """ try: results = await self.neo4j.execute_query(query, {}) references = [] for r in results: references.append({ "page_id": r.get("page_id"), "document_id": r.get("document_id"), "doc_type": r.get("doc_type", "wiki"), "title": r.get("title") }) logger.info(f"Found {len(references)} document references for user {user}") return references except Exception as e: logger.error(f"Failed to get document references: {e}", exc_info=True) return [] async def purge_stale_documents_by_ids( self, user: str, page_ids: List[int] = None, document_ids: List[str] = None ) -> int: """ Delete specific stale Document nodes by their IDs. Args: user: User identifier page_ids: List of wiki page IDs to delete document_ids: List of Document Store document IDs to delete Returns: Number of documents purged """ user_doc_label = get_neo4j_user_label(user) total_purged = 0 try: # Purge by page_id (wiki docs) if page_ids: query = f""" MATCH (d:{user_doc_label}:Document) WHERE d.page_id IN $page_ids DETACH DELETE d RETURN count(d) as purged_count """ results = await self.neo4j.execute_write(query, {"page_ids": page_ids}) count = results[0]["purged_count"] if results else 0 total_purged += count logger.info(f"Purged {count} wiki Document nodes") # Purge by document_id (Document Store docs) if document_ids: query = f""" MATCH (d:{user_doc_label}:Document) WHERE d.document_id IN $document_ids DETACH DELETE d RETURN count(d) as purged_count """ results = await self.neo4j.execute_write(query, {"document_ids": document_ids}) count = results[0]["purged_count"] if results else 0 total_purged += count logger.info(f"Purged {count} Document Store Document nodes") return total_purged except Exception as e: logger.error(f"Failed to purge stale documents: {e}", exc_info=True) return 0 async def cleanup_broken_relationships( self, user: str ) -> int: """ Clean up broken FOUND relationships from SearchQuery nodes. Removes relationships pointing to deleted documents. Args: user: User identifier Returns: Number of relationships cleaned """ from src.core.multi_tenancy import get_neo4j_user_base_label user_base_label = get_neo4j_user_base_label(user) query = f""" MATCH (sq:{user_base_label}_SearchQuery:SearchQuery)-[r:FOUND]->(d) WHERE NOT EXISTS {{ (d) }} DELETE r RETURN count(r) as cleaned_count """ try: results = await self.neo4j.execute_write(query, {}) cleaned_count = results[0]["cleaned_count"] if results else 0 if cleaned_count > 0: logger.info(f"Cleaned {cleaned_count} broken FOUND relationships") return cleaned_count except Exception as e: logger.error(f"Failed to cleanup broken relationships: {e}", exc_info=True) return 0 async def find_documents_without_vectors( self, user: str, vector_references: List[Dict[str, Any]] ) -> List[Dict[str, Any]]: """ Find Document nodes that have no corresponding vectors. Used for bidirectional orphan detection - graph nodes without vector data. Args: user: User identifier vector_references: List of vector refs from VectorService.get_all_chunk_references() Returns: List of orphan documents {page_id, document_id, doc_type, title} """ # Get all graph document references graph_docs = await self.get_all_document_references(user) if not graph_docs: return [] # Build sets of IDs that have vectors vector_page_ids = { ref.get("page_id") for ref in vector_references if ref.get("doc_type") == "wiki" and ref.get("page_id") } vector_doc_ids = { ref.get("document_id") for ref in vector_references if ref.get("doc_type") != "wiki" and ref.get("document_id") } # Find graph docs with no vectors orphans = [] for doc in graph_docs: doc_type = doc.get("doc_type", "wiki") if doc_type == "wiki": page_id = doc.get("page_id") if page_id and page_id not in vector_page_ids: orphans.append(doc) else: document_id = doc.get("document_id") if document_id and document_id not in vector_doc_ids: orphans.append(doc) logger.info(f"Found {len(orphans)} graph documents without vectors for user {user}") return orphans