diff --git a/services/library-desk/src/services/graph_service.py b/services/library-desk/src/services/graph_service.py index 2288901..849b2a6 100644 --- a/services/library-desk/src/services/graph_service.py +++ b/services/library-desk/src/services/graph_service.py @@ -264,48 +264,113 @@ class GraphService: """ Extract entities from page content. - Simple regex-based extraction for now. - TODO: Use spaCy or similar NLP library for better extraction. + Uses multiple strategies: + 1. Markdown links (explicit entity references) + 2. @mentions (person references) + 3. [[WikiLinks]] (concept references) + 4. Capitalized multi-word phrases (proper nouns) + 5. Hardcoded technology keywords Args: content: Markdown content Returns: - List of extracted entities + List of extracted entities with deduplication """ entities = [] + seen_entities = set() # Track unique entities (text, type) pairs - # Extract mentions in format: @username, #tag, [[WikiLink]] - # Person mentions: @username + def add_entity(text: str, entity_type: str, confidence: float): + """Helper to add entity with deduplication.""" + # Normalize text + text = text.strip() + if not text or len(text) < 2: + return + + # Create unique key + key = (text.lower(), entity_type) + if key not in seen_entities: + seen_entities.add(key) + entities.append(EntityMention( + text=text, + type=entity_type, + confidence=confidence + )) + + # Strategy 1: Markdown links - explicit entity references + # Examples: [Docker](/technology/docker), [John Doe](/people/john-doe) + markdown_link_pattern = r'\[([^\]]+)\]\(([^\)]+)\)' + for match in re.finditer(markdown_link_pattern, content): + link_text = match.group(1) + link_path = match.group(2) + + # Skip external links (http/https) + if link_path.startswith(('http://', 'https://')): + continue + + # Infer entity type from path + entity_type = "Entity" # Default + if '/people/' in link_path or '/person/' in link_path: + entity_type = "Person" + elif '/companies/' in link_path or '/company/' in link_path or '/organizations/' in link_path: + entity_type = "Organization" + elif '/places/' in link_path or '/locations/' in link_path: + entity_type = "Place" + elif '/technology/' in link_path or '/tech/' in link_path: + entity_type = "Technology" + elif '/products/' in link_path or '/product/' in link_path: + entity_type = "Product" + elif '/projects/' in link_path or '/project/' in link_path: + entity_type = "Project" + elif '/events/' in link_path or '/event/' in link_path: + entity_type = "Event" + + add_entity(link_text, entity_type, confidence=0.95) + + # Strategy 2: Person mentions: @username person_pattern = r'@([a-zA-Z0-9_-]+)' for match in re.finditer(person_pattern, content): - entities.append(EntityMention( - text=match.group(1), - type="Person", - confidence=0.8 - )) + add_entity(match.group(1), "Person", confidence=0.8) - # Project/Concept mentions: [[WikiLink]] + # Strategy 3: WikiLink mentions: [[WikiLink]] wikilink_pattern = r'\[\[([^\]]+)\]\]' for match in re.finditer(wikilink_pattern, content): - entities.append(EntityMention( - text=match.group(1), - type="Concept", - confidence=0.9 - )) + add_entity(match.group(1), "Concept", confidence=0.9) - # Technology mentions: docker, kubernetes, python, etc. + # Strategy 4: Capitalized multi-word phrases (proper nouns) + # Matches phrases like "RSG Lingecollege", "John Cabot University", "Google Cloud Platform" + # Pattern: Word starting with capital, followed by 1-4 more capitalized words + proper_noun_pattern = r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b' + for match in re.finditer(proper_noun_pattern, content): + phrase = match.group(1) + + # Filter out common false positives + # Skip if starts with common sentence starters + first_word = phrase.split()[0] + if first_word in {'The', 'This', 'That', 'These', 'Those', 'A', 'An', + 'My', 'Your', 'His', 'Her', 'Our', 'Their', + 'Some', 'Many', 'Few', 'Several', 'All', 'Most'}: + continue + + # Skip if all words are common words (likely not a proper noun) + common_words = {'And', 'Or', 'But', 'For', 'With', 'From', 'About', + 'After', 'Before', 'During', 'Until', 'Since'} + if all(word in common_words for word in phrase.split()): + continue + + # Guess entity type based on context or use generic + add_entity(phrase, "Entity", confidence=0.6) + + # Strategy 5: Technology keywords (fallback for common tech) tech_keywords = ['docker', 'kubernetes', 'python', 'neo4j', 'qdrant', - 'wikijs', 'fastapi', 'ollama', 'redis'] + 'wikijs', 'fastapi', 'ollama', 'redis', 'postgresql', + 'react', 'nodejs', 'typescript', 'javascript'] content_lower = content.lower() for tech in tech_keywords: if tech in content_lower: - entities.append(EntityMention( - text=tech.capitalize(), - type="Technology", - confidence=0.7 - )) + add_entity(tech.capitalize(), "Technology", confidence=0.7) + logger.debug(f"Extracted {len(entities)} unique entities from content") return entities async def update_from_page( @@ -360,6 +425,7 @@ class GraphService: MERGE (d:{user_doc_label}:Document {{page_id: $page_id}}) SET d.title = $title, d.path = $path, + d.tags = $tags, d.updated_at = datetime(), d.content_length = $content_length RETURN d @@ -369,6 +435,7 @@ class GraphService: "page_id": page_id, "title": page.get("title"), "path": page.get("path"), + "tags": tags, "content_length": len(content) })