feat(library-desk): improve entity linking with fuzzy matching
- Add fuzzy_match_entity_to_document with confidence scoring - Filter self-referential links (entity linking to current page) - Fix path cleaning to preserve full wiki paths - Add longest-first matching to prevent partial matches 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -12,6 +12,8 @@ from pydantic import BaseModel
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
import re
|
||||
|
||||
from src.config import get_settings
|
||||
|
||||
from src.core.dependencies import (
|
||||
get_wiki_service,
|
||||
get_graph_service,
|
||||
@@ -98,6 +100,13 @@ async def link_entities_in_page(
|
||||
found_entities = find_entity_mentions(page_content, entities)
|
||||
logger.info(f"Found {len(found_entities)} entity mentions in page")
|
||||
|
||||
# Filter out self-referential links (entities linking to the current page)
|
||||
found_entities = [
|
||||
e for e in found_entities
|
||||
if e.get("doc_page_id") != request.page_id
|
||||
]
|
||||
logger.info(f"After filtering self-references: {len(found_entities)} entities to link")
|
||||
|
||||
# Step 4: Add hyperlinks to wiki content for entities with pages
|
||||
content_links_added = 0
|
||||
content_updated = False
|
||||
@@ -209,7 +218,8 @@ def find_entity_mentions(content: str, entities: List[Dict[str, Any]]) -> List[D
|
||||
"type": entity.get("type", "unknown"),
|
||||
"mentions": len(matches),
|
||||
"entity_id": entity.get("id"),
|
||||
"path": entity.get("path") # Include path if available
|
||||
"path": entity.get("path"), # Include path if available
|
||||
"doc_page_id": entity.get("doc_page_id") # Include for self-reference filtering
|
||||
})
|
||||
|
||||
# Sort by number of mentions (descending)
|
||||
@@ -218,45 +228,156 @@ def find_entity_mentions(content: str, entities: List[Dict[str, Any]]) -> List[D
|
||||
return found
|
||||
|
||||
|
||||
def fuzzy_match_entity_to_document(
|
||||
entity_name: str,
|
||||
doc_title: str,
|
||||
settings
|
||||
) -> Tuple[bool, float]:
|
||||
"""
|
||||
Match entity name to document title with tolerance for variations.
|
||||
|
||||
Uses multiple strategies with confidence scoring:
|
||||
1. Exact match (confidence: 1.0)
|
||||
2. Containment match (confidence: 0.85-0.80)
|
||||
3. Token overlap match (confidence: 0.60-0.80)
|
||||
|
||||
Args:
|
||||
entity_name: Entity name to match
|
||||
doc_title: Document title to match against
|
||||
settings: Application settings with matching thresholds
|
||||
|
||||
Returns:
|
||||
Tuple of (is_match, confidence_score)
|
||||
"""
|
||||
e_lower = entity_name.lower().strip()
|
||||
d_lower = doc_title.lower().strip()
|
||||
|
||||
# Strategy 1: Exact match (confidence: 1.0)
|
||||
if e_lower == d_lower:
|
||||
return True, 1.0
|
||||
|
||||
# Strategy 2: Containment match (confidence: 0.85-0.80)
|
||||
# Require minimum length to avoid false positives
|
||||
if len(entity_name) >= settings.entity_linking_min_entity_length:
|
||||
# Entity is substring of title: "lingecollege" in "RSG Lingecollege"
|
||||
if e_lower in d_lower:
|
||||
confidence = len(entity_name) / len(doc_title)
|
||||
if confidence >= settings.entity_linking_min_containment_ratio:
|
||||
return True, 0.85
|
||||
|
||||
# Title is substring of entity: "Google" in "Google Cloud Platform"
|
||||
if d_lower in e_lower:
|
||||
confidence = len(doc_title) / len(entity_name)
|
||||
if confidence >= settings.entity_linking_min_containment_ratio:
|
||||
return True, 0.80
|
||||
|
||||
# Strategy 3: Word-level overlap (confidence: 0.60-0.80)
|
||||
entity_tokens = set(e_lower.split())
|
||||
title_tokens = set(d_lower.split())
|
||||
|
||||
# Remove common stop words to reduce false positives
|
||||
stop_words = {'the', 'a', 'an', 'of', 'for', 'and', 'or', 'in', 'on', 'at', 'to'}
|
||||
entity_tokens -= stop_words
|
||||
title_tokens -= stop_words
|
||||
|
||||
if entity_tokens and title_tokens:
|
||||
overlap = len(entity_tokens & title_tokens)
|
||||
total = len(entity_tokens | title_tokens)
|
||||
overlap_ratio = overlap / total
|
||||
|
||||
# Require significant overlap to avoid weak matches
|
||||
if overlap_ratio >= settings.entity_linking_min_token_overlap:
|
||||
confidence = 0.75 * overlap_ratio # Scale: 0.45-0.75
|
||||
return True, confidence
|
||||
|
||||
return False, 0.0
|
||||
|
||||
|
||||
async def get_entities_with_paths(graph_service: GraphService, user: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all entities and check which ones have corresponding wiki pages.
|
||||
|
||||
Returns entities with their names, types, and paths (if they have wiki pages).
|
||||
Uses fuzzy matching to handle name variations, typos, and partial matches
|
||||
while minimizing false positives through confidence thresholds.
|
||||
|
||||
Returns entities with their names, types, paths, and match confidence.
|
||||
"""
|
||||
from src.core.multi_tenancy import get_neo4j_user_base_label
|
||||
|
||||
settings = get_settings()
|
||||
user_base_label = get_neo4j_user_base_label(user)
|
||||
|
||||
# Query to get entities and optionally join with Document nodes by matching title
|
||||
query = f"""
|
||||
// Get all entities
|
||||
# Step 1: Get all entities
|
||||
entities_query = f"""
|
||||
MATCH (e:{user_base_label})
|
||||
WHERE NOT e:Document
|
||||
|
||||
// Optionally match Document with same name (case-insensitive)
|
||||
OPTIONAL MATCH (d:Document)
|
||||
WHERE toLower(d.title) = toLower(e.name)
|
||||
|
||||
RETURN e.name as name,
|
||||
e.type as type,
|
||||
e.id as id,
|
||||
e.id as id
|
||||
"""
|
||||
|
||||
# Step 2: Get all documents
|
||||
documents_query = """
|
||||
MATCH (d:Document)
|
||||
RETURN d.title as title,
|
||||
d.path as path,
|
||||
d.page_id as doc_page_id
|
||||
d.page_id as page_id
|
||||
"""
|
||||
|
||||
try:
|
||||
results = await graph_service.neo4j.execute_query(query, {})
|
||||
return [
|
||||
{
|
||||
"name": r["name"],
|
||||
"type": r.get("type", "unknown"),
|
||||
"id": r.get("id"),
|
||||
"path": r.get("path"), # Will be None if no matching Document
|
||||
"doc_page_id": r.get("doc_page_id")
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
entities = await graph_service.neo4j.execute_query(entities_query, {})
|
||||
documents = await graph_service.neo4j.execute_query(documents_query, {})
|
||||
|
||||
# Step 3: Fuzzy match entities to documents
|
||||
results = []
|
||||
for entity in entities:
|
||||
entity_name = entity["name"]
|
||||
if not entity_name:
|
||||
continue
|
||||
|
||||
best_match = None
|
||||
best_confidence = 0.0
|
||||
|
||||
# Try to match this entity to any document
|
||||
for doc in documents:
|
||||
doc_title = doc["title"]
|
||||
if not doc_title:
|
||||
continue
|
||||
|
||||
is_match, confidence = fuzzy_match_entity_to_document(
|
||||
entity_name,
|
||||
doc_title,
|
||||
settings
|
||||
)
|
||||
|
||||
if is_match and confidence > best_confidence:
|
||||
best_match = doc
|
||||
best_confidence = confidence
|
||||
|
||||
# Only include matches above minimum confidence threshold
|
||||
if best_match and best_confidence >= settings.entity_linking_min_confidence:
|
||||
results.append({
|
||||
"name": entity_name,
|
||||
"type": entity.get("type", "unknown"),
|
||||
"id": entity.get("id"),
|
||||
"path": best_match["path"],
|
||||
"doc_page_id": best_match["page_id"],
|
||||
"match_confidence": best_confidence # NEW: track confidence
|
||||
})
|
||||
else:
|
||||
# No matching document found (or below threshold)
|
||||
results.append({
|
||||
"name": entity_name,
|
||||
"type": entity.get("type", "unknown"),
|
||||
"id": entity.get("id"),
|
||||
"path": None,
|
||||
"doc_page_id": None,
|
||||
"match_confidence": 0.0
|
||||
})
|
||||
|
||||
logger.info(f"Matched {len([r for r in results if r['path']])} entities to documents (threshold: {settings.entity_linking_min_confidence})")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get entities with paths: {e}")
|
||||
return []
|
||||
@@ -300,10 +421,9 @@ def add_entity_links_to_content(
|
||||
if len(entity_name) < 3:
|
||||
continue
|
||||
|
||||
# Create markdown link
|
||||
# Remove any "users/{username}/" prefix to get clean relative path
|
||||
clean_path = re.sub(r'^users/[^/]+/', '', entity_path)
|
||||
markdown_link = f"[{entity_name}](/{clean_path})"
|
||||
# Create markdown link with full path (including user namespace)
|
||||
# Wiki.js expects full paths like /users/jpmschweitzer/...
|
||||
markdown_link = f"[{entity_name}](/{entity_path})"
|
||||
|
||||
# Find all existing markdown links to protect them (recompute each iteration)
|
||||
link_pattern = r'\[([^\]]+)\]\([^\)]+\)'
|
||||
|
||||
Reference in New Issue
Block a user