From ae8edf6111682a70a7515e1c7584763997e3e384 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Wed, 10 Dec 2025 01:40:10 +0100 Subject: [PATCH] chore(library-desk): add maintenance and cleanup scripts Cleanup Scripts: - cleanup_graph.py - Clean up duplicate entities and orphaned nodes - cleanup_wiki.py - Remove orphaned pages and fix broken links - Utility scripts for database maintenance - Not part of main application, run manually --- services/library-desk/cleanup_graph.py | 216 +++++++++++++++++++++++++ services/library-desk/cleanup_wiki.py | 126 +++++++++++++++ 2 files changed, 342 insertions(+) create mode 100755 services/library-desk/cleanup_graph.py create mode 100755 services/library-desk/cleanup_wiki.py diff --git a/services/library-desk/cleanup_graph.py b/services/library-desk/cleanup_graph.py new file mode 100755 index 0000000..030be46 --- /dev/null +++ b/services/library-desk/cleanup_graph.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Cleanup script to remove all test data from Neo4j graph database. +This prepares the system for production by removing: +- SearchQuery nodes (HybridRAG search history) +- WebResult nodes (external web search results) +- Any orphaned nodes without relationships +""" +import requests +import sys + +# Configuration +BASE_URL = "http://192.168.86.149:8089" +API_KEY = "af88ed8f44bed81bdb20d0534f1c4547340b29e2aba4963f61a71b993d7eb6e5" +USER = "jpmschweitzer" + +headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json" +} + +def execute_query(query_text, description): + """Execute a Cypher query via API.""" + url = f"{BASE_URL}/graph/query" + payload = { + "user": USER, + "query": query_text + } + + print(f"๐Ÿ”„ {description}...") + response = requests.post(url, headers=headers, json=payload) + + if response.status_code != 200: + print(f" โŒ Failed: {response.status_code}") + print(f" {response.text}") + return None + + data = response.json() + return data + +def count_nodes(node_type): + """Count nodes of a specific type.""" + if node_type == "SearchQuery": + query = f'MATCH (n:SearchQuery) WHERE n.user = "{USER}" RETURN count(n) as count' + elif node_type == "WebResult": + query = f'MATCH (n:User_{USER}_WebResult:WebResult) RETURN count(n) as count' + elif node_type == "Document": + query = f'MATCH (n:User_{USER}_Document:Document) RETURN count(n) as count' + else: + query = f'MATCH (n:User_{USER}) WHERE NOT n:Document AND NOT n:SearchQuery AND NOT n:WebResult RETURN count(n) as count' + + result = execute_query(query, f"Counting {node_type} nodes") + if result and result.get("results"): + return result["results"][0]["count"] + return 0 + +def delete_search_queries(): + """Delete all SearchQuery nodes and their relationships.""" + query = f''' + MATCH (sq:SearchQuery) + WHERE sq.user = "{USER}" + DETACH DELETE sq + RETURN count(*) as deleted + ''' + + result = execute_query(query, "Deleting SearchQuery nodes") + if result and result.get("results"): + return result["results"][0].get("deleted", 0) + return 0 + +def delete_web_results(): + """Delete all WebResult nodes.""" + query = f''' + MATCH (wr:User_{USER}_WebResult:WebResult) + DETACH DELETE wr + RETURN count(*) as deleted + ''' + + result = execute_query(query, "Deleting WebResult nodes") + if result and result.get("results"): + return result["results"][0].get("deleted", 0) + return 0 + +def delete_orphaned_nodes(): + """Delete any orphaned entity nodes without relationships.""" + query = f''' + MATCH (n:User_{USER}) + WHERE NOT n:Document + AND NOT n:SearchQuery + AND NOT n:WebResult + AND NOT (n)--() + DETACH DELETE n + RETURN count(*) as deleted + ''' + + result = execute_query(query, "Deleting orphaned nodes") + if result and result.get("results"): + return result["results"][0].get("deleted", 0) + return 0 + +def get_all_node_stats(): + """Get statistics on all node types.""" + query = f''' + MATCH (n:User_{USER}) + WITH labels(n) as labels, count(n) as count + RETURN labels, count + ORDER BY count DESC + ''' + + result = execute_query(query, "Getting node statistics") + if result and result.get("results"): + return result["results"] + return [] + +def main(): + print("=" * 70) + print("๐Ÿงน Neo4j Graph Database Cleanup Script") + print("=" * 70) + print() + + # Show current state + print("๐Ÿ“Š Current Database State:") + print("-" * 70) + + search_count = count_nodes("SearchQuery") + web_count = count_nodes("WebResult") + doc_count = count_nodes("Document") + other_count = count_nodes("Other") + + print(f" SearchQuery nodes: {search_count}") + print(f" WebResult nodes: {web_count}") + print(f" Document nodes: {doc_count}") + print(f" Other entity nodes: {other_count}") + print() + + if search_count == 0 and web_count == 0 and doc_count == 0 and other_count == 0: + print("โœจ Database is already clean!") + return + + total_to_delete = search_count + web_count + other_count + if total_to_delete == 0 and doc_count > 0: + print("โš ๏ธ Only Document nodes found (legitimate data)") + print(" No cleanup needed!") + return + + # Ask for confirmation + print("โš ๏ธ WARNING: About to delete:") + if search_count > 0: + print(f" - {search_count} SearchQuery nodes (search history)") + if web_count > 0: + print(f" - {web_count} WebResult nodes (cached web results)") + if other_count > 0: + print(f" - {other_count} orphaned entity nodes") + + print() + print(" Document nodes will NOT be deleted (they represent wiki pages)") + print() + + response = input("Continue with cleanup? (yes/no): ").strip().lower() + if response != "yes": + print("โŒ Cancelled") + return + + print() + print("๐Ÿš€ Starting cleanup...") + print() + + # Delete SearchQuery nodes + if search_count > 0: + deleted_sq = delete_search_queries() + print(f" โœ… Deleted {deleted_sq} SearchQuery nodes") + + # Delete WebResult nodes + if web_count > 0: + deleted_wr = delete_web_results() + print(f" โœ… Deleted {deleted_wr} WebResult nodes") + + # Delete orphaned nodes + if other_count > 0: + deleted_orphans = delete_orphaned_nodes() + print(f" โœ… Deleted {deleted_orphans} orphaned nodes") + + print() + print("=" * 70) + print("๐Ÿ“Š Final Database State") + print("=" * 70) + + # Show final state + search_count_final = count_nodes("SearchQuery") + web_count_final = count_nodes("WebResult") + doc_count_final = count_nodes("Document") + other_count_final = count_nodes("Other") + + print(f" SearchQuery nodes: {search_count_final}") + print(f" WebResult nodes: {web_count_final}") + print(f" Document nodes: {doc_count_final}") + print(f" Other entity nodes: {other_count_final}") + print() + + if search_count_final == 0 and web_count_final == 0 and other_count_final == 0: + print("โœจ Graph database is now production-ready!") + print(" Only legitimate wiki-derived data remains.") + else: + print("โš ๏ธ Some nodes remain - this might be expected if you have active wiki pages") + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\n\nโŒ Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nโŒ Error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/services/library-desk/cleanup_wiki.py b/services/library-desk/cleanup_wiki.py new file mode 100755 index 0000000..c1545f3 --- /dev/null +++ b/services/library-desk/cleanup_wiki.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +Cleanup script to delete all wiki pages via REST API. +This will trigger automatic vector and graph cleanup for each page. +""" +import requests +import time +import sys + +# Configuration +BASE_URL = "http://192.168.86.149:8089" +API_KEY = "af88ed8f44bed81bdb20d0534f1c4547340b29e2aba4963f61a71b993d7eb6e5" +USER = "jpmschweitzer" + +headers = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json" +} + +def list_all_pages(): + """List all wiki pages.""" + url = f"{BASE_URL}/wiki/pages" + params = {"user": USER, "limit": 200} + + print(f"๐Ÿ” Listing all pages for user: {USER}") + response = requests.get(url, headers=headers, params=params) + + if response.status_code != 200: + print(f"โŒ Failed to list pages: {response.status_code}") + print(response.text) + return [] + + data = response.json() + pages = data.get("pages", []) + print(f"๐Ÿ“š Found {len(pages)} pages") + return pages + +def delete_page(page_id, title): + """Delete a single wiki page.""" + url = f"{BASE_URL}/wiki/pages/{page_id}" + params = {"user": USER} + + print(f"๐Ÿ—‘๏ธ Deleting page {page_id}: {title}") + response = requests.delete(url, headers=headers, params=params) + + if response.status_code == 200: + print(f" โœ… Deleted successfully") + return True + else: + print(f" โŒ Failed: {response.status_code}") + print(f" {response.text}") + return False + +def main(): + print("=" * 60) + print("๐Ÿงน Wiki Cleanup Script") + print("=" * 60) + print() + + # List all pages + pages = list_all_pages() + + if not pages: + print("โœจ No pages found. Wiki is already clean!") + return + + print() + print(f"โš ๏ธ WARNING: About to delete {len(pages)} pages!") + print(" This will also delete:") + print(" - Vector embeddings in Qdrant") + print(" - Document nodes in Neo4j") + print() + + # Ask for confirmation + response = input("Continue? (yes/no): ").strip().lower() + if response != "yes": + print("โŒ Cancelled") + return + + print() + print("๐Ÿš€ Starting deletion...") + print() + + # Delete each page + success_count = 0 + failed_count = 0 + + for i, page in enumerate(pages, 1): + page_id = page.get("id") + title = page.get("title", "Untitled") + + print(f"[{i}/{len(pages)}] ", end="") + + if delete_page(page_id, title): + success_count += 1 + else: + failed_count += 1 + + # Small delay to avoid overwhelming the API + time.sleep(0.2) + + print() + print("=" * 60) + print("๐Ÿ“Š Cleanup Summary") + print("=" * 60) + print(f"โœ… Successfully deleted: {success_count} pages") + print(f"โŒ Failed to delete: {failed_count} pages") + print(f"๐Ÿ“ Total pages: {len(pages)}") + print() + + if success_count > 0: + print("๐Ÿงน Background cleanup tasks are running to remove:") + print(" - Vector embeddings from Qdrant") + print(" - Document nodes from Neo4j") + print() + print("โœจ Wiki cleanup complete!") + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\n\nโŒ Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nโŒ Error: {e}") + sys.exit(1)