chore(library-desk): add maintenance and cleanup scripts
Cleanup Scripts: - cleanup_graph.py - Clean up duplicate entities and orphaned nodes - cleanup_wiki.py - Remove orphaned pages and fix broken links - Utility scripts for database maintenance - Not part of main application, run manually
This commit is contained in:
Executable
+216
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cleanup script to remove all test data from Neo4j graph database.
|
||||
This prepares the system for production by removing:
|
||||
- SearchQuery nodes (HybridRAG search history)
|
||||
- WebResult nodes (external web search results)
|
||||
- Any orphaned nodes without relationships
|
||||
"""
|
||||
import requests
|
||||
import sys
|
||||
|
||||
# Configuration
|
||||
BASE_URL = "http://192.168.86.149:8089"
|
||||
API_KEY = "af88ed8f44bed81bdb20d0534f1c4547340b29e2aba4963f61a71b993d7eb6e5"
|
||||
USER = "jpmschweitzer"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {API_KEY}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
def execute_query(query_text, description):
|
||||
"""Execute a Cypher query via API."""
|
||||
url = f"{BASE_URL}/graph/query"
|
||||
payload = {
|
||||
"user": USER,
|
||||
"query": query_text
|
||||
}
|
||||
|
||||
print(f"🔄 {description}...")
|
||||
response = requests.post(url, headers=headers, json=payload)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f" ❌ Failed: {response.status_code}")
|
||||
print(f" {response.text}")
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
def count_nodes(node_type):
|
||||
"""Count nodes of a specific type."""
|
||||
if node_type == "SearchQuery":
|
||||
query = f'MATCH (n:SearchQuery) WHERE n.user = "{USER}" RETURN count(n) as count'
|
||||
elif node_type == "WebResult":
|
||||
query = f'MATCH (n:User_{USER}_WebResult:WebResult) RETURN count(n) as count'
|
||||
elif node_type == "Document":
|
||||
query = f'MATCH (n:User_{USER}_Document:Document) RETURN count(n) as count'
|
||||
else:
|
||||
query = f'MATCH (n:User_{USER}) WHERE NOT n:Document AND NOT n:SearchQuery AND NOT n:WebResult RETURN count(n) as count'
|
||||
|
||||
result = execute_query(query, f"Counting {node_type} nodes")
|
||||
if result and result.get("results"):
|
||||
return result["results"][0]["count"]
|
||||
return 0
|
||||
|
||||
def delete_search_queries():
|
||||
"""Delete all SearchQuery nodes and their relationships."""
|
||||
query = f'''
|
||||
MATCH (sq:SearchQuery)
|
||||
WHERE sq.user = "{USER}"
|
||||
DETACH DELETE sq
|
||||
RETURN count(*) as deleted
|
||||
'''
|
||||
|
||||
result = execute_query(query, "Deleting SearchQuery nodes")
|
||||
if result and result.get("results"):
|
||||
return result["results"][0].get("deleted", 0)
|
||||
return 0
|
||||
|
||||
def delete_web_results():
|
||||
"""Delete all WebResult nodes."""
|
||||
query = f'''
|
||||
MATCH (wr:User_{USER}_WebResult:WebResult)
|
||||
DETACH DELETE wr
|
||||
RETURN count(*) as deleted
|
||||
'''
|
||||
|
||||
result = execute_query(query, "Deleting WebResult nodes")
|
||||
if result and result.get("results"):
|
||||
return result["results"][0].get("deleted", 0)
|
||||
return 0
|
||||
|
||||
def delete_orphaned_nodes():
|
||||
"""Delete any orphaned entity nodes without relationships."""
|
||||
query = f'''
|
||||
MATCH (n:User_{USER})
|
||||
WHERE NOT n:Document
|
||||
AND NOT n:SearchQuery
|
||||
AND NOT n:WebResult
|
||||
AND NOT (n)--()
|
||||
DETACH DELETE n
|
||||
RETURN count(*) as deleted
|
||||
'''
|
||||
|
||||
result = execute_query(query, "Deleting orphaned nodes")
|
||||
if result and result.get("results"):
|
||||
return result["results"][0].get("deleted", 0)
|
||||
return 0
|
||||
|
||||
def get_all_node_stats():
|
||||
"""Get statistics on all node types."""
|
||||
query = f'''
|
||||
MATCH (n:User_{USER})
|
||||
WITH labels(n) as labels, count(n) as count
|
||||
RETURN labels, count
|
||||
ORDER BY count DESC
|
||||
'''
|
||||
|
||||
result = execute_query(query, "Getting node statistics")
|
||||
if result and result.get("results"):
|
||||
return result["results"]
|
||||
return []
|
||||
|
||||
def main():
|
||||
print("=" * 70)
|
||||
print("🧹 Neo4j Graph Database Cleanup Script")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Show current state
|
||||
print("📊 Current Database State:")
|
||||
print("-" * 70)
|
||||
|
||||
search_count = count_nodes("SearchQuery")
|
||||
web_count = count_nodes("WebResult")
|
||||
doc_count = count_nodes("Document")
|
||||
other_count = count_nodes("Other")
|
||||
|
||||
print(f" SearchQuery nodes: {search_count}")
|
||||
print(f" WebResult nodes: {web_count}")
|
||||
print(f" Document nodes: {doc_count}")
|
||||
print(f" Other entity nodes: {other_count}")
|
||||
print()
|
||||
|
||||
if search_count == 0 and web_count == 0 and doc_count == 0 and other_count == 0:
|
||||
print("✨ Database is already clean!")
|
||||
return
|
||||
|
||||
total_to_delete = search_count + web_count + other_count
|
||||
if total_to_delete == 0 and doc_count > 0:
|
||||
print("⚠️ Only Document nodes found (legitimate data)")
|
||||
print(" No cleanup needed!")
|
||||
return
|
||||
|
||||
# Ask for confirmation
|
||||
print("⚠️ WARNING: About to delete:")
|
||||
if search_count > 0:
|
||||
print(f" - {search_count} SearchQuery nodes (search history)")
|
||||
if web_count > 0:
|
||||
print(f" - {web_count} WebResult nodes (cached web results)")
|
||||
if other_count > 0:
|
||||
print(f" - {other_count} orphaned entity nodes")
|
||||
|
||||
print()
|
||||
print(" Document nodes will NOT be deleted (they represent wiki pages)")
|
||||
print()
|
||||
|
||||
response = input("Continue with cleanup? (yes/no): ").strip().lower()
|
||||
if response != "yes":
|
||||
print("❌ Cancelled")
|
||||
return
|
||||
|
||||
print()
|
||||
print("🚀 Starting cleanup...")
|
||||
print()
|
||||
|
||||
# Delete SearchQuery nodes
|
||||
if search_count > 0:
|
||||
deleted_sq = delete_search_queries()
|
||||
print(f" ✅ Deleted {deleted_sq} SearchQuery nodes")
|
||||
|
||||
# Delete WebResult nodes
|
||||
if web_count > 0:
|
||||
deleted_wr = delete_web_results()
|
||||
print(f" ✅ Deleted {deleted_wr} WebResult nodes")
|
||||
|
||||
# Delete orphaned nodes
|
||||
if other_count > 0:
|
||||
deleted_orphans = delete_orphaned_nodes()
|
||||
print(f" ✅ Deleted {deleted_orphans} orphaned nodes")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("📊 Final Database State")
|
||||
print("=" * 70)
|
||||
|
||||
# Show final state
|
||||
search_count_final = count_nodes("SearchQuery")
|
||||
web_count_final = count_nodes("WebResult")
|
||||
doc_count_final = count_nodes("Document")
|
||||
other_count_final = count_nodes("Other")
|
||||
|
||||
print(f" SearchQuery nodes: {search_count_final}")
|
||||
print(f" WebResult nodes: {web_count_final}")
|
||||
print(f" Document nodes: {doc_count_final}")
|
||||
print(f" Other entity nodes: {other_count_final}")
|
||||
print()
|
||||
|
||||
if search_count_final == 0 and web_count_final == 0 and other_count_final == 0:
|
||||
print("✨ Graph database is now production-ready!")
|
||||
print(" Only legitimate wiki-derived data remains.")
|
||||
else:
|
||||
print("⚠️ Some nodes remain - this might be expected if you have active wiki pages")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n❌ Interrupted by user")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cleanup script to delete all wiki pages via REST API.
|
||||
This will trigger automatic vector and graph cleanup for each page.
|
||||
"""
|
||||
import requests
|
||||
import time
|
||||
import sys
|
||||
|
||||
# Configuration
|
||||
BASE_URL = "http://192.168.86.149:8089"
|
||||
API_KEY = "af88ed8f44bed81bdb20d0534f1c4547340b29e2aba4963f61a71b993d7eb6e5"
|
||||
USER = "jpmschweitzer"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {API_KEY}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
def list_all_pages():
|
||||
"""List all wiki pages."""
|
||||
url = f"{BASE_URL}/wiki/pages"
|
||||
params = {"user": USER, "limit": 200}
|
||||
|
||||
print(f"🔍 Listing all pages for user: {USER}")
|
||||
response = requests.get(url, headers=headers, params=params)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ Failed to list pages: {response.status_code}")
|
||||
print(response.text)
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
pages = data.get("pages", [])
|
||||
print(f"📚 Found {len(pages)} pages")
|
||||
return pages
|
||||
|
||||
def delete_page(page_id, title):
|
||||
"""Delete a single wiki page."""
|
||||
url = f"{BASE_URL}/wiki/pages/{page_id}"
|
||||
params = {"user": USER}
|
||||
|
||||
print(f"🗑️ Deleting page {page_id}: {title}")
|
||||
response = requests.delete(url, headers=headers, params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
print(f" ✅ Deleted successfully")
|
||||
return True
|
||||
else:
|
||||
print(f" ❌ Failed: {response.status_code}")
|
||||
print(f" {response.text}")
|
||||
return False
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("🧹 Wiki Cleanup Script")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# List all pages
|
||||
pages = list_all_pages()
|
||||
|
||||
if not pages:
|
||||
print("✨ No pages found. Wiki is already clean!")
|
||||
return
|
||||
|
||||
print()
|
||||
print(f"⚠️ WARNING: About to delete {len(pages)} pages!")
|
||||
print(" This will also delete:")
|
||||
print(" - Vector embeddings in Qdrant")
|
||||
print(" - Document nodes in Neo4j")
|
||||
print()
|
||||
|
||||
# Ask for confirmation
|
||||
response = input("Continue? (yes/no): ").strip().lower()
|
||||
if response != "yes":
|
||||
print("❌ Cancelled")
|
||||
return
|
||||
|
||||
print()
|
||||
print("🚀 Starting deletion...")
|
||||
print()
|
||||
|
||||
# Delete each page
|
||||
success_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for i, page in enumerate(pages, 1):
|
||||
page_id = page.get("id")
|
||||
title = page.get("title", "Untitled")
|
||||
|
||||
print(f"[{i}/{len(pages)}] ", end="")
|
||||
|
||||
if delete_page(page_id, title):
|
||||
success_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
# Small delay to avoid overwhelming the API
|
||||
time.sleep(0.2)
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("📊 Cleanup Summary")
|
||||
print("=" * 60)
|
||||
print(f"✅ Successfully deleted: {success_count} pages")
|
||||
print(f"❌ Failed to delete: {failed_count} pages")
|
||||
print(f"📝 Total pages: {len(pages)}")
|
||||
print()
|
||||
|
||||
if success_count > 0:
|
||||
print("🧹 Background cleanup tasks are running to remove:")
|
||||
print(" - Vector embeddings from Qdrant")
|
||||
print(" - Document nodes from Neo4j")
|
||||
print()
|
||||
print("✨ Wiki cleanup complete!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n❌ Interrupted by user")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Error: {e}")
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user