From 79dfff68117519b5f33bd0866acc965c197c1046 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 11:26:59 +0200 Subject: [PATCH] feat: add dry-run-first purge tooling for test-tenant residue scripts/purge_test_artifacts.py removes confirmed test residue from the shared stores: - Qdrant: library_desk_llm_tester, test_user, library_desk_test_user, memories_llm_tester, volatile_llm_tester, core_ai_user_test_* and any collection containing llm_tester / llm-tester - Neo4j: nodes labelled User_Llm_Tester* (SearchQuery/Document/WebResult and sub-tenants) plus legacy llm-tester Document nodes matched by users/llm* path - Redis: *llm_tester* / *llm-tester* keys on the service DB Safety: --dry-run is the DEFAULT (prints identifiers and counts only); --execute is required for real deletion; the script exits fatally if a target rule ever matches a jpmschweitzer-namespaced identifier; the snapshot prerequisite (Qdrant snapshot API, neo4j-admin database dump) is documented in the module docstring. Connection settings come from the repo .env; secrets are never printed. Verified with a read-only --dry-run against the live stores. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + scripts/purge_test_artifacts.py | 279 ++++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 scripts/purge_test_artifacts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9993b86..48941e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Degradation signaling** - `HybridRAGResponse` now includes `source_status` (per-leg `'ok'`/`'failed'`/`'disabled'` for vector, graph, web, volatile, documents) and `degraded` (true when any enabled leg failed). Retrieval legs report errors instead of silently swallowing them; failed legs are logged at WARNING. Both fields are additive and optional, so clients that ignore them are unaffected. - **Hard-isolated test-tenant lifecycle for the test suite** - `tests/conftest.py` rewritten: the production host default (`192.168.86.149`) is gone (`TEST_HOST` env, safe `localhost` default; the API under test is the local wakeup server via `LIBRARY_DESK_URL`, never the production container on 8089). The suite is pinned to the reserved test tenant `llm_tester`; a session guard aborts the entire run if the effective tenant is `jpmschweitzer` or outside the reserved `llm_tester*` namespace. Integration tests are marked and only run with `RUN_INTEGRATION_TESTS=1` (plus a passing guard). A session-scoped teardown deletes ALL `llm_tester` artifacts created during the run — Qdrant `*_llm_tester` collections, Neo4j `User_Llm_Tester*`-labelled nodes, the `users/llm_tester` wiki subtree, and `llm_tester` Redis keys on the service DB — with hard tenant assertions before every delete. Legacy integration tests were pinned to the test tenant (no more production-namespace reads). - **Live tenant-isolation test** - `tests/test_tenant_isolation_live.py` (integration-marked, guard-gated): creates and ingests a wiki page as `llm_tester` against the local wakeup server + shared services, asserts `/query/hybrid` as `llm_tester` retrieves its own content with ZERO results from the `jpmschweitzer` tenant, and asserts a third nonexistent tenant (`llm_tester_void`, still inside the reserved namespace — nothing is ever written as `jpmschweitzer`) gets zero results entirely. Teardown removes everything it created. +- **Test-residue purge tooling** - `scripts/purge_test_artifacts.py`: dry-run by DEFAULT (`--execute` required for real deletion), targets only confirmed test residue (Qdrant `library_desk_llm_tester` / `test_user` / `library_desk_test_user` / `memories_llm_tester` / `volatile_llm_tester` / `core_ai_user_test_*` / anything containing `llm_tester`; Neo4j `User_Llm_Tester*`-labelled nodes and legacy `users/llm*` Document nodes; `llm_tester` Redis keys on the service DB), prints counts per target, hard-aborts if any target rule ever matches a `jpmschweitzer`-namespaced identifier, and documents the snapshot prerequisite (Qdrant snapshot API + `neo4j-admin database dump`) in its docstring. - **Offline unit tests** - New mock-based tests (no live services) for model-name resolution under the env collision, per-leg failure signaling, the `/stats` page-count prefix, LLM-call timeouts, and Wiki.js listing pagination. ### Fixed diff --git a/scripts/purge_test_artifacts.py b/scripts/purge_test_artifacts.py new file mode 100644 index 0000000..b6252f3 --- /dev/null +++ b/scripts/purge_test_artifacts.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +""" +Purge test-tenant residue from the SHARED production services. + +Targets ONLY the confirmed test residue left behind by earlier test runs: + + Qdrant collections + - library_desk_llm_tester, memories_llm_tester, volatile_llm_tester + - test_user, library_desk_test_user + - core_ai_user_test_* (prefix) + - anything containing "llm_tester" / "llm-tester" + + Neo4j + - all nodes carrying a label starting with "User_Llm_Tester" + (covers User_Llm_Tester, User_Llm_Tester_Document, + User_Llm_Tester_SearchQuery, User_Llm_Tester_WebResult and + sub-tenants like User_Llm_Tester_Void_*) + - legacy "llm-tester" Document nodes matched by path + (d.path STARTS WITH 'users/llm') + + Redis (service DB from settings, default DB 4) + - keys matching *llm_tester* / *llm-tester* + +SAFETY +====== +- DRY-RUN IS THE DEFAULT. Nothing is deleted unless --execute is passed. +- The script REFUSES to touch anything namespaced to the production + tenant "jpmschweitzer": every candidate identifier is checked and the + script aborts (exit 2) if a production-namespaced identifier ever + matches a target rule. +- Connection settings (hosts, credentials) come from the repo .env via + src.config.Settings; nothing is printed except identifiers and counts. + +SNAPSHOT PREREQUISITE (before any --execute run) +================================================ +Take snapshots of both stores first so an erroneous deletion can be +rolled back: + + Qdrant - full-storage snapshot via the snapshot API: + curl -X POST http://:6333/snapshots + (or per collection: + curl -X POST http://:6333/collections//snapshots) + + Neo4j - offline dump from inside the container: + docker exec neo4j-admin database dump neo4j \ + --to-path=/backups + +Only proceed with --execute after both snapshots completed successfully. + +USAGE +===== + .venv/bin/python scripts/purge_test_artifacts.py # dry run (default) + .venv/bin/python scripts/purge_test_artifacts.py --dry-run # explicit dry run + .venv/bin/python scripts/purge_test_artifacts.py --execute # REALLY delete +""" + +import argparse +import asyncio +import sys +from pathlib import Path + +# Allow running from the repo root or the scripts/ directory +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +PRODUCTION_TENANT = "jpmschweitzer" + +# Confirmed residue: exact Qdrant collection names +QDRANT_EXACT_TARGETS = { + "library_desk_llm_tester", + "memories_llm_tester", + "volatile_llm_tester", + "test_user", + "library_desk_test_user", +} +# Confirmed residue: Qdrant collection name prefixes +QDRANT_PREFIX_TARGETS = ("core_ai_user_test_",) +# Reserved test-tenant substrings (any collection containing these is residue) +QDRANT_SUBSTRING_TARGETS = ("llm_tester", "llm-tester") + +# Neo4j: tenant label prefix for the reserved test tenant +NEO4J_TEST_LABEL_PREFIX = "User_Llm_Tester" +# Neo4j: legacy Document nodes matched by wiki path (llm-tester / llm_tester) +NEO4J_TEST_DOC_PATH_PREFIX = "users/llm" + +# Redis key patterns for the reserved test tenant +REDIS_PATTERNS = ("*llm_tester*", "*llm-tester*") + + +def guard_not_production(identifier: str) -> str: + """Abort the whole run if a production-namespaced identifier shows up.""" + if PRODUCTION_TENANT.lower() in identifier.lower(): + print( + f"FATAL: target rule matched production-namespaced identifier " + f"{identifier!r} - aborting without deleting anything.", + file=sys.stderr, + ) + sys.exit(2) + return identifier + + +def qdrant_is_target(name: str) -> bool: + if name in QDRANT_EXACT_TARGETS: + return True + if any(name.startswith(p) for p in QDRANT_PREFIX_TARGETS): + return True + if any(sub in name for sub in QDRANT_SUBSTRING_TARGETS): + return True + return False + + +def purge_qdrant(settings, execute: bool) -> int: + from qdrant_client import QdrantClient + + client = QdrantClient(url=settings.qdrant_url, timeout=15) + try: + collections = [c.name for c in client.get_collections().collections] + targets = [] + for name in collections: + if qdrant_is_target(name): + guard_not_production(name) + targets.append(name) + + print(f"\nQdrant ({settings.qdrant_url}): {len(targets)} target collection(s)") + for name in sorted(targets): + try: + points = client.get_collection(name).points_count or 0 + except Exception: + points = "?" + print(f" - {name} ({points} points)") + if execute: + client.delete_collection(name) + print(f" DELETED {name}") + return len(targets) + finally: + client.close() + + +async def purge_neo4j(settings, execute: bool) -> int: + from src.clients.neo4j_client import Neo4jClient + + guard_not_production(NEO4J_TEST_LABEL_PREFIX) + guard_not_production(NEO4J_TEST_DOC_PATH_PREFIX) + + client = Neo4jClient( + uri=settings.neo4j_uri, + user=settings.neo4j_user, + password=settings.neo4j_password, + ) + try: + await client.connect() + + label_count_q = """ + MATCH (n) + WHERE any(l IN labels(n) WHERE l STARTS WITH $prefix) + RETURN count(n) AS c + """ + doc_count_q = """ + MATCH (d:Document) + WHERE d.path STARTS WITH $path_prefix + AND NOT any(l IN labels(d) WHERE l STARTS WITH $prefix) + RETURN count(d) AS c + """ + params = { + "prefix": NEO4J_TEST_LABEL_PREFIX, + "path_prefix": NEO4J_TEST_DOC_PATH_PREFIX, + } + + labelled = (await client.execute_read(label_count_q, params))[0]["c"] + legacy_docs = (await client.execute_read(doc_count_q, params))[0]["c"] + + print(f"\nNeo4j ({settings.neo4j_uri}):") + print(f" - {labelled} node(s) with label prefix {NEO4J_TEST_LABEL_PREFIX}*") + print( + f" - {legacy_docs} legacy Document node(s) with path prefix " + f"'{NEO4J_TEST_DOC_PATH_PREFIX}' (no tenant label)" + ) + + if execute: + r1 = await client.execute_write( + """ + MATCH (n) + WHERE any(l IN labels(n) WHERE l STARTS WITH $prefix) + DETACH DELETE n + RETURN count(n) AS c + """, + params, + ) + r2 = await client.execute_write( + """ + MATCH (d:Document) + WHERE d.path STARTS WITH $path_prefix + AND NOT any(l IN labels(d) WHERE l STARTS WITH $prefix) + DETACH DELETE d + RETURN count(d) AS c + """, + params, + ) + print(f" DELETED {r1[0]['c']} labelled + {r2[0]['c']} legacy nodes") + + return labelled + legacy_docs + finally: + await client.close() + + +async def purge_redis(settings, execute: bool) -> int: + import redis.asyncio as aioredis + + client = aioredis.from_url( + settings.redis_url, encoding="utf-8", decode_responses=True + ) + try: + keys: set[str] = set() + for pattern in REDIS_PATTERNS: + async for key in client.scan_iter(match=pattern, count=500): + guard_not_production(key) + keys.add(key) + + print(f"\nRedis ({settings.redis_url}): {len(keys)} target key(s)") + for key in sorted(keys): + print(f" - {key}") + if execute: + await client.delete(key) + print(f" DELETED {key}") + return len(keys) + finally: + await client.aclose() + + +async def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Purge confirmed test-tenant residue from the shared Qdrant, " + "Neo4j, and Redis stores. DRY-RUN by default; refuses anything " + "namespaced to the production tenant." + ), + epilog="Read the module docstring for the snapshot prerequisite.", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--dry-run", + action="store_true", + default=True, + help="List targets and counts without deleting (DEFAULT behaviour)", + ) + group.add_argument( + "--execute", + action="store_true", + help=( + "REALLY delete the listed targets. Take Qdrant + Neo4j snapshots " + "first (see module docstring)." + ), + ) + args = parser.parse_args() + execute = bool(args.execute) + + from src.config import get_settings + + settings = get_settings() + + mode = "EXECUTE (deleting!)" if execute else "DRY-RUN (nothing is deleted)" + print(f"purge_test_artifacts: mode = {mode}") + print(f"production tenant guard: refusing anything containing " + f"'{PRODUCTION_TENANT}'") + + totals = {} + totals["qdrant_collections"] = purge_qdrant(settings, execute) + totals["neo4j_nodes"] = await purge_neo4j(settings, execute) + totals["redis_keys"] = await purge_redis(settings, execute) + + print("\n=== Summary ===") + for target, count in totals.items(): + print(f" {target}: {count}") + if not execute: + print("\nDry run only. Re-run with --execute (after snapshots) to delete.") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main()))