From 0b346d3a57a206e4e6584c673b630fdfd450c914 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Tue, 14 Jul 2026 12:30:27 +0200 Subject: [PATCH] feat: add job cleanup loop, Scheduler task definitions, and registrar - job_cleanup_loop (src/jobs/job_manager.py): hourly in-process pass over JobManager.cleanup_expired_jobs, started at app startup and cancelled at shutdown; Redis job payloads auto-expire but set memberships do not. - docs/scheduler-tasks.md: the four production Scheduler task payloads for the deploy checklist - nightly integrity check 04:30, weekly quality report Sunday 03:00 (day_of_week=6, 0=Monday), daily Paperless orphan-cleanup 05:00 hitting the existing /maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false endpoint, and disabling test_example_task - with exact HTTP bodies (explicit user=jpmschweitzer, Authorization: Bearer ${LIBRARY_API_KEY} placeholder). - scripts/register_scheduler_tasks.py: reads SCHEDULER_URL from env, DRY-RUN BY DEFAULT (prints the exact payloads, provably contacts nothing), --execute gated and requiring LIBRARY_API_KEY to fill the placeholder. NOT executed - definitions delivered for the deploy checklist only. 9 new offline tests (loop passes/error-resilience/cancellation, payload schedules, explicit production user, placeholder, dry-run default). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L --- CHANGELOG.md | 1 + docs/scheduler-tasks.md | 159 ++++++++++++++++++ scripts/register_scheduler_tasks.py | 244 ++++++++++++++++++++++++++++ src/jobs/job_manager.py | 36 ++++ src/main.py | 21 ++- tests/test_job_plumbing.py | 121 ++++++++++++++ 6 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 docs/scheduler-tasks.md create mode 100644 scripts/register_scheduler_tasks.py create mode 100644 tests/test_job_plumbing.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a2927b5..4ec2417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `/ingest/status/{job_id}` is backed by the Redis `JobManager` (jobs are tenant-scoped; other tenants' jobs return 404). `/ingest/page`, `/ingest/batch` and `/ingest/all` now record job entries and return a `job_id`. - `/ingest/repo-status/{repository}` reports wiki page count vs indexed Document-node count under `users/{tenant}/{repository}` plus the tenant's Redis job statistics. - `/deduplicate/check` runs a tenant-scoped Qdrant similarity scan: wiki chunk pairs above the threshold (default 0.9 cosine) grouped per page pair with best score, matching chunk-pair count, and page references. Read-only. +- **Job + Scheduler task plumbing** — In-process hourly `job_cleanup_loop` (started at app startup, cancelled at shutdown) reclaims expired Redis job-set memberships (`JobManager.cleanup_expired_jobs`). `docs/scheduler-tasks.md` defines the four production Scheduler task payloads for the deploy checklist (nightly integrity 04:30, weekly quality report Sunday 03:00, daily Paperless orphan-cleanup 05:00 on the existing endpoint, and disabling `test_example_task`) with exact HTTP bodies (explicit `user=jpmschweitzer`, `${LIBRARY_API_KEY}` auth placeholder). `scripts/register_scheduler_tasks.py` reads the Scheduler API location from `SCHEDULER_URL` and registers them — dry-run by default (prints payloads), `--execute` gated and requiring `LIBRARY_API_KEY`. - **Weekly quality report** — `POST /maintenance/quality-report {user}` runs the duplicate scan, flags stale pages (not updated in N days AND ≤ M SearchQuery hits from the graph data), lists pages missing tags/description, folds in the latest integrity-check results (Redis-cached or run inline), and writes a dated report page to `users/{user}/system/quality-reports/YYYY-MM-DD` (same-day reruns update the same page — the page id is remembered in Redis because the Wiki.js listing lags page creation). Response returns the full report content + page path. Verified end-to-end against the local dev server as `llm_tester`. - **Nightly integrity check** — `POST /maintenance/integrity-check {user}` (read-only: reports, never auto-fixes) reports per tenant: wiki pages with ZERO vectors in Qdrant (silent-skip reindex victims), orphaned vectors whose wiki page no longer exists, unexpected Qdrant collections (test-tenant residue and unknown namespaces flagged; other services' collections counted as foreign), Neo4j Document nodes without wiki counterparts, plus counts and duration. The latest report is cached in Redis (30 days) so the weekly quality report can fold it in. diff --git a/docs/scheduler-tasks.md b/docs/scheduler-tasks.md new file mode 100644 index 0000000..bc02aca --- /dev/null +++ b/docs/scheduler-tasks.md @@ -0,0 +1,159 @@ +# Scheduler Task Definitions (Phase C deploy checklist) + +Production task payloads for the homelab's database-driven **Scheduler** +service. These are **definitions only** — nothing in this repo registers +them automatically. Register them as part of the deploy checklist, either +via the Scheduler UI/API or with the helper script: + +```bash +# Preview exactly what would be sent (default): +SCHEDULER_URL=http://:8090 \ + .venv/bin/python scripts/register_scheduler_tasks.py + +# Actually register/update the tasks (deploy checklist step): +SCHEDULER_URL=http://:8090 \ +LIBRARY_API_KEY= \ + .venv/bin/python scripts/register_scheduler_tasks.py --execute +``` + +Conventions: + +- All tasks call the **production** library-desk container + (`http://library-desk:8089`) with the explicit production tenant + `user=jpmschweitzer` (there is no default tenant — Phase B). +- `${LIBRARY_API_KEY}` is a placeholder for the library-desk API key + (`LIBRARY_API_KEY` in the container env). Never commit the real value. +- Schedule fields use the Scheduler's convention: `-1` = every, + `day_of_week`: `0` = Monday … `6` = Sunday. + +--- + +## 1. Nightly integrity check — 04:30 daily + +Read-only report: pages without vectors, orphaned vectors, unexpected +Qdrant collections, Document nodes without wiki pages. Caches its result +in Redis for the weekly quality report. + +```json +{ + "task_name": "library_integrity_check", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": "Nightly read-only integrity check for the library (vectors/graph/wiki/collections)", + "enabled": true, + "max_retries": 2, + "timeout_seconds": 900, + "minute": 30, + "hour": 4, + "day_of_month": -1, + "month": -1, + "day_of_week": -1, + "config": { + "method": "POST", + "url": "http://library-desk:8089/maintenance/integrity-check", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${LIBRARY_API_KEY}" + }, + "body": { + "user": "jpmschweitzer" + } + } +} +``` + +## 2. Weekly quality report — Sunday 03:00 + +Runs the duplicate scan, flags stale/metadata-poor pages, folds in the +latest integrity results, and writes the dated report page to +`users/jpmschweitzer/system/quality-reports/YYYY-MM-DD`. + +```json +{ + "task_name": "library_quality_report", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": "Weekly library quality report (dedup, stale pages, missing metadata, integrity) written to the wiki", + "enabled": true, + "max_retries": 2, + "timeout_seconds": 1800, + "minute": 0, + "hour": 3, + "day_of_month": -1, + "month": -1, + "day_of_week": 6, + "config": { + "method": "POST", + "url": "http://library-desk:8089/maintenance/quality-report", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${LIBRARY_API_KEY}" + }, + "body": { + "user": "jpmschweitzer", + "stale_days": 30, + "dedup_threshold": 0.9, + "write_page": true + } + } +} +``` + +## 3. Daily Paperless orphan cleanup — 05:00 + +Hits the **existing** cleanup endpoint (query parameters, empty body). +`dry_run=false` deletes vectors/graph nodes for documents that were +removed from Paperless-ngx. + +```json +{ + "task_name": "library_paperless_orphan_cleanup", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": "Daily cleanup of vectors/graph nodes for documents deleted from Paperless-ngx", + "enabled": true, + "max_retries": 2, + "timeout_seconds": 900, + "minute": 0, + "hour": 5, + "day_of_month": -1, + "month": -1, + "day_of_week": -1, + "config": { + "method": "POST", + "url": "http://library-desk:8089/maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false", + "headers": { + "Authorization": "Bearer ${LIBRARY_API_KEY}" + }, + "body": {} + } +} +``` + +## 4. Disable `test_example_task` + +Not a new task: the leftover example task must be **disabled** (not +deleted, so its history is preserved). + +``` +PUT ${SCHEDULER_URL}/tasks/test_example_task +Content-Type: application/json + +{"enabled": false} +``` + +--- + +## Related (already registered / in-process) + +- `knowledge_consolidation` — every 30 minutes, POST + `/consolidate/knowledge` (already registered; after the Phase C + consolidation repair its runs log `searches_processed` and + `duration_ms`, and searches are no longer consumed while the LLM is + unavailable). +- Redis job-set cleanup — runs **in-process** inside library-desk + (hourly `job_cleanup_loop` started at app startup); no Scheduler task + needed. diff --git a/scripts/register_scheduler_tasks.py b/scripts/register_scheduler_tasks.py new file mode 100644 index 0000000..c87ef99 --- /dev/null +++ b/scripts/register_scheduler_tasks.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +Register the Phase C production Scheduler tasks (deploy-checklist helper). + +Defines the four task payloads from docs/scheduler-tasks.md: +1. library_integrity_check - nightly 04:30 +2. library_quality_report - Sunday 03:00 +3. library_paperless_orphan_cleanup - daily 05:00 +4. test_example_task - DISABLED (update, not create) + +SAFETY MODEL +============ +- DRY-RUN BY DEFAULT: without --execute the script only prints the exact + payloads it would send. Nothing is contacted except (optionally) the + Scheduler health endpoint. +- --execute performs the registration: create task if absent, update it + if present, and disable test_example_task. +- The Scheduler API location comes from the environment (SCHEDULER_URL); + there is no hardcoded production default. +- The library-desk API key is embedded as the literal placeholder + ``${LIBRARY_API_KEY}`` in dry-run output. On --execute it is replaced + with the LIBRARY_API_KEY environment variable (required then, never + printed). + +Usage: + # Preview (default) + SCHEDULER_URL=http://scheduler-host:8090 \ + python scripts/register_scheduler_tasks.py + + # Register for real (deploy checklist step) + SCHEDULER_URL=http://scheduler-host:8090 LIBRARY_API_KEY=... \ + python scripts/register_scheduler_tasks.py --execute +""" + +import argparse +import copy +import json +import os +import sys + +import httpx + +API_KEY_PLACEHOLDER = "${LIBRARY_API_KEY}" +PRODUCTION_TENANT = "jpmschweitzer" +LIBRARY_BASE_URL = "http://library-desk:8089" + +#: Tasks to create-or-update (see docs/scheduler-tasks.md). +TASKS = [ + { + "task_name": "library_integrity_check", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": ( + "Nightly read-only integrity check for the library " + "(vectors/graph/wiki/collections)" + ), + "enabled": True, + "max_retries": 2, + "timeout_seconds": 900, + "minute": 30, + "hour": 4, + "day_of_month": -1, + "month": -1, + "day_of_week": -1, + "config": { + "method": "POST", + "url": f"{LIBRARY_BASE_URL}/maintenance/integrity-check", + "headers": { + "Content-Type": "application/json", + "Authorization": f"Bearer {API_KEY_PLACEHOLDER}", + }, + "body": {"user": PRODUCTION_TENANT}, + }, + }, + { + "task_name": "library_quality_report", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": ( + "Weekly library quality report (dedup, stale pages, missing " + "metadata, integrity) written to the wiki" + ), + "enabled": True, + "max_retries": 2, + "timeout_seconds": 1800, + "minute": 0, + "hour": 3, + "day_of_month": -1, + "month": -1, + "day_of_week": 6, # Sunday (0 = Monday) + "config": { + "method": "POST", + "url": f"{LIBRARY_BASE_URL}/maintenance/quality-report", + "headers": { + "Content-Type": "application/json", + "Authorization": f"Bearer {API_KEY_PLACEHOLDER}", + }, + "body": { + "user": PRODUCTION_TENANT, + "stale_days": 30, + "dedup_threshold": 0.9, + "write_page": True, + }, + }, + }, + { + "task_name": "library_paperless_orphan_cleanup", + "service": "library-desk", + "executor": "rest_api_executor", + "priority": 60, + "description": ( + "Daily cleanup of vectors/graph nodes for documents deleted " + "from Paperless-ngx" + ), + "enabled": True, + "max_retries": 2, + "timeout_seconds": 900, + "minute": 0, + "hour": 5, + "day_of_month": -1, + "month": -1, + "day_of_week": -1, + "config": { + "method": "POST", + "url": ( + f"{LIBRARY_BASE_URL}/maintenance/cleanup/paperless" + f"?user={PRODUCTION_TENANT}&dry_run=false" + ), + "headers": {"Authorization": f"Bearer {API_KEY_PLACEHOLDER}"}, + "body": {}, + }, + }, +] + +#: Existing tasks to update in place. +TASK_UPDATES = [ + {"task_name": "test_example_task", "updates": {"enabled": False}}, +] + + +def substitute_api_key(task: dict, api_key: str) -> dict: + """Return a deep copy of the task with the API-key placeholder filled in.""" + resolved = copy.deepcopy(task) + headers = resolved.get("config", {}).get("headers", {}) + for name, value in headers.items(): + if API_KEY_PLACEHOLDER in value: + headers[name] = value.replace(API_KEY_PLACEHOLDER, api_key) + return resolved + + +def dry_run(scheduler_url: str) -> None: + print("=" * 72) + print("DRY RUN - nothing will be sent. Re-run with --execute to register.") + print(f"Scheduler API: {scheduler_url or '(SCHEDULER_URL not set)'}") + print("=" * 72) + for task in TASKS: + print(f"\n--- create-or-update: POST {scheduler_url}/tasks " + f"(or PUT /tasks/{task['task_name']}) ---") + print(json.dumps(task, indent=2)) + for update in TASK_UPDATES: + print(f"\n--- update: PUT {scheduler_url}/tasks/{update['task_name']} ---") + print(json.dumps(update["updates"], indent=2)) + print("\nDry run complete: " + f"{len(TASKS)} task definition(s), {len(TASK_UPDATES)} update(s).") + + +def execute(scheduler_url: str, api_key: str) -> int: + failures = 0 + with httpx.Client(base_url=scheduler_url, timeout=30.0) as client: + health = client.get("/health") + if health.status_code != 200: + print(f"ERROR: Scheduler health check failed: {health.status_code}") + return 1 + + for task in TASKS: + name = task["task_name"] + payload = substitute_api_key(task, api_key) + exists = client.get(f"/tasks/{name}").status_code == 200 + if exists: + resp = client.put(f"/tasks/{name}", json=payload) + action = "updated" + else: + resp = client.post("/tasks", json=payload) + action = "created" + if resp.status_code == 200: + print(f"[ok] {action} {name}") + else: + failures += 1 + print(f"[FAIL] {action} {name}: {resp.status_code} {resp.text[:200]}") + + for update in TASK_UPDATES: + name = update["task_name"] + if client.get(f"/tasks/{name}").status_code != 200: + print(f"[skip] {name} does not exist - nothing to disable") + continue + resp = client.put(f"/tasks/{name}", json=update["updates"]) + if resp.status_code == 200: + print(f"[ok] updated {name}: {update['updates']}") + else: + failures += 1 + print(f"[FAIL] update {name}: {resp.status_code} {resp.text[:200]}") + + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Register library-desk production Scheduler tasks " + "(dry-run by default)" + ) + parser.add_argument( + "--execute", + action="store_true", + help="Actually register/update the tasks (default: dry-run print only)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Explicitly print payloads without sending (the default behavior)", + ) + args = parser.parse_args() + + scheduler_url = os.environ.get("SCHEDULER_URL", "").rstrip("/") + + if not args.execute or args.dry_run: + dry_run(scheduler_url) + return 0 + + if not scheduler_url: + print("ERROR: SCHEDULER_URL must be set for --execute") + return 1 + api_key = os.environ.get("LIBRARY_API_KEY", "") + if not api_key: + print("ERROR: LIBRARY_API_KEY must be set for --execute " + "(it fills the Authorization header placeholder)") + return 1 + + return execute(scheduler_url, api_key) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/jobs/job_manager.py b/src/jobs/job_manager.py index 895ac55..73523dd 100644 --- a/src/jobs/job_manager.py +++ b/src/jobs/job_manager.py @@ -9,6 +9,7 @@ Provides background job management with: - User-scoped job queries """ +import asyncio import redis.asyncio as redis import json import uuid @@ -424,3 +425,38 @@ class JobManager: stats[status] += 1 return stats + + +async def job_cleanup_loop( + job_manager: JobManager, + interval_seconds: float = 3600, + max_iterations: Optional[int] = None +) -> int: + """ + Periodically clean up expired job-set memberships. + + Redis auto-expires the job payloads (24h TTL) but set memberships + (library:active_jobs, library:user_jobs:{user}) need manual cleanup. + Started as an in-process background task at application startup. + + Args: + job_manager: JobManager whose cleanup_expired_jobs is invoked + interval_seconds: Sleep between cleanup passes (default hourly) + max_iterations: Stop after N passes (None = run forever; used by tests) + + Returns: + Number of completed cleanup passes (only reachable with max_iterations) + """ + iterations = 0 + while max_iterations is None or iterations < max_iterations: + try: + await asyncio.sleep(interval_seconds) + await job_manager.cleanup_expired_jobs() + except asyncio.CancelledError: + logger.info("Job cleanup loop cancelled") + raise + except Exception as e: + # Never let a transient Redis error kill the loop + logger.error(f"Job cleanup pass failed: {e}") + iterations += 1 + return iterations diff --git a/src/main.py b/src/main.py index c1f3720..7529bf9 100644 --- a/src/main.py +++ b/src/main.py @@ -598,7 +598,10 @@ async def check_duplicates( @app.on_event("startup") async def startup_event(): """Initialize connections and resources on startup.""" - from src.core.dependencies import startup_clients + import asyncio + + from src.core.dependencies import startup_clients, get_job_manager + from src.jobs.job_manager import job_cleanup_loop from src.services.wiki_change_listener import WikiChangeListener settings = get_settings() @@ -613,6 +616,13 @@ async def startup_event(): # Initialize all service clients await startup_clients() + # Hourly in-process cleanup of expired Redis job-set memberships + # (job payloads auto-expire via TTL; set memberships do not) + app.state.job_cleanup_task = asyncio.create_task( + job_cleanup_loop(get_job_manager(), interval_seconds=3600) + ) + logger.info("Job cleanup loop started (hourly)") + # Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN) # This enables automatic processing of user-edited pages try: @@ -633,6 +643,15 @@ async def shutdown_event(): logger.info("Shutting down Library Desk API") + # Stop the job cleanup loop + if hasattr(app.state, "job_cleanup_task"): + app.state.job_cleanup_task.cancel() + try: + await app.state.job_cleanup_task + except Exception: + pass + logger.info("Job cleanup loop stopped") + # Stop Wiki.js change listener if running if hasattr(app.state, "wiki_listener"): try: diff --git a/tests/test_job_plumbing.py b/tests/test_job_plumbing.py new file mode 100644 index 0000000..bd69988 --- /dev/null +++ b/tests/test_job_plumbing.py @@ -0,0 +1,121 @@ +""" +Offline unit tests for job + Scheduler task plumbing (Phase C item 5). + +Covers: +- job_cleanup_loop: invokes cleanup_expired_jobs per pass, survives + transient errors, honors cancellation +- register_scheduler_tasks.py: dry-run default, payload contents + (explicit production user, auth placeholder, schedules) +""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from src.jobs.job_manager import job_cleanup_loop + + +class TestJobCleanupLoop: + @pytest.mark.asyncio + async def test_invokes_cleanup_each_pass(self): + manager = AsyncMock() + + passes = await job_cleanup_loop(manager, interval_seconds=0, max_iterations=3) + + assert passes == 3 + assert manager.cleanup_expired_jobs.await_count == 3 + + @pytest.mark.asyncio + async def test_transient_error_does_not_kill_loop(self): + manager = AsyncMock() + manager.cleanup_expired_jobs = AsyncMock( + side_effect=[RuntimeError("redis hiccup"), None] + ) + + passes = await job_cleanup_loop(manager, interval_seconds=0, max_iterations=2) + + assert passes == 2 + assert manager.cleanup_expired_jobs.await_count == 2 + + @pytest.mark.asyncio + async def test_cancellation_stops_loop(self): + manager = AsyncMock() + + task = asyncio.create_task(job_cleanup_loop(manager, interval_seconds=60)) + await asyncio.sleep(0) # let it start sleeping + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +class TestSchedulerTaskDefinitions: + def _load_module(self): + import importlib + return importlib.import_module("scripts.register_scheduler_tasks") + + def test_four_production_payloads_defined(self): + mod = self._load_module() + names = {t["task_name"] for t in mod.TASKS} + assert names == { + "library_integrity_check", + "library_quality_report", + "library_paperless_orphan_cleanup", + } + updates = {u["task_name"]: u["updates"] for u in mod.TASK_UPDATES} + assert updates == {"test_example_task": {"enabled": False}} + + def test_schedules(self): + mod = self._load_module() + by_name = {t["task_name"]: t for t in mod.TASKS} + + integrity = by_name["library_integrity_check"] + assert (integrity["hour"], integrity["minute"], integrity["day_of_week"]) == (4, 30, -1) + + quality = by_name["library_quality_report"] + # Sunday 03:00 (Scheduler: 0 = Monday .. 6 = Sunday) + assert (quality["hour"], quality["minute"], quality["day_of_week"]) == (3, 0, 6) + + paperless = by_name["library_paperless_orphan_cleanup"] + assert (paperless["hour"], paperless["minute"], paperless["day_of_week"]) == (5, 0, -1) + + def test_payloads_use_explicit_production_user_and_placeholder(self): + mod = self._load_module() + for task in mod.TASKS: + config = task["config"] + auth = config["headers"]["Authorization"] + assert auth == f"Bearer {mod.API_KEY_PLACEHOLDER}" + # Explicit production tenant in body or query string (Phase B) + body_user = config.get("body", {}).get("user") + assert body_user == "jpmschweitzer" or "user=jpmschweitzer" in config["url"] + + def test_paperless_task_hits_existing_endpoint(self): + mod = self._load_module() + task = next(t for t in mod.TASKS + if t["task_name"] == "library_paperless_orphan_cleanup") + assert "/maintenance/cleanup/paperless" in task["config"]["url"] + assert "dry_run=false" in task["config"]["url"] + + def test_substitute_api_key_replaces_placeholder_without_mutating(self): + mod = self._load_module() + original = mod.TASKS[0] + resolved = mod.substitute_api_key(original, "sekret") + assert resolved["config"]["headers"]["Authorization"] == "Bearer sekret" + # The module-level definition keeps the placeholder + assert mod.API_KEY_PLACEHOLDER in original["config"]["headers"]["Authorization"] + + def test_dry_run_is_default_and_sends_nothing(self, capsys, monkeypatch): + mod = self._load_module() + monkeypatch.setattr("sys.argv", ["register_scheduler_tasks.py"]) + monkeypatch.setenv("SCHEDULER_URL", "http://scheduler.test:8090") + + def _boom(*args, **kwargs): # any HTTP client construction = failure + raise AssertionError("dry-run must not contact the Scheduler") + + monkeypatch.setattr(mod.httpx, "Client", _boom) + + assert mod.main() == 0 + out = capsys.readouterr().out + assert "DRY RUN" in out + assert "library_integrity_check" in out + assert mod.API_KEY_PLACEHOLDER in out # placeholder, never a real key