feat: add job cleanup loop, Scheduler task definitions, and registrar

- job_cleanup_loop (src/jobs/job_manager.py): hourly in-process pass over
  JobManager.cleanup_expired_jobs, started at app startup and cancelled
  at shutdown; Redis job payloads auto-expire but set memberships do not.
- docs/scheduler-tasks.md: the four production Scheduler task payloads
  for the deploy checklist - nightly integrity check 04:30, weekly
  quality report Sunday 03:00 (day_of_week=6, 0=Monday), daily Paperless
  orphan-cleanup 05:00 hitting the existing
  /maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false
  endpoint, and disabling test_example_task - with exact HTTP bodies
  (explicit user=jpmschweitzer, Authorization: Bearer ${LIBRARY_API_KEY}
  placeholder).
- scripts/register_scheduler_tasks.py: reads SCHEDULER_URL from env,
  DRY-RUN BY DEFAULT (prints the exact payloads, provably contacts
  nothing), --execute gated and requiring LIBRARY_API_KEY to fill the
  placeholder. NOT executed - definitions delivered for the deploy
  checklist only.

9 new offline tests (loop passes/error-resilience/cancellation, payload
schedules, explicit production user, placeholder, dry-run default).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L

scripts/register_scheduler_tasks.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""
+Register the Phase C production Scheduler tasks (deploy-checklist helper).
+
+Defines the four task payloads from docs/scheduler-tasks.md:
+1. library_integrity_check          - nightly 04:30
+2. library_quality_report           - Sunday 03:00
+3. library_paperless_orphan_cleanup - daily 05:00
+4. test_example_task                - DISABLED (update, not create)
+
+SAFETY MODEL
+============
+- DRY-RUN BY DEFAULT: without --execute the script only prints the exact
+  payloads it would send. Nothing is contacted except (optionally) the
+  Scheduler health endpoint.
+- --execute performs the registration: create task if absent, update it
+  if present, and disable test_example_task.
+- The Scheduler API location comes from the environment (SCHEDULER_URL);
+  there is no hardcoded production default.
+- The library-desk API key is embedded as the literal placeholder
+  ``${LIBRARY_API_KEY}`` in dry-run output. On --execute it is replaced
+  with the LIBRARY_API_KEY environment variable (required then, never
+  printed).
+
+Usage:
+    # Preview (default)
+    SCHEDULER_URL=http://scheduler-host:8090 \
+        python scripts/register_scheduler_tasks.py
+
+    # Register for real (deploy checklist step)
+    SCHEDULER_URL=http://scheduler-host:8090 LIBRARY_API_KEY=... \
+        python scripts/register_scheduler_tasks.py --execute
+"""
+
+import argparse
+import copy
+import json
+import os
+import sys
+
+import httpx
+
+API_KEY_PLACEHOLDER = "${LIBRARY_API_KEY}"
+PRODUCTION_TENANT = "jpmschweitzer"
+LIBRARY_BASE_URL = "http://library-desk:8089"
+
+#: Tasks to create-or-update (see docs/scheduler-tasks.md).
+TASKS = [
+    {
+        "task_name": "library_integrity_check",
+        "service": "library-desk",
+        "executor": "rest_api_executor",
+        "priority": 60,
+        "description": (
+            "Nightly read-only integrity check for the library "
+            "(vectors/graph/wiki/collections)"
+        ),
+        "enabled": True,
+        "max_retries": 2,
+        "timeout_seconds": 900,
+        "minute": 30,
+        "hour": 4,
+        "day_of_month": -1,
+        "month": -1,
+        "day_of_week": -1,
+        "config": {
+            "method": "POST",
+            "url": f"{LIBRARY_BASE_URL}/maintenance/integrity-check",
+            "headers": {
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {API_KEY_PLACEHOLDER}",
+            },
+            "body": {"user": PRODUCTION_TENANT},
+        },
+    },
+    {
+        "task_name": "library_quality_report",
+        "service": "library-desk",
+        "executor": "rest_api_executor",
+        "priority": 60,
+        "description": (
+            "Weekly library quality report (dedup, stale pages, missing "
+            "metadata, integrity) written to the wiki"
+        ),
+        "enabled": True,
+        "max_retries": 2,
+        "timeout_seconds": 1800,
+        "minute": 0,
+        "hour": 3,
+        "day_of_month": -1,
+        "month": -1,
+        "day_of_week": 6,  # Sunday (0 = Monday)
+        "config": {
+            "method": "POST",
+            "url": f"{LIBRARY_BASE_URL}/maintenance/quality-report",
+            "headers": {
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {API_KEY_PLACEHOLDER}",
+            },
+            "body": {
+                "user": PRODUCTION_TENANT,
+                "stale_days": 30,
+                "dedup_threshold": 0.9,
+                "write_page": True,
+            },
+        },
+    },
+    {
+        "task_name": "library_paperless_orphan_cleanup",
+        "service": "library-desk",
+        "executor": "rest_api_executor",
+        "priority": 60,
+        "description": (
+            "Daily cleanup of vectors/graph nodes for documents deleted "
+            "from Paperless-ngx"
+        ),
+        "enabled": True,
+        "max_retries": 2,
+        "timeout_seconds": 900,
+        "minute": 0,
+        "hour": 5,
+        "day_of_month": -1,
+        "month": -1,
+        "day_of_week": -1,
+        "config": {
+            "method": "POST",
+            "url": (
+                f"{LIBRARY_BASE_URL}/maintenance/cleanup/paperless"
+                f"?user={PRODUCTION_TENANT}&dry_run=false"
+            ),
+            "headers": {"Authorization": f"Bearer {API_KEY_PLACEHOLDER}"},
+            "body": {},
+        },
+    },
+]
+
+#: Existing tasks to update in place.
+TASK_UPDATES = [
+    {"task_name": "test_example_task", "updates": {"enabled": False}},
+]
+
+
+def substitute_api_key(task: dict, api_key: str) -> dict:
+    """Return a deep copy of the task with the API-key placeholder filled in."""
+    resolved = copy.deepcopy(task)
+    headers = resolved.get("config", {}).get("headers", {})
+    for name, value in headers.items():
+        if API_KEY_PLACEHOLDER in value:
+            headers[name] = value.replace(API_KEY_PLACEHOLDER, api_key)
+    return resolved
+
+
+def dry_run(scheduler_url: str) -> None:
+    print("=" * 72)
+    print("DRY RUN - nothing will be sent. Re-run with --execute to register.")
+    print(f"Scheduler API: {scheduler_url or '(SCHEDULER_URL not set)'}")
+    print("=" * 72)
+    for task in TASKS:
+        print(f"\n--- create-or-update: POST {scheduler_url}/tasks "
+              f"(or PUT /tasks/{task['task_name']}) ---")
+        print(json.dumps(task, indent=2))
+    for update in TASK_UPDATES:
+        print(f"\n--- update: PUT {scheduler_url}/tasks/{update['task_name']} ---")
+        print(json.dumps(update["updates"], indent=2))
+    print("\nDry run complete: "
+          f"{len(TASKS)} task definition(s), {len(TASK_UPDATES)} update(s).")
+
+
+def execute(scheduler_url: str, api_key: str) -> int:
+    failures = 0
+    with httpx.Client(base_url=scheduler_url, timeout=30.0) as client:
+        health = client.get("/health")
+        if health.status_code != 200:
+            print(f"ERROR: Scheduler health check failed: {health.status_code}")
+            return 1
+
+        for task in TASKS:
+            name = task["task_name"]
+            payload = substitute_api_key(task, api_key)
+            exists = client.get(f"/tasks/{name}").status_code == 200
+            if exists:
+                resp = client.put(f"/tasks/{name}", json=payload)
+                action = "updated"
+            else:
+                resp = client.post("/tasks", json=payload)
+                action = "created"
+            if resp.status_code == 200:
+                print(f"[ok] {action} {name}")
+            else:
+                failures += 1
+                print(f"[FAIL] {action} {name}: {resp.status_code} {resp.text[:200]}")
+
+        for update in TASK_UPDATES:
+            name = update["task_name"]
+            if client.get(f"/tasks/{name}").status_code != 200:
+                print(f"[skip] {name} does not exist - nothing to disable")
+                continue
+            resp = client.put(f"/tasks/{name}", json=update["updates"])
+            if resp.status_code == 200:
+                print(f"[ok] updated {name}: {update['updates']}")
+            else:
+                failures += 1
+                print(f"[FAIL] update {name}: {resp.status_code} {resp.text[:200]}")
+
+    return 1 if failures else 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Register library-desk production Scheduler tasks "
+                    "(dry-run by default)"
+    )
+    parser.add_argument(
+        "--execute",
+        action="store_true",
+        help="Actually register/update the tasks (default: dry-run print only)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Explicitly print payloads without sending (the default behavior)",
+    )
+    args = parser.parse_args()
+
+    scheduler_url = os.environ.get("SCHEDULER_URL", "").rstrip("/")
+
+    if not args.execute or args.dry_run:
+        dry_run(scheduler_url)
+        return 0
+
+    if not scheduler_url:
+        print("ERROR: SCHEDULER_URL must be set for --execute")
+        return 1
+    api_key = os.environ.get("LIBRARY_API_KEY", "")
+    if not api_key:
+        print("ERROR: LIBRARY_API_KEY must be set for --execute "
+              "(it fills the Authorization header placeholder)")
+        return 1
+
+    return execute(scheduler_url, api_key)
+
+
+if __name__ == "__main__":
+    sys.exit(main())