From 80229f275c50112467f0512ef967f2fbef187dcd Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Sun, 16 Aug 2026 18:27:08 +0200 Subject: [PATCH] fix(health): bound and parallelise the five dependency probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_service_health() ran neo4j, qdrant, wikijs, searxng and ollama serially with await, and only ollama's client carried its own timeout. Neo4j (connection_timeout=30.0) and Qdrant (timeout=30.0) fell back to driver defaults far past the container healthcheck's 10s timeout, so a hung (not failing) dependency blocked the whole chain and flipped the container unhealthy for a reason unrelated to its own liveness. Each probe now runs under asyncio.wait_for bounded at 2s — chosen against the 10s healthcheck timeout so five concurrent bounded probes cannot approach it even if all five hang — and all five run concurrently under asyncio.gather(return_exceptions=True), so one probe timing out or raising cannot block or cancel the others. Gating (neo4j+qdrant only), the unconditional 200 response, ollama's existing 5.0s client-level timeout, and the paperless/system_settings/ scheduler probes are unchanged. --- CHANGELOG.md | 11 +++ src/core/dependencies.py | 162 ++++++++++++++++++++++++++++----------- 2 files changed, 127 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 820caa8..268e1b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `/health`'s five dependency probes (neo4j, qdrant, wikijs, searxng, ollama) + now run concurrently under `asyncio.gather`, each bounded at 2s via + `asyncio.wait_for`, instead of serially with no bound on neo4j or qdrant. + A hung (not failing) dependency previously blocked the whole chain past the + container healthcheck's 10s timeout, marking the container unhealthy for a + reason unrelated to its own liveness. A probe that times out now reports + that service unhealthy, the same as one that raises; the others are + unaffected. + ## [1.9.1] - 2026-08-08 ### Fixed diff --git a/src/core/dependencies.py b/src/core/dependencies.py index 7e3a76e..931489e 100644 --- a/src/core/dependencies.py +++ b/src/core/dependencies.py @@ -8,6 +8,7 @@ Provides FastAPI dependencies for service clients with: - Type aliases for clean endpoint signatures """ +import asyncio from functools import lru_cache from typing import TYPE_CHECKING, Annotated from fastapi import Depends, HTTPException, Query @@ -565,10 +566,100 @@ async def shutdown_clients(): logger.info("Service clients shutdown complete") +# Per-probe timeout for the concurrent health checks below, in seconds. +# +# Bounded well under the container healthcheck's 10s timeout (see Dockerfile) +# so that five probes run concurrently under asyncio.gather cannot approach +# it even if all five hang. A probe that has no timeout of its own (neo4j, +# qdrant) otherwise falls back to its driver's default — 30s for both — which +# is what let a hung dependency (not a failing one) flip the container +# unhealthy. +HEALTH_PROBE_TIMEOUT = 2.0 + + +async def _probe_neo4j() -> bool: + """Neo4j connectivity probe. Returns True if healthy.""" + try: + neo4j = get_neo4j_client() + # Simple query to check connectivity + await neo4j.execute_query("RETURN 1 as test", {}) + return True + except Exception as e: + logger.error(f"Neo4j health check failed: {e}") + return False + + +async def _probe_qdrant() -> bool: + """Qdrant connectivity probe. Returns True if healthy.""" + try: + qdrant = get_qdrant_client() + # Check if we can list collections + await qdrant.client.get_collections() + return True + except Exception as e: + logger.error(f"Qdrant health check failed: {e}") + return False + + +async def _probe_wikijs() -> bool: + """Wiki.js connectivity probe. Returns True if healthy.""" + try: + wikijs = get_wikijs_client() + # Try a simple query (list pages with limit 1) + await wikijs.list_pages(limit=1) + return True + except Exception as e: + logger.error(f"Wiki.js health check failed: {e}") + return False + + +async def _probe_searxng() -> bool: + """SearXNG connectivity probe. Returns True if healthy.""" + try: + searxng = get_searxng_client() + # Just check if service is up (no actual search) + return await searxng.health_check() + except Exception as e: + logger.error(f"SearXNG health check failed: {e}") + return False + + +async def _probe_ollama() -> bool: + """Ollama connectivity probe. Returns True if healthy.""" + try: + ollama = get_ollama_client() + return await ollama.health_check() + except Exception as e: + logger.error(f"Ollama health check failed: {e}") + return False + + +async def _bounded_probe(coro) -> bool: + """ + Run a probe coroutine bounded by HEALTH_PROBE_TIMEOUT. + + A probe that times out is reported unhealthy, the same as one that + raises. Wrapping happens here rather than in each _probe_* function so + the bound applies uniformly regardless of whether the underlying client + has its own (looser, or absent) timeout. + """ + try: + return await asyncio.wait_for(coro, timeout=HEALTH_PROBE_TIMEOUT) + except asyncio.TimeoutError: + logger.error(f"Health probe timed out after {HEALTH_PROBE_TIMEOUT}s") + return False + + async def check_service_health() -> dict: """ Check health of all service clients. + The five core probes (neo4j, qdrant, wikijs, searxng, ollama) run + concurrently, each bounded at HEALTH_PROBE_TIMEOUT, so a single hung + dependency cannot block the others or push the endpoint past the + container healthcheck's timeout. return_exceptions=True on the gather + means one probe raising cannot cancel its siblings. + Returns: Dictionary with health status of each service: { @@ -586,53 +677,32 @@ async def check_service_health() -> dict: """ health = {} - # Neo4j - try: - neo4j = get_neo4j_client() - # Simple query to check connectivity - await neo4j.execute_query("RETURN 1 as test", {}) - health["neo4j"] = True - except Exception as e: - logger.error(f"Neo4j health check failed: {e}") - health["neo4j"] = False + neo4j_result, qdrant_result, wikijs_result, searxng_result, ollama_result = ( + await asyncio.gather( + _bounded_probe(_probe_neo4j()), + _bounded_probe(_probe_qdrant()), + _bounded_probe(_probe_wikijs()), + _bounded_probe(_probe_searxng()), + _bounded_probe(_probe_ollama()), + return_exceptions=True, + ) + ) - # Qdrant - try: - qdrant = get_qdrant_client() - # Check if we can list collections - await qdrant.client.get_collections() - health["qdrant"] = True - except Exception as e: - logger.error(f"Qdrant health check failed: {e}") - health["qdrant"] = False - - # Wiki.js - try: - wikijs = get_wikijs_client() - # Try a simple query (list pages with limit 1) - await wikijs.list_pages(limit=1) - health["wikijs"] = True - except Exception as e: - logger.error(f"Wiki.js health check failed: {e}") - health["wikijs"] = False - - # SearXNG - try: - searxng = get_searxng_client() - # Just check if service is up (no actual search) - health["searxng"] = await searxng.health_check() - except Exception as e: - logger.error(f"SearXNG health check failed: {e}") - health["searxng"] = False - - # Ollama - try: - ollama = get_ollama_client() - is_healthy = await ollama.health_check() - health["ollama"] = is_healthy - except Exception as e: - logger.error(f"Ollama health check failed: {e}") - health["ollama"] = False + # _bounded_probe already catches everything from its own probe, but + # return_exceptions=True also guards against a bug in _bounded_probe + # itself surfacing as an unhandled exception here. + for name, result in ( + ("neo4j", neo4j_result), + ("qdrant", qdrant_result), + ("wikijs", wikijs_result), + ("searxng", searxng_result), + ("ollama", ollama_result), + ): + if isinstance(result, BaseException): + logger.error(f"{name} health check raised unexpectedly: {result}") + health[name] = False + else: + health[name] = result # Paperless-ngx settings = get_settings()