fix(health): bound and parallelise the five dependency probes

check_service_health() ran neo4j, qdrant, wikijs, searxng and ollama
serially with await, and only ollama's client carried its own timeout.
Neo4j (connection_timeout=30.0) and Qdrant (timeout=30.0) fell back to
driver defaults far past the container healthcheck's 10s timeout, so a
hung (not failing) dependency blocked the whole chain and flipped the
container unhealthy for a reason unrelated to its own liveness.

Each probe now runs under asyncio.wait_for bounded at 2s — chosen
against the 10s healthcheck timeout so five concurrent bounded probes
cannot approach it even if all five hang — and all five run
concurrently under asyncio.gather(return_exceptions=True), so one
probe timing out or raising cannot block or cancel the others.

Gating (neo4j+qdrant only), the unconditional 200 response, ollama's
existing 5.0s client-level timeout, and the paperless/system_settings/
scheduler probes are unchanged.
This commit is contained in:
2026-08-16 18:27:08 +02:00
parent a687b770ef
commit 80229f275c
2 changed files with 127 additions and 46 deletions
+11
View File
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- `/health`'s five dependency probes (neo4j, qdrant, wikijs, searxng, ollama)
now run concurrently under `asyncio.gather`, each bounded at 2s via
`asyncio.wait_for`, instead of serially with no bound on neo4j or qdrant.
A hung (not failing) dependency previously blocked the whole chain past the
container healthcheck's 10s timeout, marking the container unhealthy for a
reason unrelated to its own liveness. A probe that times out now reports
that service unhealthy, the same as one that raises; the others are
unaffected.
## [1.9.1] - 2026-08-08
### Fixed
+116 -46
View File
@@ -8,6 +8,7 @@ Provides FastAPI dependencies for service clients with:
- Type aliases for clean endpoint signatures
"""
import asyncio
from functools import lru_cache
from typing import TYPE_CHECKING, Annotated
from fastapi import Depends, HTTPException, Query
@@ -565,10 +566,100 @@ async def shutdown_clients():
logger.info("Service clients shutdown complete")
# Per-probe timeout for the concurrent health checks below, in seconds.
#
# Bounded well under the container healthcheck's 10s timeout (see Dockerfile)
# so that five probes run concurrently under asyncio.gather cannot approach
# it even if all five hang. A probe that has no timeout of its own (neo4j,
# qdrant) otherwise falls back to its driver's default — 30s for both — which
# is what let a hung dependency (not a failing one) flip the container
# unhealthy.
HEALTH_PROBE_TIMEOUT = 2.0
async def _probe_neo4j() -> bool:
"""Neo4j connectivity probe. Returns True if healthy."""
try:
neo4j = get_neo4j_client()
# Simple query to check connectivity
await neo4j.execute_query("RETURN 1 as test", {})
return True
except Exception as e:
logger.error(f"Neo4j health check failed: {e}")
return False
async def _probe_qdrant() -> bool:
"""Qdrant connectivity probe. Returns True if healthy."""
try:
qdrant = get_qdrant_client()
# Check if we can list collections
await qdrant.client.get_collections()
return True
except Exception as e:
logger.error(f"Qdrant health check failed: {e}")
return False
async def _probe_wikijs() -> bool:
"""Wiki.js connectivity probe. Returns True if healthy."""
try:
wikijs = get_wikijs_client()
# Try a simple query (list pages with limit 1)
await wikijs.list_pages(limit=1)
return True
except Exception as e:
logger.error(f"Wiki.js health check failed: {e}")
return False
async def _probe_searxng() -> bool:
"""SearXNG connectivity probe. Returns True if healthy."""
try:
searxng = get_searxng_client()
# Just check if service is up (no actual search)
return await searxng.health_check()
except Exception as e:
logger.error(f"SearXNG health check failed: {e}")
return False
async def _probe_ollama() -> bool:
"""Ollama connectivity probe. Returns True if healthy."""
try:
ollama = get_ollama_client()
return await ollama.health_check()
except Exception as e:
logger.error(f"Ollama health check failed: {e}")
return False
async def _bounded_probe(coro) -> bool:
"""
Run a probe coroutine bounded by HEALTH_PROBE_TIMEOUT.
A probe that times out is reported unhealthy, the same as one that
raises. Wrapping happens here rather than in each _probe_* function so
the bound applies uniformly regardless of whether the underlying client
has its own (looser, or absent) timeout.
"""
try:
return await asyncio.wait_for(coro, timeout=HEALTH_PROBE_TIMEOUT)
except asyncio.TimeoutError:
logger.error(f"Health probe timed out after {HEALTH_PROBE_TIMEOUT}s")
return False
async def check_service_health() -> dict:
"""
Check health of all service clients.
The five core probes (neo4j, qdrant, wikijs, searxng, ollama) run
concurrently, each bounded at HEALTH_PROBE_TIMEOUT, so a single hung
dependency cannot block the others or push the endpoint past the
container healthcheck's timeout. return_exceptions=True on the gather
means one probe raising cannot cancel its siblings.
Returns:
Dictionary with health status of each service:
{
@@ -586,53 +677,32 @@ async def check_service_health() -> dict:
"""
health = {}
# Neo4j
try:
neo4j = get_neo4j_client()
# Simple query to check connectivity
await neo4j.execute_query("RETURN 1 as test", {})
health["neo4j"] = True
except Exception as e:
logger.error(f"Neo4j health check failed: {e}")
health["neo4j"] = False
neo4j_result, qdrant_result, wikijs_result, searxng_result, ollama_result = (
await asyncio.gather(
_bounded_probe(_probe_neo4j()),
_bounded_probe(_probe_qdrant()),
_bounded_probe(_probe_wikijs()),
_bounded_probe(_probe_searxng()),
_bounded_probe(_probe_ollama()),
return_exceptions=True,
)
)
# Qdrant
try:
qdrant = get_qdrant_client()
# Check if we can list collections
await qdrant.client.get_collections()
health["qdrant"] = True
except Exception as e:
logger.error(f"Qdrant health check failed: {e}")
health["qdrant"] = False
# Wiki.js
try:
wikijs = get_wikijs_client()
# Try a simple query (list pages with limit 1)
await wikijs.list_pages(limit=1)
health["wikijs"] = True
except Exception as e:
logger.error(f"Wiki.js health check failed: {e}")
health["wikijs"] = False
# SearXNG
try:
searxng = get_searxng_client()
# Just check if service is up (no actual search)
health["searxng"] = await searxng.health_check()
except Exception as e:
logger.error(f"SearXNG health check failed: {e}")
health["searxng"] = False
# Ollama
try:
ollama = get_ollama_client()
is_healthy = await ollama.health_check()
health["ollama"] = is_healthy
except Exception as e:
logger.error(f"Ollama health check failed: {e}")
health["ollama"] = False
# _bounded_probe already catches everything from its own probe, but
# return_exceptions=True also guards against a bug in _bounded_probe
# itself surfacing as an unhandled exception here.
for name, result in (
("neo4j", neo4j_result),
("qdrant", qdrant_result),
("wikijs", wikijs_result),
("searxng", searxng_result),
("ollama", ollama_result),
):
if isinstance(result, BaseException):
logger.error(f"{name} health check raised unexpectedly: {result}")
health[name] = False
else:
health[name] = result
# Paperless-ngx
settings = get_settings()