fix: supervise the Wiki.js change listener so it survives a database restart
The listener opened one asyncpg connection, called add_listener, and set running = True. Nothing watched that connection afterwards. When it dropped, the subscription was gone for good while running still reported True, so the service stayed healthy in every way anything could observe and silently stopped indexing page edits. Recovery needed a manual container restart. That happened on 2026-08-08 when postgres-shared was redeployed. The sibling settings_client survived the same event because it uses asyncpg.create_pool, which replaces dead connections; a bare LISTEN connection has no such recovery. A supervisor task now waits on asyncpg's termination callback and reconnects with bounded exponential backoff, 1s doubling to a 60s cap. It retries forever rather than giving up after N attempts: a database under maintenance does come back, and a listener that stopped trying would reproduce exactly the silent deafness this exists to prevent. The termination listener is re-registered on every new connection because asyncpg clears its listener list as soon as it fires them, so a one-time registration survives exactly one drop. running is now derived from the connection rather than assigned, and stop() sets a flag the termination callback and supervisor both check so a deliberate shutdown cannot race into a reconnect. NOTIFY is fire-and-forget, so events emitted during an outage are lost and cannot be replayed. The reconnect logs the gap and names POST /maintenance/integrity-check rather than reporting a clean recovery. Reconciling automatically is left out on purpose: deriving the tenant for a changed page is subtle here, and getting it wrong writes into the wrong user's namespace. Verified against the real database by terminating the listener's backend with pg_terminate_backend. Old code: running=True with is_closed()=True, dead forever. New code: reconnects on its own onto a new server pid. The same probe was run against both implementations so the check is known to discriminate. One existing test mocked the connection with a bare AsyncMock, which models asyncpg's synchronous is_closed() as a coroutine — always truthy, so the connection read as closed once running started deriving from it. Corrected. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -308,6 +308,10 @@ class TestWikiChangeListener:
|
||||
async def test_connection_lifecycle(self, listener, mock_settings):
|
||||
"""Test listener connection start and stop lifecycle."""
|
||||
mock_connection = AsyncMock()
|
||||
# asyncpg's is_closed() is synchronous. Left as an AsyncMock it returns a
|
||||
# coroutine, which is always truthy, so the connection would read as
|
||||
# closed the moment `running` started deriving from it.
|
||||
mock_connection.is_closed = MagicMock(return_value=False)
|
||||
|
||||
with patch('src.services.wiki_change_listener.asyncpg.connect', return_value=mock_connection) as mock_connect:
|
||||
# Start listener
|
||||
@@ -426,3 +430,168 @@ class TestWikiChangeListenerIntegration:
|
||||
|
||||
# This would test actual pg_notify() calls from triggers
|
||||
# and verify the listener receives and processes them
|
||||
|
||||
|
||||
class TestWikiChangeListenerReconnect:
|
||||
"""
|
||||
Supervision and reconnect behaviour.
|
||||
|
||||
Regression cover for 2026-08-08: redeploying postgres-shared dropped the
|
||||
listener's connection and it never came back. The container kept reporting
|
||||
healthy because nothing observed the subscription, so Wiki.js page edits
|
||||
silently stopped being indexed until someone restarted the service.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_running_is_false_when_connection_closed(self, listener):
|
||||
"""running reflects the live connection, not a flag set once at startup."""
|
||||
connection = MagicMock()
|
||||
connection.is_closed.return_value = False
|
||||
listener.connection = connection
|
||||
assert listener.running is True
|
||||
|
||||
# The exact failure mode: connection dies, nothing reassigns a flag.
|
||||
connection.is_closed.return_value = True
|
||||
assert listener.running is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_running_is_false_while_stopping(self, listener):
|
||||
"""A listener being torn down does not advertise itself as subscribed."""
|
||||
connection = MagicMock()
|
||||
connection.is_closed.return_value = False
|
||||
listener.connection = connection
|
||||
listener._stopping = True
|
||||
assert listener.running is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_termination_callback_wakes_supervisor(self, listener):
|
||||
"""asyncpg's termination callback signals the supervisor and records the time."""
|
||||
assert not listener._disconnected.is_set()
|
||||
|
||||
listener._on_connection_lost(MagicMock())
|
||||
|
||||
assert listener._disconnected.is_set()
|
||||
assert listener._disconnected_at is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_termination_callback_ignored_while_stopping(self, listener):
|
||||
"""A deliberate shutdown must not trigger a reconnect."""
|
||||
listener._stopping = True
|
||||
|
||||
listener._on_connection_lost(MagicMock())
|
||||
|
||||
assert not listener._disconnected.is_set()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supervisor_reconnects_after_drop(self, listener):
|
||||
"""One drop produces one reconnect cycle."""
|
||||
connection = MagicMock()
|
||||
connection.is_closed.return_value = True
|
||||
listener.connection = connection
|
||||
|
||||
with patch.object(listener, '_connect', new=AsyncMock()) as connect:
|
||||
listener._disconnected.set()
|
||||
cycles = await listener._supervise(max_iterations=1)
|
||||
|
||||
assert cycles == 1
|
||||
connect.assert_awaited_once()
|
||||
assert listener.reconnects == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supervisor_exits_without_reconnecting_when_stopping(self, listener):
|
||||
"""stop() wakes the supervisor to exit, not to re-establish the connection."""
|
||||
with patch.object(listener, '_connect', new=AsyncMock()) as connect:
|
||||
listener._stopping = True
|
||||
listener._disconnected.set()
|
||||
cycles = await listener._supervise(max_iterations=1)
|
||||
|
||||
assert cycles == 0
|
||||
connect.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconnect_backs_off_and_retries(self, listener):
|
||||
"""A database that is still down is retried, with growing delay."""
|
||||
attempts = []
|
||||
|
||||
async def fail_twice_then_succeed():
|
||||
attempts.append(1)
|
||||
if len(attempts) < 3:
|
||||
raise OSError("connection refused")
|
||||
|
||||
sleeps = []
|
||||
|
||||
async def fake_sleep(seconds):
|
||||
sleeps.append(seconds)
|
||||
|
||||
with patch.object(listener, '_connect', new=AsyncMock(side_effect=fail_twice_then_succeed)), \
|
||||
patch.object(listener, '_close_connection', new=AsyncMock()), \
|
||||
patch('asyncio.sleep', new=fake_sleep):
|
||||
ok = await listener._reconnect_with_backoff()
|
||||
|
||||
assert ok is True
|
||||
assert len(attempts) == 3
|
||||
assert sleeps == [1.0, 2.0] # doubling
|
||||
assert listener.reconnects == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconnect_backoff_is_capped(self, listener):
|
||||
"""Backoff does not grow without bound during a long outage."""
|
||||
sleeps = []
|
||||
|
||||
async def fake_sleep(seconds):
|
||||
sleeps.append(seconds)
|
||||
|
||||
with patch.object(listener, '_connect', new=AsyncMock(side_effect=OSError("down"))), \
|
||||
patch.object(listener, '_close_connection', new=AsyncMock()), \
|
||||
patch('asyncio.sleep', new=fake_sleep):
|
||||
ok = await listener._reconnect_with_backoff(max_attempts=12)
|
||||
|
||||
assert ok is False
|
||||
assert max(sleeps) == listener.BACKOFF_MAX_SECONDS
|
||||
assert listener.reconnects == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconnect_records_outage_duration(self, listener):
|
||||
"""The gap is measured so the lost-notification window is reportable."""
|
||||
listener._disconnected_at = datetime.now() - timedelta(seconds=30)
|
||||
|
||||
with patch.object(listener, '_connect', new=AsyncMock()), \
|
||||
patch.object(listener, '_close_connection', new=AsyncMock()):
|
||||
await listener._reconnect_with_backoff()
|
||||
|
||||
assert listener.last_gap_seconds is not None
|
||||
assert 29 <= listener.last_gap_seconds <= 32
|
||||
assert listener._disconnected_at is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_connection_tolerates_dead_connection(self, listener):
|
||||
"""Cleaning up an already-terminated connection must not raise."""
|
||||
connection = MagicMock()
|
||||
connection.is_closed.return_value = False
|
||||
connection.remove_listener = AsyncMock(side_effect=Exception("connection is closed"))
|
||||
connection.close = AsyncMock()
|
||||
listener.connection = connection
|
||||
|
||||
await listener._close_connection()
|
||||
|
||||
assert listener.connection is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_registers_termination_listener(self, listener):
|
||||
"""
|
||||
The termination listener is re-registered on every connection.
|
||||
|
||||
asyncpg clears its termination listeners as soon as it fires them, so a
|
||||
registration that happened only once would survive exactly one drop.
|
||||
"""
|
||||
connection = MagicMock()
|
||||
connection.add_listener = AsyncMock()
|
||||
connection.is_closed.return_value = False
|
||||
|
||||
with patch('asyncpg.connect', new=AsyncMock(return_value=connection)):
|
||||
await listener._connect()
|
||||
|
||||
connection.add_listener.assert_awaited_once()
|
||||
connection.add_termination_listener.assert_called_once_with(
|
||||
listener._on_connection_lost
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user