fix(runtime): terminate invalid background followups

This commit is contained in:
Alexandre Teixeira
2026-10-02 23:22:42 +01:00
parent 8d5ff852de
commit fab3c6a15d
3 changed files with 136 additions and 19 deletions
+23 -3
View File
@@ -214,7 +214,8 @@ def _prune(jobs: Dict[str, Dict[str, Any]], now: float) -> bool:
followed up, and are older than the retention window. Mutates `jobs`."""
stale = [jid for jid, rec in jobs.items()
if rec.get("status") in {"done", "failed"}
and rec.get("followed_up") and rec.get("ended_at")
and (rec.get("followed_up") or rec.get("followup_state") == "terminal_unfollowable")
and rec.get("ended_at")
and (rec.get("teardown") or {}).get("dead") is not False
and (now - rec["ended_at"]) > _RETENTION_S]
for jid in stale:
@@ -344,10 +345,29 @@ def _kill_record(rec):
def pending_followups() -> List[Dict[str, Any]]:
"""Finished jobs the agent hasn't been re-invoked for yet. The monitor
drains these; mark_followed_up() flips the flag only on success."""
drains these; valid continuations acknowledge success, invalid immutable
linkage receives a terminal disposition without fabricating delivery."""
jobs = refresh()
return [r for r in jobs.values()
if r.get("status") in ("done", "failed") and not r.get("followed_up")]
if r.get("status") in ("done", "failed") and not r.get("followed_up")
and r.get("followup_state") != "terminal_unfollowable"]
@store_transaction(lambda: _STORE)
def mark_unfollowable(job_id: str, *, expected_record) -> bool:
"""Suppress only the exact completed snapshot inspected by the monitor.
This conveys no read/signal/continuation authority and cannot renew a PID.
It deliberately needs no invalid/missing authority sidecar to suppress it.
"""
jobs = _load()
record = jobs.get(job_id)
if (record is None or record != expected_record or record.get("id") != job_id
or record.get("status") not in {"done", "failed"}):
return False
record["followup_state"] = "terminal_unfollowable"
_save(jobs)
return True
@store_transaction(lambda: _STORE)
+29 -16
View File
@@ -13,6 +13,7 @@ from __future__ import annotations
import asyncio
import json
import logging
from enum import Enum, auto
from src import bg_jobs
from src.prompt_security import untrusted_context_message
@@ -26,6 +27,12 @@ POLL_INTERVAL_S = 5
_FOLLOWUP_MAX_ROUNDS = 12
class FollowupResult(Enum):
RETRYABLE_LATER = auto()
COMPLETED = auto()
TERMINAL_UNFOLLOWABLE = auto()
def _background_result_message(rec):
inject = (
f"[Background job {rec['id']} finished]\n\n"
@@ -104,22 +111,20 @@ async def _drain_agent(sess, messages, request_authority=None):
return full, tool_events
async def _run_followup(rec: dict) -> bool:
"""Re-invoke the agent in the job's session with the result. Returns True
if the follow-up completed (or there's nothing to do) — i.e. it's safe to
mark followed_up. Returns False to retry on the next tick."""
async def _run_followup(rec: dict) -> FollowupResult:
"""Continue only an exactly linked result; distinguish retry from terminal."""
from src.ai_interaction import get_session_manager
from core.models import ChatMessage
sm = get_session_manager()
if not sm:
return False # not ready yet — retry
return FollowupResult.RETRYABLE_LATER
sess = sm.get_session(rec["session_id"])
if not sess:
# Session was deleted — nothing to continue. Consider it handled so we
# don't retry forever.
logger.info("bg-followup: session %s gone for job %s — skipping", rec.get("session_id"), rec.get("id"))
return True
return FollowupResult.TERMINAL_UNFOLLOWABLE
# Don't write into a session that's mid-stream. The followup appends to
# history + save_sessions(); a concurrent live turn does the same, and with
@@ -129,13 +134,10 @@ async def _run_followup(rec: dict) -> bool:
from src import agent_runs
if agent_runs.is_active(sess.id):
logger.info("bg-followup: session %s busy (live turn) — deferring job %s", sess.id, rec.get("id"))
return False
return FollowupResult.RETRYABLE_LATER
except Exception:
pass
context = sess.get_context_messages()
context.append(_background_result_message(rec))
from src.agent_runtime.authority import restore_background_authority
from src.settings import get_setting
authority = restore_background_authority(
@@ -148,9 +150,11 @@ async def _run_followup(rec: dict) -> bool:
validate_job(resource)
if not authority.grants or (resource.owner, resource.thread_id, resource.request_id) != (
str(getattr(sess, "owner", None) or "").strip().casefold(), sess.id, authority.request_id):
return False
return FollowupResult.TERMINAL_UNFOLLOWABLE
except (ValueError, TypeError, OSError, RuntimeError):
return False
return FollowupResult.TERMINAL_UNFOLLOWABLE
context = sess.get_context_messages()
context.append(_background_result_message(rec))
authority = authority.restrict(disabled_tools=get_setting("disabled_tools", []) or ())
full, tool_events = await _drain_agent(sess, context, request_authority=authority)
@@ -171,7 +175,18 @@ async def _run_followup(rec: dict) -> bool:
sm.save_sessions()
logger.info("bg-followup: auto-continued session %s for job %s (%d chars, %d tools)",
sess.id, rec["id"], len(full), len(tool_events))
return True
return FollowupResult.COMPLETED
async def _process_followup(rec):
outcome = await _run_followup(rec)
if outcome is FollowupResult.COMPLETED:
from src.agent_runtime.process_resources import job_from_record
bg_jobs.mark_followed_up(rec["id"], expected=job_from_record(rec))
elif outcome is FollowupResult.TERMINAL_UNFOLLOWABLE:
bg_jobs.mark_unfollowable(rec["id"], expected_record=rec)
logger.warning("bg-followup: job %s has no valid continuation linkage; retired from pending", rec.get("id"))
return outcome
async def _loop():
@@ -179,9 +194,7 @@ async def _loop():
try:
for rec in bg_jobs.pending_followups():
try:
if await _run_followup(rec):
from src.agent_runtime.process_resources import job_from_record
bg_jobs.mark_followed_up(rec["id"], expected=job_from_record(rec))
await _process_followup(rec)
except Exception as e:
# Idempotent: leave followed_up=False so the next tick retries.
logger.warning("bg-followup failed for %s (will retry): %s", rec.get("id"), e)