mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-07 07:22:21 +02:00
fix(runtime): terminate invalid background followups
This commit is contained in:
+23
-3
@@ -214,7 +214,8 @@ def _prune(jobs: Dict[str, Dict[str, Any]], now: float) -> bool:
|
||||
followed up, and are older than the retention window. Mutates `jobs`."""
|
||||
stale = [jid for jid, rec in jobs.items()
|
||||
if rec.get("status") in {"done", "failed"}
|
||||
and rec.get("followed_up") and rec.get("ended_at")
|
||||
and (rec.get("followed_up") or rec.get("followup_state") == "terminal_unfollowable")
|
||||
and rec.get("ended_at")
|
||||
and (rec.get("teardown") or {}).get("dead") is not False
|
||||
and (now - rec["ended_at"]) > _RETENTION_S]
|
||||
for jid in stale:
|
||||
@@ -344,10 +345,29 @@ def _kill_record(rec):
|
||||
|
||||
def pending_followups() -> List[Dict[str, Any]]:
|
||||
"""Finished jobs the agent hasn't been re-invoked for yet. The monitor
|
||||
drains these; mark_followed_up() flips the flag only on success."""
|
||||
drains these; valid continuations acknowledge success, invalid immutable
|
||||
linkage receives a terminal disposition without fabricating delivery."""
|
||||
jobs = refresh()
|
||||
return [r for r in jobs.values()
|
||||
if r.get("status") in ("done", "failed") and not r.get("followed_up")]
|
||||
if r.get("status") in ("done", "failed") and not r.get("followed_up")
|
||||
and r.get("followup_state") != "terminal_unfollowable"]
|
||||
|
||||
|
||||
@store_transaction(lambda: _STORE)
|
||||
def mark_unfollowable(job_id: str, *, expected_record) -> bool:
|
||||
"""Suppress only the exact completed snapshot inspected by the monitor.
|
||||
|
||||
This conveys no read/signal/continuation authority and cannot renew a PID.
|
||||
It deliberately needs no invalid/missing authority sidecar to suppress it.
|
||||
"""
|
||||
jobs = _load()
|
||||
record = jobs.get(job_id)
|
||||
if (record is None or record != expected_record or record.get("id") != job_id
|
||||
or record.get("status") not in {"done", "failed"}):
|
||||
return False
|
||||
record["followup_state"] = "terminal_unfollowable"
|
||||
_save(jobs)
|
||||
return True
|
||||
|
||||
|
||||
@store_transaction(lambda: _STORE)
|
||||
|
||||
+29
-16
@@ -13,6 +13,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from enum import Enum, auto
|
||||
|
||||
from src import bg_jobs
|
||||
from src.prompt_security import untrusted_context_message
|
||||
@@ -26,6 +27,12 @@ POLL_INTERVAL_S = 5
|
||||
_FOLLOWUP_MAX_ROUNDS = 12
|
||||
|
||||
|
||||
class FollowupResult(Enum):
|
||||
RETRYABLE_LATER = auto()
|
||||
COMPLETED = auto()
|
||||
TERMINAL_UNFOLLOWABLE = auto()
|
||||
|
||||
|
||||
def _background_result_message(rec):
|
||||
inject = (
|
||||
f"[Background job {rec['id']} finished]\n\n"
|
||||
@@ -104,22 +111,20 @@ async def _drain_agent(sess, messages, request_authority=None):
|
||||
return full, tool_events
|
||||
|
||||
|
||||
async def _run_followup(rec: dict) -> bool:
|
||||
"""Re-invoke the agent in the job's session with the result. Returns True
|
||||
if the follow-up completed (or there's nothing to do) — i.e. it's safe to
|
||||
mark followed_up. Returns False to retry on the next tick."""
|
||||
async def _run_followup(rec: dict) -> FollowupResult:
|
||||
"""Continue only an exactly linked result; distinguish retry from terminal."""
|
||||
from src.ai_interaction import get_session_manager
|
||||
from core.models import ChatMessage
|
||||
|
||||
sm = get_session_manager()
|
||||
if not sm:
|
||||
return False # not ready yet — retry
|
||||
return FollowupResult.RETRYABLE_LATER
|
||||
sess = sm.get_session(rec["session_id"])
|
||||
if not sess:
|
||||
# Session was deleted — nothing to continue. Consider it handled so we
|
||||
# don't retry forever.
|
||||
logger.info("bg-followup: session %s gone for job %s — skipping", rec.get("session_id"), rec.get("id"))
|
||||
return True
|
||||
return FollowupResult.TERMINAL_UNFOLLOWABLE
|
||||
|
||||
# Don't write into a session that's mid-stream. The followup appends to
|
||||
# history + save_sessions(); a concurrent live turn does the same, and with
|
||||
@@ -129,13 +134,10 @@ async def _run_followup(rec: dict) -> bool:
|
||||
from src import agent_runs
|
||||
if agent_runs.is_active(sess.id):
|
||||
logger.info("bg-followup: session %s busy (live turn) — deferring job %s", sess.id, rec.get("id"))
|
||||
return False
|
||||
return FollowupResult.RETRYABLE_LATER
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
context = sess.get_context_messages()
|
||||
context.append(_background_result_message(rec))
|
||||
|
||||
from src.agent_runtime.authority import restore_background_authority
|
||||
from src.settings import get_setting
|
||||
authority = restore_background_authority(
|
||||
@@ -148,9 +150,11 @@ async def _run_followup(rec: dict) -> bool:
|
||||
validate_job(resource)
|
||||
if not authority.grants or (resource.owner, resource.thread_id, resource.request_id) != (
|
||||
str(getattr(sess, "owner", None) or "").strip().casefold(), sess.id, authority.request_id):
|
||||
return False
|
||||
return FollowupResult.TERMINAL_UNFOLLOWABLE
|
||||
except (ValueError, TypeError, OSError, RuntimeError):
|
||||
return False
|
||||
return FollowupResult.TERMINAL_UNFOLLOWABLE
|
||||
context = sess.get_context_messages()
|
||||
context.append(_background_result_message(rec))
|
||||
authority = authority.restrict(disabled_tools=get_setting("disabled_tools", []) or ())
|
||||
full, tool_events = await _drain_agent(sess, context, request_authority=authority)
|
||||
|
||||
@@ -171,7 +175,18 @@ async def _run_followup(rec: dict) -> bool:
|
||||
sm.save_sessions()
|
||||
logger.info("bg-followup: auto-continued session %s for job %s (%d chars, %d tools)",
|
||||
sess.id, rec["id"], len(full), len(tool_events))
|
||||
return True
|
||||
return FollowupResult.COMPLETED
|
||||
|
||||
|
||||
async def _process_followup(rec):
|
||||
outcome = await _run_followup(rec)
|
||||
if outcome is FollowupResult.COMPLETED:
|
||||
from src.agent_runtime.process_resources import job_from_record
|
||||
bg_jobs.mark_followed_up(rec["id"], expected=job_from_record(rec))
|
||||
elif outcome is FollowupResult.TERMINAL_UNFOLLOWABLE:
|
||||
bg_jobs.mark_unfollowable(rec["id"], expected_record=rec)
|
||||
logger.warning("bg-followup: job %s has no valid continuation linkage; retired from pending", rec.get("id"))
|
||||
return outcome
|
||||
|
||||
|
||||
async def _loop():
|
||||
@@ -179,9 +194,7 @@ async def _loop():
|
||||
try:
|
||||
for rec in bg_jobs.pending_followups():
|
||||
try:
|
||||
if await _run_followup(rec):
|
||||
from src.agent_runtime.process_resources import job_from_record
|
||||
bg_jobs.mark_followed_up(rec["id"], expected=job_from_record(rec))
|
||||
await _process_followup(rec)
|
||||
except Exception as e:
|
||||
# Idempotent: leave followed_up=False so the next tick retries.
|
||||
logger.warning("bg-followup failed for %s (will retry): %s", rec.get("id"), e)
|
||||
|
||||
Reference in New Issue
Block a user