jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-22 06:42:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Consolidate Odysseus agent harness and tool contracts

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-17 10:07:40 +00:00
parent 84aa9a91de
commit 218d762427
229 changed files with 28899 additions and 1551 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_sft_environment_expansion.py
+266 -10
View File
@@ -2,22 +2,25 @@
"""Execute generated SFT workflows through Odysseus with rollback and gating."""
from __future__ import annotations
import os
import argparse
import contextlib
import json
import os
import re
import shutil
import signal
import time
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
import httpx
from dotenv import load_dotenv
ROOT = Path(__file__).resolve().parents[1]
load_dotenv(ROOT / ".env")
if str(ROOT) not in __import__("sys").path:
__import__("sys").path.insert(0, str(ROOT))
@@ -37,12 +40,15 @@ from scripts.eval_odysseus_tool_use import ( # noqa: E402
_visible_event_text,
)
DATA_DIR = ROOT / "data"
from src.constants import DATA_DIR as CONFIGURED_DATA_DIR # noqa: E402
DATA_DIR = Path(CONFIGURED_DATA_DIR)
BAD_ANSWER_RE = re.compile(
r"\b(?:can't|cannot|don't have|do not have|not available|no .*tool|enable .*integration|"
r"invalid credentials|not authenticated|i can only|i'm unable)\b",
re.I,
)
_COOKIE_CACHE: dict[str, str] = {}
TOOL_FAILURE_RE = re.compile(r"(?:tool (?:failed|error)|exit_code[^\d]*[1-9]|permission denied|not found)", re.I)
INTERNAL_NARRATION_RE = re.compile(
r"(?:^|\n)(?:The user (?:asks|asked|wants)|I (?:should|need to|can see)|Let me (?:call|use|retry|try))\b",
@@ -65,7 +71,158 @@ def atomic_json(path: Path, payload: Any) -> None:
temp.replace(path)
def install_fixture_environments(path: Path) -> dict[str, int]:
"""Materialize an inventory snapshot for an isolated replay app."""
payload = json.loads(path.read_text(encoding="utf-8"))
environments = payload.get("environments", []) if isinstance(payload, dict) else []
messages: list[dict[str, Any]] = []
counts = {
"emails": 0, "notes": 0, "memories": 0, "documents": 0,
"tasks": 0, "calendars": 0, "events": 0,
}
db = SessionLocal()
owners = [
str(row.get("owner") or "").strip()
for row in environments if isinstance(row, dict)
]
try:
for owner in filter(None, owners):
document_ids = [
value[0] for value in db.query(Document.id).filter(Document.owner == owner).all()
]
if document_ids:
db.query(DocumentVersion).filter(
DocumentVersion.document_id.in_(document_ids)
).delete(synchronize_session=False)
calendar_ids = [
value[0] for value in db.query(CalendarCal.id).filter(CalendarCal.owner == owner).all()
]
if calendar_ids:
db.query(CalendarEvent).filter(
CalendarEvent.calendar_id.in_(calendar_ids)
).delete(synchronize_session=False)
db.query(Document).filter(Document.owner == owner).delete(synchronize_session=False)
db.query(Note).filter(Note.owner == owner).delete(synchronize_session=False)
db.query(Memory).filter(Memory.owner == owner).delete(synchronize_session=False)
db.query(ScheduledTask).filter(ScheduledTask.owner == owner).delete(synchronize_session=False)
db.query(CalendarCal).filter(CalendarCal.owner == owner).delete(synchronize_session=False)
for environment in environments:
if not isinstance(environment, dict):
continue
owner = str(environment.get("owner") or "").strip()
for row in environment.get("notes") or []:
db.add(Note(
id=str(row.get("id") or uuid.uuid4()), owner=owner,
title=str(row.get("title") or ""), content=str(row.get("content") or ""),
note_type=str(row.get("type") or "note"), label=row.get("label"),
archived=False, source="user",
))
counts["notes"] += 1
for row in environment.get("memories") or []:
db.add(Memory(
id=str(row.get("id") or uuid.uuid4()), owner=owner,
text=str(row.get("text") or ""),
category=str(row.get("category") or "fact"), source="user",
))
counts["memories"] += 1
for row in environment.get("documents") or []:
document_id = str(row.get("id") or uuid.uuid4())
content = str(row.get("content") or "")
db.add(Document(
id=document_id, owner=owner, title=str(row.get("title") or "Untitled"),
language=str(row.get("language") or "text"), current_content=content,
version_count=1, is_active=True, archived=False,
))
db.add(DocumentVersion(
id=str(uuid.uuid4()), document_id=document_id, version_number=1,
content=content, summary="Isolated replay fixture", source="user",
))
counts["documents"] += 1
for row in environment.get("tasks") or []:
db.add(ScheduledTask(
id=str(row.get("id") or uuid.uuid4()), owner=owner,
name=str(row.get("name") or "Untitled Task"),
status=str(row.get("status") or "active"),
schedule=row.get("schedule"), task_type="llm",
))
counts["tasks"] += 1
calendar_map: dict[str, str] = {}
for row in environment.get("calendars") or []:
calendar_id = str(row.get("id") or uuid.uuid4())
calendar_map[calendar_id] = calendar_id
db.add(CalendarCal(
id=calendar_id, owner=owner, name=str(row.get("name") or "Personal"),
source=str(row.get("source") or "local"),
))
counts["calendars"] += 1
default_calendar = next(iter(calendar_map), None)
for row in environment.get("events") or []:
if default_calendar is None:
default_calendar = str(uuid.uuid4())
db.add(CalendarCal(
id=default_calendar, owner=owner, name="Personal", source="local",
))
counts["calendars"] += 1
start = datetime.fromisoformat(str(row.get("start") or "").replace("Z", "+00:00"))
db.add(CalendarEvent(
uid=str(row.get("uid") or uuid.uuid4()), calendar_id=default_calendar,
summary=str(row.get("summary") or ""), dtstart=start,
dtend=start + timedelta(hours=1), all_day=bool(row.get("all_day")),
))
counts["events"] += 1
db.commit()
except Exception:
db.rollback()
raise
finally:
db.close()
for environment in environments:
if not isinstance(environment, dict):
continue
owner = str(environment.get("owner") or "").strip()
profile = environment.get("profile") if isinstance(environment.get("profile"), dict) else {}
primary_name = str(profile.get("primary_account") or "Primary Inbox")
secondary_name = str(profile.get("secondary_account") or "Secondary Inbox")
for source in environment.get("emails") or []:
if not isinstance(source, dict):
continue
row = dict(source)
account = str(row.get("account") or primary_name)
secondary = account == secondary_name
row.update({
"owner": owner,
"account": account,
"account_email": str(profile.get("secondary" if secondary else "primary") or owner),
"account_id": "secondary-inbox" if secondary else "primary-inbox",
"folder": str(row.get("folder") or "INBOX"),
"body": str(row.get("body") or (
f"Fixture message for: {row.get('subject') or '(no subject)'}. "
"Please review the referenced materials and reply with the next step."
)),
})
messages.append(row)
atomic_json(DATA_DIR / "fixture_email_messages.json", {"messages": messages})
counts["emails"] = len(messages)
return counts
def login(client: httpx.Client, base_url: str, owner: str, password: str) -> None:
token = _COOKIE_CACHE.get(owner)
if not token:
sessions_path = DATA_DIR / "sessions.json"
if sessions_path.exists():
with contextlib.suppress(Exception):
sessions = json.loads(sessions_path.read_text(encoding="utf-8"))
token = next(
key for key, value in reversed(list(sessions.items()))
if isinstance(value, dict) and value.get("username") == owner
)
if token:
_COOKIE_CACHE[owner] = token
client.cookies.set("odysseus_session", token)
return
response = client.post(
base_url.rstrip("/") + "/api/auth/login",
json={"username": owner, "password": password, "remember": True},
@@ -74,6 +231,9 @@ def login(client: httpx.Client, base_url: str, owner: str, password: str) -> Non
_raise_for_status_with_body(response)
if not response.json().get("ok"):
raise RuntimeError(f"login failed for {owner}")
token = client.cookies.get("odysseus_session")
if token:
_COOKIE_CACHE[owner] = token
def create_session(client: httpx.Client, args: argparse.Namespace, case: dict[str, Any]) -> str:
@@ -94,7 +254,8 @@ def create_session(client: httpx.Client, args: argparse.Namespace, case: dict[st
def stream_turn(
client: httpx.Client, args: argparse.Namespace, session_id: str, prompt: str
client: httpx.Client, args: argparse.Namespace, session_id: str, prompt: str,
*, active_doc_id: str = "",
) -> tuple[list[dict[str, Any]], str]:
events: list[dict[str, Any]] = []
text: list[str] = []
@@ -106,10 +267,15 @@ def stream_turn(
"selected_endpoint_id": args.endpoint_id,
"selected_endpoint_url": args.endpoint,
"selected_model": args.model,
"thinking_mode": args.thinking_mode,
"client_runtime_context": json.dumps(
{"timezone": args.timezone, "tz_offset_min": args.tz_offset_min}, separators=(",", ":")
),
}
if active_doc_id:
form["active_doc_id"] = active_doc_id
if getattr(args, "allow_web_search", False):
form["allow_web_search"] = "true"
with client.stream(
"POST",
args.base_url.rstrip("/") + "/api/chat_stream",
@@ -154,6 +320,46 @@ def tool_outputs(events: list[dict[str, Any]]) -> str:
return "\n".join(str(e.get("output") or "") for e in events if e.get("type") == "tool_output")
def has_unrecovered_tool_failure(events: list[dict[str, Any]]) -> bool:
"""Count a tool failure only when that tool never subsequently succeeds."""
pending: set[str] = set()
for event in events:
if event.get("type") != "tool_output":
continue
name = normalized_tool(str(event.get("tool") or "unknown"))
output = str(event.get("output") or "")
failed = bool(
event.get("error")
or event.get("exit_code") not in (None, 0)
or TOOL_FAILURE_RE.search(output)
)
if failed:
pending.add(name)
else:
pending.discard(name)
return bool(pending)
def compact_evidence(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Keep routing and execution evidence without bloating the replay report."""
retained = {
"turn_contract", "tool_start", "tool_output", "tool_resolution_audit",
"error", "parse_error", "metrics", "final_response",
}
rows = []
for event in events:
if event.get("type") not in retained:
continue
row = dict(event)
for key in ("output", "text", "delta"):
if isinstance(row.get(key), str) and len(row[key]) > 3000:
row[key] = row[key][:3000] + "..."
if row.get("type") == "turn_contract":
row.pop("executable", None)
rows.append(row)
return rows
def tool_actions(events: list[dict[str, Any]], tool_name: str) -> set[str]:
actions: set[str] = set()
for event in events:
@@ -200,6 +406,11 @@ def score_turn(turn: dict[str, Any], events: list[dict[str, Any]], answer: str)
failures: list[str] = []
names = tool_names(events)
expected = {normalized_tool(str(name)) for name in turn.get("expected_tools") or []}
# Both document writers satisfy a requested active-draft mutation. Which
# one is most efficient depends on how much of the draft the model changes;
# exact-name imitation is not a functional correctness requirement.
if expected & {"edit_document", "update_document"}:
expected.update({"edit_document", "update_document"})
if expected and not expected.intersection(names):
failures.append(f"missing_acceptable_tool expected={sorted(expected)} got={names}")
for tool_name, expected_actions in inferred_expected_actions(turn).items():
@@ -213,8 +424,7 @@ def score_turn(turn: dict[str, Any], events: list[dict[str, Any]], answer: str)
failures.append("stream_error")
if BAD_ANSWER_RE.search(answer):
failures.append("tool_unavailable_answer")
output = tool_outputs(events)
if TOOL_FAILURE_RE.search(output):
if has_unrecovered_tool_failure(events):
failures.append("tool_output_failure")
if INTERNAL_NARRATION_RE.search(answer):
failures.append("internal_narration_leaked")
@@ -375,10 +585,11 @@ def marker_fields(value: Any, marker: str) -> Any:
return value
def apply_fixture_plan(case: dict[str, Any], owner: str, session_id: str, marker: str) -> None:
def apply_fixture_plan(case: dict[str, Any], owner: str, session_id: str, marker: str) -> dict[str, str]:
"""Create only owner-scoped local fixtures required before the first turn."""
first_tools = set((case.get("turns") or [{}])[0].get("expected_tools") or [])
db = SessionLocal()
context: dict[str, str] = {}
try:
for fixture in case.get("fixture_plan") or []:
if not isinstance(fixture, dict):
@@ -407,6 +618,7 @@ def apply_fixture_plan(case: dict[str, Any], owner: str, session_id: str, marker
summary="Expansion fixture",
source="user",
))
context["active_doc_id"] = document_id
elif fixture_type == "note":
db.add(Note(
id=str(uuid.uuid4()),
@@ -426,6 +638,7 @@ def apply_fixture_plan(case: dict[str, Any], owner: str, session_id: str, marker
raise
finally:
db.close()
return context
def delete_session(client: httpx.Client, base_url: str, session_id: str) -> None:
@@ -482,10 +695,14 @@ def run_case(args: argparse.Namespace, case: dict[str, Any]) -> dict[str, Any]:
snapshot.capture()
login(client, args.base_url, owner, args.password)
session_id = create_session(client, args, case)
apply_fixture_plan(case, owner, session_id, marker)
fixture_context = apply_fixture_plan(case, owner, session_id, marker)
upstream_failed = False
for turn in case["turns"]:
prompt = str(turn["prompt"]).replace("{marker}", marker)
events, answer = stream_turn(client, args, session_id, prompt)
events, answer = stream_turn(
client, args, session_id, prompt,
active_doc_id=fixture_context.get("active_doc_id", ""),
)
turn_failures = score_turn(turn, events, answer)
turns_out.append({
"id": turn["id"],
@@ -494,10 +711,14 @@ def run_case(args: argparse.Namespace, case: dict[str, Any]) -> dict[str, Any]:
"observed_tools": tool_names(events),
"answer": answer,
"failures": turn_failures,
"evidence": compact_evidence(events),
"upstream_failed": upstream_failed,
})
failures.extend(f"{turn['id']}:{failure}" for failure in turn_failures)
if turn_failures:
break
# Keep executing the full 3-4 turn trajectory. Later misses may be
# causal fallout from an earlier failed create/read, so the judge
# receives this marker and can separate root causes from cascades.
upstream_failed = upstream_failed or bool(turn_failures)
except Exception as exc:
failures.append(f"exception:{exc!r}")
finally:
@@ -538,6 +759,20 @@ def main() -> None:
parser.add_argument("--endpoint-id", default="f3904562")
parser.add_argument("--endpoint", default="https://openrouter.ai/api/v1/chat/completions")
parser.add_argument("--model", default="moonshotai/kimi-k3")
parser.add_argument("--thinking-mode", choices=("on", "off"), default="off")
parser.add_argument(
"--allow-web-search",
action="store_true",
help="Enable Odysseus public web_search/web_fetch for this replay.",
)
parser.add_argument(
"--fixture-environments",
type=Path,
help=(
"Install owner-scoped synthetic inventory rows for replay. "
"Use only against an isolated app with ODYSSEUS_EMAIL_FIXTURE=1."
),
)
parser.add_argument("--turn-timeout", type=float, default=180)
parser.add_argument("--case-timeout", type=float, default=600)
parser.add_argument("--timezone", default="Asia/Tokyo")
@@ -545,13 +780,34 @@ def main() -> None:
parser.add_argument("--limit", type=int)
parser.add_argument("--owner", action="append")
parser.add_argument("--case-id", action="append")
parser.add_argument(
"--one-per-seed",
action="store_true",
help="Run the first validated environment variant for each source seed family",
)
args = parser.parse_args()
if args.fixture_environments:
if os.environ.get("ODYSSEUS_EMAIL_FIXTURE") != "1":
parser.error("--fixture-environments requires ODYSSEUS_EMAIL_FIXTURE=1")
installed = install_fixture_environments(args.fixture_environments)
print(f"installed isolated fixture inventory: {installed}", flush=True)
cases = json.loads(args.cases.read_text(encoding="utf-8"))["cases"]
if args.owner:
cases = [case for case in cases if case["owner"] in set(args.owner)]
if args.case_id:
cases = [case for case in cases if case["case_id"] in set(args.case_id)]
if args.one_per_seed:
seen_seeds: set[str] = set()
first_cases = []
for case in cases:
seed_id = str(case.get("seed_family_id") or "")
if seed_id in seen_seeds:
continue
seen_seeds.add(seed_id)
first_cases.append(case)
cases = first_cases
if args.limit:
cases = cases[: args.limit]
existing = {row["case_id"]: row for row in json.loads(args.out.read_text(encoding="utf-8")).get("results", [])} if args.out.exists() else {}