jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-09-13 19:52:20 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
6ee65020107a9d81386d2ce427002cbc609ef090
odysseus/scripts/sft_email_overseer.py
T

801 lines
28 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Email SFT overseer: expand curated seed traces across coherent fixture envs.
This script is intentionally conservative:
- it can enrich target users' fixture mailboxes from Alex's richer mailbox;
- it builds a run plan from audited keep rows plus Kimi/manual repairs;
- it does not mutate chat history or run the harness unless a future run
subcommand is added explicitly.
"""
from __future__ import annotations
import argparse
import copy
import json
import re
import sqlite3
import time
import uuid
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DB = ROOT / "data" / "app.db"
FIXTURE = ROOT / "data" / "fixture_email_messages.json"
AUDIT_DIR = ROOT / "data" / "audits"
OUT_DIR = ROOT / "data" / "evals"
DEFAULT_BASE_URL = "http://127.0.0.1:7011"
DEFAULT_PASSWORD = "SftDemo!2026"
DEFAULT_ENDPOINT_ID = "f3904562"
DEFAULT_ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
DEFAULT_MODEL = "moonshotai/kimi-k3"
SOURCE_OWNER = "sft_alex_creator"
TARGET_OWNERS = ["sft_maya_ops", "sft_jules_research", "sft_nora_design", "sft_omar_finance"]
PROFILES: dict[str, dict[str, str]] = {
"sft_alex_creator": {
"name": "Alex Rowan",
"first": "Alex",
"primary": "alex.rowan@rowan.studio",
"secondary": "alex.research@rowan.studio",
"primary_account": "Primary Inbox",
"secondary_account": "Research Mail",
"topic": "creator operations",
"org": "Rowan Studio",
"domain": "rowan.studio",
"secondary_domain": "northstar-research.co",
},
"sft_maya_ops": {
"name": "Maya Chen",
"first": "Maya",
"primary": "maya.chen@northstar-ops.co",
"secondary": "maya.research@northstar-ops.co",
"primary_account": "Primary Inbox",
"secondary_account": "Ops Research",
"topic": "operations planning",
"org": "Northstar Ops",
"domain": "northstar-ops.co",
"secondary_domain": "northstar-research.co",
},
"sft_jules_research": {
"name": "Jules Rivera",
"first": "Jules",
"primary": "jules.rivera@rivera-lab.org",
"secondary": "jules.review@rivera-lab.org",
"primary_account": "Primary Inbox",
"secondary_account": "Research Mail",
"topic": "research synthesis",
"org": "Rivera Lab",
"domain": "rivera-lab.org",
"secondary_domain": "northstar-research.co",
},
"sft_nora_design": {
"name": "Nora Patel",
"first": "Nora",
"primary": "nora.patel@northpier.design",
"secondary": "nora.research@northpier.design",
"primary_account": "Primary Inbox",
"secondary_account": "Design Research",
"topic": "product design",
"org": "Northpier Design",
"domain": "northpier.design",
"secondary_domain": "northstar-research.co",
},
"sft_omar_finance": {
"name": "Omar Singh",
"first": "Omar",
"primary": "omar.singh@bayledger.finance",
"secondary": "omar.research@bayledger.finance",
"primary_account": "Primary Inbox",
"secondary_account": "Finance Research",
"topic": "finance analysis",
"org": "Bayledger Finance",
"domain": "bayledger.finance",
"secondary_domain": "northstar-research.co",
},
}
SENDER_DOMAIN_MAP = {
"collab.rowan.studio": "collab.{domain}",
"metrics.rowan.studio": "metrics.{domain}",
"rowan.studio": "{domain}",
"mail.rowan.studio": "mail.{domain}",
}
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
def db() -> sqlite3.Connection:
con = sqlite3.connect(DB)
con.row_factory = sqlite3.Row
return con
def latest_deepseek_audit() -> Path:
paths = sorted(AUDIT_DIR.glob("email_sft_deepseek_audit_sft_alex_creator_*.jsonl"))
if not paths:
raise RuntimeError("No DeepSeek email audit found")
return paths[-1]
def repair_artifact_paths() -> list[Path]:
return sorted(AUDIT_DIR.glob("email_sft_kimi_repairs_*.jsonl")) + sorted(
AUDIT_DIR.glob("email_sft_kimi_repairs_manual_date_*.jsonl")
)
def fixture_rows() -> list[dict[str, Any]]:
payload = read_json(FIXTURE)
rows = payload.get("messages") if isinstance(payload, dict) else payload
if not isinstance(rows, list):
raise RuntimeError(f"Unexpected fixture shape: {type(payload).__name__}")
return rows
def save_fixture_rows(rows: list[dict[str, Any]]) -> None:
write_json(FIXTURE, {"messages": rows})
def owner_counts(rows: list[dict[str, Any]]) -> Counter:
return Counter(str(row.get("owner") or "") for row in rows)
def account_counts(rows: list[dict[str, Any]]) -> dict[str, Counter]:
out: dict[str, Counter] = defaultdict(Counter)
for row in rows:
owner = str(row.get("owner") or "")
account = str(row.get("account") or row.get("account_id") or "Primary Inbox")
out[owner][account] += 1
return out
def transform_text(text: str, target_owner: str) -> str:
src = PROFILES[SOURCE_OWNER]
tgt = PROFILES[target_owner]
replacements = {
src["name"]: tgt["name"],
src["first"]: tgt["first"],
src["primary"]: tgt["primary"],
src["secondary"]: tgt["secondary"],
src["topic"]: tgt["topic"],
src["org"]: tgt["org"],
"creator operations": tgt["topic"],
"creator ops": tgt["topic"],
"creator cohort": "workstream cohort",
"creator": "workstream",
"Rowan Studio": tgt["org"],
"rowan.studio": tgt["domain"],
"alex-rowan": f"{tgt['first'].lower()}-{tgt['name'].split()[-1].lower()}",
}
out = text
for old, new in replacements.items():
out = out.replace(old, new)
return out
def transform_email_address(addr: str, target_owner: str) -> str:
tgt = PROFILES[target_owner]
out = addr
for old_domain, new_template in SENDER_DOMAIN_MAP.items():
out = out.replace(old_domain, new_template.format(domain=tgt["domain"]))
return out
def retarget_row(row: dict[str, Any], target_owner: str, uid_offset: int) -> dict[str, Any]:
tgt = PROFILES[target_owner]
cloned = copy.deepcopy(row)
source_uid = str(row.get("uid") or "")
try:
new_uid = str(uid_offset + int(source_uid))
except ValueError:
new_uid = f"{uid_offset}{re.sub(r'\\W+', '', source_uid)[:8]}"
cloned["owner"] = target_owner
cloned["uid"] = new_uid
cloned["source_seed_owner"] = SOURCE_OWNER
cloned["source_seed_uid"] = source_uid
cloned["overseer_generated"] = True
cloned["overseer_version"] = 1
account_id = str(row.get("account_id") or "primary-inbox")
if account_id == "research-mail":
cloned["account_id"] = "research-mail"
cloned["account"] = tgt["secondary_account"]
cloned["account_email"] = tgt["secondary"]
cloned["to"] = f"{tgt['name']} <{tgt['secondary']}>"
else:
cloned["account_id"] = "primary-inbox"
cloned["account"] = tgt["primary_account"]
cloned["account_email"] = tgt["primary"]
cloned["to"] = f"{tgt['name']} <{tgt['primary']}>"
for key in ["subject", "summary", "body", "message_id", "references"]:
if isinstance(cloned.get(key), str):
cloned[key] = transform_text(cloned[key], target_owner)
for key in ["from", "sender"]:
if isinstance(cloned.get(key), str):
cloned[key] = transform_email_address(transform_text(cloned[key], target_owner), target_owner)
if cloned.get("message_id"):
cloned["message_id"] = f"<sft-overseer-{target_owner}-{new_uid}@mail.{tgt['domain']}>"
for att in cloned.get("attachments") or []:
if isinstance(att, dict):
for key in ["filename", "content"]:
if isinstance(att.get(key), str):
att[key] = transform_text(att[key], target_owner)
return cloned
def seed_target_fixtures(targets: list[str], *, dry_run: bool = False) -> dict[str, Any]:
rows = fixture_rows()
source_rows = [
row for row in rows
if row.get("owner") == SOURCE_OWNER and not row.get("overseer_generated")
]
before = owner_counts(rows)
kept = [
row for row in rows
if not (row.get("owner") in targets and row.get("overseer_generated"))
]
generated: list[dict[str, Any]] = []
for idx, target in enumerate(targets, start=1):
offset = 1000 * idx
generated.extend(retarget_row(row, target, offset) for row in source_rows)
after_rows = kept + generated
after = owner_counts(after_rows)
summary = {
"source_owner": SOURCE_OWNER,
"source_rows": len(source_rows),
"targets": targets,
"removed_old_generated": len(rows) - len(kept),
"generated_rows": len(generated),
"before_counts": dict(sorted(before.items())),
"after_counts": dict(sorted(after.items())),
"dry_run": dry_run,
}
if not dry_run:
backup = FIXTURE.with_suffix(f".json.bak-{time.strftime('%Y%m%d_%H%M%S')}")
backup.write_text(FIXTURE.read_text(encoding="utf-8"), encoding="utf-8")
save_fixture_rows(after_rows)
summary["backup"] = str(backup)
return summary
def load_audit_rows() -> list[dict[str, Any]]:
return [json.loads(line) for line in latest_deepseek_audit().read_text(encoding="utf-8").splitlines() if line.strip()]
def load_repair_rows() -> dict[str, dict[str, Any]]:
repairs: dict[str, dict[str, Any]] = {}
for path in repair_artifact_paths():
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
row = json.loads(line)
sid = str(row.get("session_id") or "")
if sid:
repairs[sid] = row
return repairs
def session_user_messages(session_id: str) -> list[str]:
con = db()
try:
return [
str(row["content"] or "")
for row in con.execute(
"SELECT content FROM chat_messages WHERE session_id = ? AND role = 'user' ORDER BY timestamp, id",
(session_id,),
)
if str(row["content"] or "").strip()
]
finally:
con.close()
def usable_seed_records(min_keep_score: int = 0) -> list[dict[str, Any]]:
audit_rows = load_audit_rows()
repairs = load_repair_rows()
seeds: list[dict[str, Any]] = []
for row in audit_rows:
sid = str(row.get("session_id") or "")
verdict = row.get("verdict")
score = int(row.get("trainable_score") or 0)
if verdict == "keep" and score >= min_keep_score:
users = session_user_messages(sid)
seeds.append({
"session_id": sid,
"source": "keep",
"score": score,
"session_name": row.get("session_name"),
"user_messages": users,
"first_user": users[0] if users else "",
})
elif verdict == "repair":
repair = repairs.get(sid)
if repair and repair.get("repair_decision") == "repair":
users = [
str(m.get("content") or "")
for m in repair.get("messages") or []
if m.get("role") == "user" and str(m.get("content") or "").strip()
]
seeds.append({
"session_id": sid,
"source": "repair",
"score": int(repair.get("sft_quality_after_repair") or score),
"session_name": row.get("session_name"),
"user_messages": users,
"first_user": users[0] if users else "",
})
seeds.sort(key=lambda item: (-int(item["score"]), str(item["session_name"] or "")))
return seeds
CONTEXTLESS_FIRST_TURN_RE = re.compile(
r"^\s*(?:"
r"yes\b|yeah\b|ok\b|okay\b|open (?:it|the att|the attachment)\b|"
r"read (?:it|the att|the attachment)\b|"
r"reply\b|draft reply\b|"
r".*\bthis email\b|.*\bthat email\b|.*\bopen it\b|.*\bthe attachment\b"
r")",
re.IGNORECASE,
)
def seed_is_standalone(seed: dict[str, Any]) -> bool:
first = str(seed.get("first_user") or "").strip()
if not first:
return False
if CONTEXTLESS_FIRST_TURN_RE.search(first):
return False
return True
def retarget_prompt(text: str, target_owner: str) -> str:
out = transform_text(text, target_owner)
target = PROFILES[target_owner]
# Keep prompts natural: "Alex" references inside user text should become the
# target user, but sender names such as Casey/Priya/Dana remain stable because
# matching fixture rows are generated for those senders.
out = out.replace(PROFILES[SOURCE_OWNER]["first"], target["first"])
return out
def build_plan(targets: list[str], per_target: int, min_keep_score: int) -> dict[str, Any]:
all_seeds = usable_seed_records(min_keep_score=min_keep_score)
seeds = [seed for seed in all_seeds if seed_is_standalone(seed)]
if not seeds:
raise RuntimeError("No usable seeds found. Run audit/repair first.")
cases: list[dict[str, Any]] = []
for target in targets:
for idx, seed in enumerate(seeds[:per_target], start=1):
turns = [retarget_prompt(msg, target) for msg in seed["user_messages"]]
cases.append({
"id": f"email_overseer_{target}_{idx:03d}_{seed['session_id'][:8]}",
"domain": "email",
"owner": target,
"source_owner": SOURCE_OWNER,
"source_session_id": seed["session_id"],
"source_type": seed["source"],
"source_score": seed["score"],
"session_name": seed["session_name"],
"turns": turns,
"current_date": "2026-08-24",
"timezone": "UTC",
"fixture_requirements": {
"mailbox_seeded_from": SOURCE_OWNER,
"target_primary": PROFILES[target]["primary"],
"target_secondary": PROFILES[target]["secondary"],
},
"acceptance": {
"must_use_email_tool": True,
"reject_bad_unavailable_answer": True,
"reject_claimed_action_without_tool": True,
"judge_with_deepseek": True,
"repair_with_kimi": True,
},
})
return {
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"source_owner": SOURCE_OWNER,
"targets": targets,
"per_target": per_target,
"seed_count_available": len(seeds),
"seed_count_before_standalone_filter": len(all_seeds),
"seed_count_skipped_contextual_first_turn": len(all_seeds) - len(seeds),
"case_count": len(cases),
"cases": cases,
}
def write_plan(plan: dict[str, Any]) -> Path:
path = OUT_DIR / f"sft_email_overseer_plan_{time.strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}.json"
write_json(path, plan)
return path
def login(client: httpx.Client, base_url: str, username: str, password: str) -> None:
res = client.post(
base_url.rstrip("/") + "/api/auth/login",
json={"username": username, "password": password, "remember": True},
timeout=30,
)
res.raise_for_status()
if not res.json().get("ok"):
raise RuntimeError(f"login failed for {username}: {res.text[:300]}")
def create_session(
client: httpx.Client,
*,
base_url: str,
owner: str,
case_id: str,
endpoint: str,
endpoint_id: str,
model: str,
) -> str:
res = client.post(
base_url.rstrip("/") + "/api/session",
data={
"name": f"SFT email overseer {owner} {case_id}",
"endpoint_url": endpoint,
"endpoint_id": endpoint_id,
"model": model,
"skip_validation": "true",
"rag": "false",
},
timeout=30,
)
res.raise_for_status()
return str(res.json()["id"])
def sse_events(response: httpx.Response) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
event_name = "message"
data_lines: list[str] = []
for raw in response.iter_lines():
line = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else raw
if line == "":
if data_lines:
raw_data = "\n".join(data_lines)
try:
payload = json.loads(raw_data)
except json.JSONDecodeError:
payload = {"type": event_name, "raw": raw_data}
events.append(payload)
event_name = "message"
data_lines = []
continue
if line.startswith("event:"):
event_name = line.split(":", 1)[1].strip()
elif line.startswith("data:"):
data_lines.append(line.split(":", 1)[1].lstrip())
if data_lines:
raw_data = "\n".join(data_lines)
try:
events.append(json.loads(raw_data))
except json.JSONDecodeError:
events.append({"type": event_name, "raw": raw_data})
return events
def event_text(event: dict[str, Any]) -> str:
for key in ("content", "text", "response", "message", "output"):
value = event.get(key)
if isinstance(value, str):
return value
return ""
def stream_turn(
client: httpx.Client,
*,
base_url: str,
session_id: str,
message: str,
endpoint: str,
endpoint_id: str,
model: str,
timeout: float,
) -> tuple[list[dict[str, Any]], str]:
form = {
"message": message,
"session": session_id,
"mode": "agent",
"agent_prompt_mode": "auto",
"selected_endpoint_id": endpoint_id,
"selected_endpoint_url": endpoint,
"selected_model": model,
"client_runtime_context": json.dumps({"timezone": "UTC", "tz_offset_min": 0}, separators=(",", ":")),
}
with client.stream(
"POST",
base_url.rstrip("/") + "/api/chat_stream",
data=form,
headers={"Accept": "text/event-stream", "X-Tz-Name": "UTC", "X-Tz-Offset": "0"},
timeout=timeout,
) as response:
response.raise_for_status()
events = sse_events(response)
final = ""
parts: list[str] = []
for event in events:
typ = str(event.get("type") or "")
text = event_text(event)
if not text:
continue
if typ == "final_response":
final = text
elif typ in {"token", "content", "assistant_delta", "message"}:
parts.append(text)
return events, (final or "".join(parts)).strip()
BAD_ANSWER_RE = re.compile(
r"\b(?:can't|cannot|don't have|do not have|not available|no .*tool|enable .*integration|setup .*integration|"
r"invalid credentials|not authenticated|i can only|i'm unable)\b",
re.IGNORECASE,
)
def tool_names(events: list[dict[str, Any]]) -> list[str]:
names = []
for event in events:
if event.get("type") == "tool_start" and event.get("tool"):
names.append(str(event["tool"]))
elif event.get("tool") and str(event.get("type") or "").startswith("tool"):
names.append(str(event["tool"]))
return names
def assistant_count(session_id: str) -> int:
con = db()
try:
return int(con.execute(
"SELECT COUNT(*) FROM chat_messages WHERE session_id = ? AND role = 'assistant'",
(session_id,),
).fetchone()[0])
finally:
con.close()
def latest_assistant_from_db(session_id: str, min_count: int) -> dict[str, Any]:
con = db()
try:
rows = list(con.execute(
"""
SELECT content, metadata, timestamp
FROM chat_messages
WHERE session_id = ? AND role = 'assistant'
ORDER BY timestamp, id
""",
(session_id,),
))
finally:
con.close()
if len(rows) <= min_count:
return {"content": "", "tool_events": [], "thinking": ""}
row = rows[-1]
meta: dict[str, Any] = {}
if row["metadata"]:
try:
meta = json.loads(row["metadata"])
except json.JSONDecodeError:
meta = {}
return {
"content": str(row["content"] or ""),
"tool_events": list(meta.get("tool_events") or []),
"thinking": str(meta.get("thinking") or ""),
}
def persisted_tool_names(tool_events: list[dict[str, Any]]) -> list[str]:
return [str(ev.get("tool") or "") for ev in tool_events if ev.get("tool")]
def score_run(case: dict[str, Any], turns: list[dict[str, Any]]) -> tuple[bool, list[str]]:
failures: list[str] = []
all_events = [event for turn in turns for event in turn.get("events", [])]
all_tools = [
name
for turn in turns
for name in (turn.get("persisted_tool_names") or turn.get("tool_names") or [])
]
combined_answer = "\n".join(str(turn.get("answer") or "") for turn in turns)
if any(str(event.get("type") or "") in {"error", "parse_error"} for event in all_events):
failures.append("stream_error")
if BAD_ANSWER_RE.search(combined_answer):
failures.append("bad_unavailable_answer")
if case.get("acceptance", {}).get("must_use_email_tool") and not any("email" in name for name in all_tools):
failures.append(f"missing_email_tool tools={all_tools}")
return not failures, failures
def run_plan(args: argparse.Namespace) -> dict[str, Any]:
plan = read_json(Path(args.plan))
cases = list(plan.get("cases") or [])
if args.owner:
owners = set(parse_targets(args.owner))
cases = [case for case in cases if case.get("owner") in owners]
cases = cases[args.offset : args.offset + args.limit]
results: list[dict[str, Any]] = []
clients: dict[str, httpx.Client] = {}
try:
for case in cases:
owner = str(case["owner"])
client = clients.get(owner)
if client is None:
client = httpx.Client(follow_redirects=True)
login(client, args.base_url, owner, args.password)
clients[owner] = client
session_id = create_session(
client,
base_url=args.base_url,
owner=owner,
case_id=case["id"],
endpoint=args.endpoint,
endpoint_id=args.endpoint_id,
model=args.model,
)
turn_results: list[dict[str, Any]] = []
started = time.time()
error = ""
try:
for message in case.get("turns") or []:
before = assistant_count(session_id)
events, streamed_answer = stream_turn(
client,
base_url=args.base_url,
session_id=session_id,
message=message,
endpoint=args.endpoint,
endpoint_id=args.endpoint_id,
model=args.model,
timeout=args.timeout,
)
persisted = latest_assistant_from_db(session_id, before)
answer = persisted["content"] or streamed_answer
ptools = persisted_tool_names(persisted["tool_events"])
turn_results.append({
"user": message,
"answer": answer,
"events": events,
"tool_names": tool_names(events),
"persisted_tool_names": ptools,
"persisted_tool_events": persisted["tool_events"],
})
passed, failures = score_run(case, turn_results)
except Exception as exc:
error = repr(exc)
passed = False
failures = [f"exception: {error}"]
results.append({
"id": case["id"],
"owner": owner,
"source_session_id": case.get("source_session_id"),
"session_id": session_id,
"pass": passed,
"failures": failures,
"turns": [
{
"user": turn["user"],
"answer": turn["answer"],
"tool_names": turn.get("persisted_tool_names") or turn["tool_names"],
}
for turn in turn_results
],
"elapsed_seconds": round(time.time() - started, 3),
"error": error,
})
finally:
for client in clients.values():
client.close()
out_dir = Path(args.out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
payload = {
"plan": str(args.plan),
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"results": results,
"summary": dict(Counter("pass" if row["pass"] else "fail" for row in results)),
}
write_json(out_dir / "actual_results.json", payload)
return payload
def status() -> dict[str, Any]:
rows = fixture_rows()
audit_rows = load_audit_rows() if latest_deepseek_audit().exists() else []
repairs = load_repair_rows()
repair_counts = Counter(row.get("repair_decision") for row in repairs.values())
usable = usable_seed_records()
return {
"fixture_counts": dict(sorted(owner_counts(rows).items())),
"fixture_accounts": {owner: dict(counter) for owner, counter in sorted(account_counts(rows).items())},
"audit_counts": dict(Counter(row.get("verdict") for row in audit_rows)),
"repair_artifact_counts": dict(repair_counts),
"usable_seed_count": len(usable),
"target_owners": TARGET_OWNERS,
}
def parse_targets(raw: str) -> list[str]:
if raw == "all":
return list(TARGET_OWNERS)
targets = [item.strip() for item in raw.split(",") if item.strip()]
unknown = [target for target in targets if target not in PROFILES or target == SOURCE_OWNER]
if unknown:
raise SystemExit(f"Unknown/non-target owners: {unknown}")
return targets
def main() -> int:
parser = argparse.ArgumentParser(description="Oversee email SFT fixture expansion and plan generation.")
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("status")
seed = sub.add_parser("seed-fixtures")
seed.add_argument("--targets", default="all", help="Comma list of target owners or 'all'")
seed.add_argument("--dry-run", action="store_true")
plan = sub.add_parser("build-plan")
plan.add_argument("--targets", default="all", help="Comma list of target owners or 'all'")
plan.add_argument("--per-target", type=int, default=95)
plan.add_argument("--min-keep-score", type=int, default=0)
run = sub.add_parser("run-plan")
run.add_argument("--plan", required=True)
run.add_argument("--owner", default="", help="Optional comma list of owners to run")
run.add_argument("--offset", type=int, default=0)
run.add_argument("--limit", type=int, default=4)
run.add_argument("--base-url", default=DEFAULT_BASE_URL)
run.add_argument("--password", default=DEFAULT_PASSWORD)
run.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
run.add_argument("--endpoint-id", default=DEFAULT_ENDPOINT_ID)
run.add_argument("--model", default=DEFAULT_MODEL)
run.add_argument("--timeout", type=float, default=90)
run.add_argument("--out-dir", default=str(OUT_DIR / f"sft_email_overseer_run_{time.strftime('%Y%m%d_%H%M%S')}"))
args = parser.parse_args()
if args.cmd == "status":
print(json.dumps(status(), indent=2, ensure_ascii=True))
return 0
if args.cmd == "seed-fixtures":
summary = seed_target_fixtures(parse_targets(args.targets), dry_run=args.dry_run)
print(json.dumps(summary, indent=2, ensure_ascii=True))
return 0
if args.cmd == "build-plan":
built = build_plan(parse_targets(args.targets), args.per_target, args.min_keep_score)
path = write_plan(built)
print(json.dumps({"plan": str(path), "case_count": built["case_count"], "targets": built["targets"]}, indent=2))
return 0
if args.cmd == "run-plan":
payload = run_plan(args)
print(json.dumps({"summary": payload["summary"], "out": str(Path(args.out_dir) / "actual_results.json")}, indent=2))
return 0 if payload["summary"].get("fail", 0) == 0 else 1
raise AssertionError(args.cmd)
if __name__ == "__main__":
raise SystemExit(main())
Reference in New Issue View Git Blame Copy Permalink