mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-25 09:32:20 +02:00
Squash Odysseus development history
This commit is contained in:
pewdiepie-archdaemon
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a small live-model evaluation for exact edit_file routing."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from src.agent_loop import (
|
||||
_WORKSPACE_AGENT_TOOLS,
|
||||
_looks_like_exact_file_replacement,
|
||||
stream_agent_loop,
|
||||
)
|
||||
|
||||
|
||||
EXACT_TEMPLATES = [
|
||||
"In {path}, change status=old to status=new.",
|
||||
"Replace `June 30` with `July 1` in {path}.",
|
||||
"In {path}, update MODE=dev to MODE=prod.",
|
||||
"Change ETA June 30 to ETA July 1 in {path}.",
|
||||
"Replace owner=alice with owner=bob in {path}.",
|
||||
"In {path}, change enabled=false to enabled=true.",
|
||||
"Update color=red to color=green in {path}.",
|
||||
"In {path}, replace port=8000 with port=9000.",
|
||||
"Change queue=slow to queue=fast in {path}.",
|
||||
"Replace draft with published in {path}.",
|
||||
"In {path}, update retry=1 to retry=3.",
|
||||
"Change region=west to region=east in {path}.",
|
||||
"Replace level=info with level=warning in {path}.",
|
||||
"In {path}, change feature=off to feature=on.",
|
||||
"Update team=alpha to team=beta in {path}.",
|
||||
"Replace pending with approved in {path}.",
|
||||
"In {path}, change timeout=30 to timeout=60.",
|
||||
"Change format=csv to format=json in {path}.",
|
||||
"Replace stage=test with stage=production in {path}.",
|
||||
"In {path}, update version=1 to version=2.",
|
||||
]
|
||||
|
||||
CONTROL_PREFIXES = [
|
||||
"Inspect {path}, then change old_value to new_value.",
|
||||
"Read {path} first, then replace old_value with new_value.",
|
||||
"Show the contents of {path}, then change old_value to new_value.",
|
||||
"Open {path} and replace old_value with new_value.",
|
||||
"Review {path} before changing old_value to new_value.",
|
||||
"Use cat to inspect {path}, then replace old_value with new_value.",
|
||||
"Examine {path}, then update old_value to new_value.",
|
||||
"Look at {path} before replacing old_value with new_value.",
|
||||
"Change old_value to new_value in {path} and verify the result.",
|
||||
"Replace old_value with new_value in {path}, then run the tests.",
|
||||
]
|
||||
|
||||
|
||||
def _values(template: str) -> tuple[str, str]:
|
||||
pairs = [
|
||||
("status=old", "status=new"), ("June 30", "July 1"),
|
||||
("MODE=dev", "MODE=prod"), ("ETA June 30", "ETA July 1"),
|
||||
("owner=alice", "owner=bob"), ("enabled=false", "enabled=true"),
|
||||
("color=red", "color=green"), ("port=8000", "port=9000"),
|
||||
("queue=slow", "queue=fast"), ("draft", "published"),
|
||||
("retry=1", "retry=3"), ("region=west", "region=east"),
|
||||
("level=info", "level=warning"), ("feature=off", "feature=on"),
|
||||
("team=alpha", "team=beta"), ("pending", "approved"),
|
||||
("timeout=30", "timeout=60"), ("format=csv", "format=json"),
|
||||
("stage=test", "stage=production"), ("version=1", "version=2"),
|
||||
]
|
||||
return pairs[EXACT_TEMPLATES.index(template)]
|
||||
|
||||
|
||||
def _event(chunk: str):
|
||||
if not chunk.startswith("data: ") or chunk.startswith("data: [DONE]"):
|
||||
return None
|
||||
try:
|
||||
return json.loads(chunk[6:])
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
async def _run_case(endpoint: str, model: str, owner: str, prompt: str, path: Path, expected: str):
|
||||
chunks = []
|
||||
starts = []
|
||||
outputs = []
|
||||
stream = stream_agent_loop(
|
||||
endpoint,
|
||||
model,
|
||||
[{"role": "user", "content": prompt}],
|
||||
temperature=0.2,
|
||||
max_tokens=1024,
|
||||
max_rounds=4,
|
||||
max_tool_calls=4,
|
||||
owner=owner,
|
||||
workspace=str(path.parent),
|
||||
relevant_tools=set(_WORKSPACE_AGENT_TOOLS),
|
||||
)
|
||||
async for chunk in stream:
|
||||
chunks.append(chunk)
|
||||
event = _event(chunk)
|
||||
if not event:
|
||||
continue
|
||||
if event.get("type") == "tool_start":
|
||||
starts.append(event.get("tool"))
|
||||
elif event.get("type") == "tool_output":
|
||||
outputs.append(event)
|
||||
actual = path.read_text() if path.exists() else ""
|
||||
return {
|
||||
"classifier_exact": _looks_like_exact_file_replacement(prompt),
|
||||
"tool_sequence": starts,
|
||||
"tool_outputs": outputs,
|
||||
"first_tool": starts[0] if starts else None,
|
||||
"content_ok": actual == expected,
|
||||
"actual_content": actual,
|
||||
"response": "".join(
|
||||
event.get("delta", "")
|
||||
for chunk in chunks
|
||||
if (event := _event(chunk)) and isinstance(event.get("delta"), str)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def main(args):
|
||||
models_url = args.endpoint.rstrip("/") + "/models"
|
||||
try:
|
||||
models_response = httpx.get(models_url, timeout=10)
|
||||
except httpx.ConnectError:
|
||||
# The same eval may run on the host or inside the backend container.
|
||||
# Docker's host alias is container-only; use the host-published loopback
|
||||
# endpoint when the evaluator is running outside Docker.
|
||||
if "host.docker.internal" not in args.endpoint:
|
||||
raise
|
||||
args.endpoint = args.endpoint.replace("host.docker.internal", "127.0.0.1")
|
||||
models_response = httpx.get(args.endpoint.rstrip("/") + "/models", timeout=10)
|
||||
models_response.raise_for_status()
|
||||
advertised = {
|
||||
item.get("id")
|
||||
for item in models_response.json().get("data", [])
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
if args.model not in advertised:
|
||||
raise SystemExit(
|
||||
f"Requested model {args.model!r} is not advertised by the endpoint; "
|
||||
f"available={sorted(name for name in advertised if name)}"
|
||||
)
|
||||
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
records = []
|
||||
with output.open("w") as handle, tempfile.TemporaryDirectory(prefix="ody-exact-edit-") as root:
|
||||
def emit(record):
|
||||
records.append(record)
|
||||
handle.write(json.dumps(record) + "\n")
|
||||
handle.flush()
|
||||
print(json.dumps(record), flush=True)
|
||||
|
||||
root_path = Path(root)
|
||||
for repetition in range(1, args.repetitions + 1):
|
||||
exact_templates = EXACT_TEMPLATES[:args.exact_limit] if args.exact_limit else EXACT_TEMPLATES
|
||||
for index, template in enumerate(exact_templates, 1):
|
||||
old, new = _values(template)
|
||||
path = root_path / f"exact_{index}.txt"
|
||||
path.write_text(old + "\n")
|
||||
prompt = template.format(path=path)
|
||||
result = await _run_case(args.endpoint, args.model, args.owner, prompt, path, new + "\n")
|
||||
emit({
|
||||
"kind": "exact", "case": index, "repetition": repetition,
|
||||
"model": args.label or args.model, "request_model": args.model,
|
||||
"prompt": prompt, **result,
|
||||
})
|
||||
|
||||
control_templates = [] if args.skip_controls else (
|
||||
CONTROL_PREFIXES[:args.control_limit] if args.control_limit else CONTROL_PREFIXES
|
||||
)
|
||||
for index, template in enumerate(control_templates, 1):
|
||||
path = root_path / f"control_{index}.txt"
|
||||
path.write_text("old_value\n")
|
||||
prompt = template.format(path=path)
|
||||
result = await _run_case(args.endpoint, args.model, args.owner, prompt, path, "new_value\n")
|
||||
emit({
|
||||
"kind": "control", "case": index, "repetition": repetition,
|
||||
"model": args.label or args.model, "request_model": args.model,
|
||||
"prompt": prompt, **result,
|
||||
})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--endpoint", required=True)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--label")
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--owner", default="pewds")
|
||||
parser.add_argument("--repetitions", type=int, default=2)
|
||||
parser.add_argument("--exact-limit", type=int, default=0)
|
||||
parser.add_argument("--control-limit", type=int, default=0)
|
||||
parser.add_argument("--skip-controls", action="store_true")
|
||||
asyncio.run(main(parser.parse_args()))
|
||||
Reference in New Issue
Block a user