jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-10 08:52:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(runtime): record execution evidence and gate completion

Browse Source
This commit is contained in:
Alexandre Teixeira
2026-09-26 13:48:12 +01:00
parent ea847e6c0b
commit eaa5668baa
16 changed files with 1105 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent_evidence.py
+72 -34
View File
@@ -9,6 +9,7 @@ from dataclasses import asdict, dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any, Iterable, Mapping, Sequence
from src.agent_runtime.identity import artifact_identity, artifact_version, executable_words, is_test_command, is_validation_command
def workspace_artifact_is_usable(path: Path) -> bool:
@@ -99,6 +100,10 @@ class EvidenceEvent:
command_sha256: str = ""
output_sha256: str = ""
detail: str = ""
action_id: str = ""
execution_id: str = ""
artifact_id: str = ""
verification_id: str = ""
def to_dict(self) -> dict[str, Any]:
data = asdict(self)
@@ -195,12 +200,12 @@ _VALIDATION_COMMAND_RE = re.compile(
def command_is_validation(command: str) -> bool:
"""Return whether a shell command provides executable verification evidence."""
value = str(command or "")
return bool(_TEST_COMMAND_RE.search(value) or _VALIDATION_COMMAND_RE.search(value))
return is_validation_command(value)
def command_is_test(command: str) -> bool:
"""Return whether a shell command executes a recognized test runner."""
return bool(_TEST_COMMAND_RE.search(str(command or "")))
return is_test_command(str(command or ""))
def _clean_path(value: str) -> str:
@@ -441,18 +446,12 @@ def _path_is_mentioned(command: str, required_path: str) -> bool:
return path in command or Path(path).name in command
def _artifact_path_matches_required(artifact_path: str, required_path: str) -> bool:
artifact = _clean_path(artifact_path)
def _artifact_path_matches_required(artifact_path: str, required_path: str, workspace: str = "") -> bool:
artifact = str(artifact_path or '').strip()
required = _clean_path(required_path)
if not artifact or not required:
return False
if artifact == required:
return True
# Absolute requirements are exact output contracts; same basename in a
# different directory is not enough.
if artifact.startswith("/") or required.startswith("/"):
return False
return Path(artifact).name == Path(required).name
return artifact_identity(artifact, workspace) == artifact_identity(required, workspace)
def _explicit_tool_paths(tool: str, command: str) -> list[str]:
@@ -462,21 +461,21 @@ def _explicit_tool_paths(tool: str, command: str) -> list[str]:
except (TypeError, json.JSONDecodeError):
args = None
if isinstance(args, Mapping):
path = _clean_path(str(args.get("path") or ""))
path = str(args.get("path") or "").strip()
return [path] if path else []
# Keep compatibility with the legacy ``path\ncontent`` transport.
path = _clean_path(str(command or "").splitlines()[0] if command else "")
path = (str(command or "").splitlines()[0] if command else "").strip()
return [path] if path else []
if tool == "edit_file":
try:
args = json.loads(command or "{}")
except (TypeError, json.JSONDecodeError):
return []
path = _clean_path(str(args.get("path") or "")) if isinstance(args, dict) else ""
path = str(args.get("path") or "").strip() if isinstance(args, dict) else ""
return [path] if path else []
if tool == "apply_patch":
return [
_clean_path(match.group(1))
match.group(1).strip()
for match in re.finditer(r"^\*\*\* (?:Add|Update|Delete) File:\s*(.+)$", command or "", re.MULTILINE)
if _clean_path(match.group(1))
]
@@ -549,13 +548,13 @@ def _command_text(value: str) -> str:
def _matches_declared_verifier(command: str, expected: Sequence[str]) -> bool:
actual = " ".join(_command_text(command).split())
actual = executable_words(_command_text(command))
if not actual:
return False
return any(
normalized == actual or normalized in actual
normalized == actual
for item in expected
if (normalized := " ".join(str(item or "").split()))
if (normalized := executable_words(str(item or "")))
)
@@ -574,6 +573,7 @@ class EvidenceLedger:
def __init__(self, requirements: CompletionRequirements | None = None) -> None:
self.requirements = requirements or CompletionRequirements()
self.events: list[EvidenceEvent] = []
self._verification_versions: dict[str, str] = {}
@classmethod
def from_tool_events(
@@ -611,6 +611,10 @@ class EvidenceLedger:
"command_sha256": _digest(command),
"output_sha256": _digest(output),
}
action_id = str(source.get('action_id') or '')
execution_id = str(source.get('execution_id') or '')
if action_id:
payload.update(action_id=action_id, execution_id=execution_id)
evidence = EvidenceEvent(
event_id=_event_id(payload, len(self.events)),
kind=kind,
@@ -623,6 +627,11 @@ class EvidenceLedger:
command_sha256=payload["command_sha256"],
output_sha256=payload["output_sha256"],
detail=detail,
action_id=action_id,
execution_id=execution_id,
artifact_id=artifact_identity(artifact_path, self.requirements.workspace_root) if artifact_path else '',
verification_id=('verification-' + _event_id(payload, len(self.events)))
if kind in {EvidenceKind.VERIFIER_RESULT, EvidenceKind.ARTIFACT_VALIDATION} else '',
)
self.events.append(evidence)
return evidence
@@ -631,8 +640,12 @@ class EvidenceLedger:
tool = str(event.get("tool") or "")
command = str(event.get("command") or "")
exit_code = event.get("exit_code")
authoritative = isinstance(exit_code, int) and not isinstance(exit_code, bool)
success = authoritative and exit_code == 0
authoritative = (
isinstance(exit_code, int) and not isinstance(exit_code, bool)
and not event.get("blocked") and not event.get("approval_required")
and event.get("execution_attempted") is not False
)
success = authoritative and exit_code == 0 and not event.get('error')
if not authoritative:
success = not bool(event.get("error"))
self._append(
@@ -644,7 +657,11 @@ class EvidenceLedger:
explicit_paths = _explicit_tool_paths(tool, command)
mutation_paths = list(explicit_paths)
if command_has_mutation_effect(command) and tool not in {
observed_changes = event.get('artifact_changes')
if isinstance(observed_changes, list) and tool in {'bash', 'python', 'host_shell'}:
mutation_paths.extend(path for path in self.requirements.required_artifacts
if artifact_identity(path, self.requirements.workspace_root) in observed_changes)
elif command_has_mutation_effect(command) and tool not in {
"write_file",
"edit_file",
"apply_patch",
@@ -657,7 +674,7 @@ class EvidenceLedger:
)
seen_paths: set[str] = set()
for path in mutation_paths:
path = _clean_path(path)
path = str(path or '').strip()
if not path or path in seen_paths:
continue
seen_paths.add(path)
@@ -675,12 +692,12 @@ class EvidenceLedger:
except (TypeError, json.JSONDecodeError):
read_args = None
read_path = (
_clean_path(str(read_args.get("path") or ""))
str(read_args.get("path") or "").strip()
if isinstance(read_args, Mapping)
else ""
else command.strip() if read_args is None else ""
)
if read_path and any(
_artifact_path_matches_required(read_path, required)
_artifact_path_matches_required(read_path, required, self.requirements.workspace_root)
for required in self.requirements.required_artifacts
):
self._append(
@@ -692,10 +709,13 @@ class EvidenceLedger:
detail="post-write artifact inspection",
)
if _TEST_COMMAND_RE.search(_command_text(command)) or _matches_declared_verifier(
if tool in {"bash", "host_shell"} and (command_is_test(_command_text(command)) or _matches_declared_verifier(
command,
self.requirements.verifier_commands,
):
)):
if authoritative:
versions = event.get('artifact_versions')
self._verification_versions = dict(versions) if isinstance(versions, Mapping) else {}
self._append(
kind=EvidenceKind.VERIFIER_RESULT,
success=success,
@@ -703,7 +723,7 @@ class EvidenceLedger:
source=event,
detail="executable test/verifier command",
)
elif _VALIDATION_COMMAND_RE.search(command) and not mutation_paths:
elif tool in {"bash", "host_shell"} and is_validation_command(command) and not mutation_paths:
for path in self.requirements.required_artifacts:
if _path_is_mentioned(command, path):
self._append(
@@ -767,6 +787,19 @@ class EvidenceLedger:
(latest_verifier.event_id,),
)
if latest_verifier and self.requirements.workspace_root:
for path in self.requirements.required_artifacts:
identity = artifact_identity(path, self.requirements.workspace_root)
expected = self._verification_versions.get(identity)
if expected in {'unobserved', 'missing-or-unreadable'}:
return CompletionDecision(CompletionStatus.BLOCKED, False,
'artifact version could not be established for verification',
(latest_verifier.event_id,))
if expected is not None and expected != artifact_version(path, self.requirements.workspace_root):
return CompletionDecision(CompletionStatus.BLOCKED, False,
'artifact content changed after verification',
(latest_verifier.event_id,))
satisfied_ids: list[str] = []
missing: list[str] = []
workspace_root = str(self.requirements.workspace_root or "").strip()
@@ -774,7 +807,7 @@ class EvidenceLedger:
matches = [
event for event in self.events
if event.kind == EvidenceKind.ARTIFACT_MUTATION
and _artifact_path_matches_required(event.artifact_path, required)
and _artifact_path_matches_required(event.artifact_path, required, self.requirements.workspace_root)
]
authoritative = [
event for event in matches
@@ -793,10 +826,11 @@ class EvidenceLedger:
and latest.tool in {"bash", "python"}
)
filesystem_missing = False
if latest_success is not None and workspace_root and required.startswith("/workspace/"):
if latest_success is not None and workspace_root:
try:
root = Path(workspace_root).resolve()
candidate = (root / required.removeprefix("/workspace/")).resolve()
identity = artifact_identity(required, workspace_root)
candidate = (root / identity.removeprefix('workspace:')).resolve() if identity.startswith('workspace:') else Path(required).resolve()
candidate.relative_to(root)
filesystem_missing = not workspace_artifact_is_usable(candidate)
except (OSError, RuntimeError, ValueError):
@@ -852,14 +886,14 @@ class EvidenceLedger:
if event.kind == EvidenceKind.ARTIFACT_MUTATION
and event.authoritative
and event.success
and _artifact_path_matches_required(event.artifact_path, required)
and _artifact_path_matches_required(event.artifact_path, required, self.requirements.workspace_root)
]
matching_validations = [
(index, event)
for index, event in enumerate(self.events)
if event.kind == EvidenceKind.ARTIFACT_VALIDATION
and event.authoritative
and _artifact_path_matches_required(event.artifact_path, required)
and _artifact_path_matches_required(event.artifact_path, required, self.requirements.workspace_root)
]
if not matching_validations:
continue
@@ -882,6 +916,10 @@ class EvidenceLedger:
current_validation_ids.append(latest_validation.event_id)
if self.requirements.verifier_required and latest_verifier is None:
if self.requirements.executable_verifier_available:
return CompletionDecision(CompletionStatus.BLOCKED, False,
'the request requires an executable verifier result',
tuple(satisfied_ids))
validation_ids: list[str] = []
for required in self.requirements.required_artifacts:
matching_validation = [
@@ -890,7 +928,7 @@ class EvidenceLedger:
if event.kind == EvidenceKind.ARTIFACT_VALIDATION
and event.authoritative
and event.success
and _artifact_path_matches_required(event.artifact_path, required)
and _artifact_path_matches_required(event.artifact_path, required, self.requirements.workspace_root)
]
latest_validation = matching_validation[-1] if matching_validation else None
if latest_validation is None or latest_validation[0] < latest_mutation_index: