fix(effects): close independent review correctness gaps

This commit is contained in:
Alexandre Teixeira
2026-10-03 02:55:31 +01:00
parent da4bf3531f
commit 7563d859bc
8 changed files with 201 additions and 22 deletions
+36
View File
@@ -98,6 +98,42 @@ async def test_disabled_page_operations_never_observe_select_or_execute(producer
assert old.target_id != producer.target
@pytest.mark.parametrize("reason,expected", [
(browser.PAGE_FAILURE, browser.PAGE_FAILURE),
("Unrelated resource identity changed", "resource_identity_denied"),
(browser.PAGE_FAILURE + ": arbitrary detail", "resource_identity_denied"),
])
async def test_dispatch_boundary_preserves_only_native_browser_page_failure(producer, tmp_path, monkeypatch, reason, expected):
from src import tool_execution
from src.agent_runtime.effect_log import EffectLog
from src.agent_runtime.journal import ActionJournal, bind_journal
await observed(producer)
producer.calls.clear()
producer.cdp_calls.clear()
journal = ActionJournal()
journal.effects = EffectLog(journal.run_id, directory=tmp_path / "fx")
attempts = []
async def unsupported_operation(*args, **kwargs):
attempts.append(1)
if reason == browser.PAGE_FAILURE:
# The legacy server page helper raises the reserved identity error.
await PrivateBrowserTool()._capture_post_click_state()
raise ResourceIdentityError(reason)
monkeypatch.setattr(tool_execution, "_execute_tool_block_impl", unsupported_operation)
monkeypatch.setattr(tool_execution, "mark_dispatch", lambda: pytest.fail("Unsupported page operation dispatched"))
with bind_journal(journal):
_, result = await dispatch(authority(), "private_browser", '{"action":"session_info"}')
assert result["failure_kind"] == expected
if expected == browser.PAGE_FAILURE:
assert result["executed"] is False and result["retryable"] is False
assert journal.actions[0].execution_id is None
assert journal.effects.history().claims == ()
assert attempts == [1]
@pytest.mark.parametrize("args", [{"action": "batch", "commands": [["click", "@e1"]]},
{"action": "tab"}, {"action": "window"}, {"action": "frame"}, {"action": "connect"},
{"action": "click", "target": "--new-tab"}, {"action": "evaluate", "--cdp": "endpoint"},
+64 -1
View File
@@ -229,6 +229,65 @@ def test_patch_obligations_follow_exact_bindings(run, ws):
assert {o.predicate for o in claim.obligations} == {fx.Predicate.CONTENT_SHA256, fx.Predicate.ABSENT}
def test_deleted_file_read_emits_known_absence(run, ws):
(ws / "old.txt").write_text("old\n")
_, result = run("apply_patch", {"patch_text": "*** Begin Patch\n*** Delete File: old.txt\n*** End Patch"})
assert result["exit_code"] == 0
_, result = run("read_file", {"path": "old.txt"})
assert result["exit_code"] == 1 # The producer still reports a missing file.
history = run.journal.effects.history()
observation, = history.observations
assert observation.exists is False and observation.content_sha256 == ""
assert observation.coverage is fx.Coverage.COMPLETE
assert fx.predicate_holds(history.claims[0].obligations[0], observation) is True
assert verdicts(run.journal) == [fx.EffectVerdict.VERIFIED]
@pytest.mark.parametrize("failure", ["identity_mismatch", "replaced_path", "post_probe_replacement", "permission", "validation"])
def test_indeterminate_deleted_file_read_cannot_prove_absence(run, ws, monkeypatch, failure):
from src.agent_runtime import effect_adapters as adapters
from src.agent_runtime.resources import ResourceIdentityError
target = ws / "old.txt"
target.write_text("old\n")
run("apply_patch", {"patch_text": "*** Begin Patch\n*** Delete File: old.txt\n*** End Patch"})
if failure == "identity_mismatch":
target.write_text("replacement\n")
original = adapters._read_whole
def indeterminate(resource, limit):
if failure == "identity_mismatch":
target.unlink() # An existing binding disappearing is an identity failure.
elif failure == "replaced_path":
target.write_text("replacement\n")
elif failure == "post_probe_replacement":
validate = type(resource).validate
calls = []
def replace_after_probe(self):
calls.append(1)
if len(calls) == 2:
target.write_text("appeared after ENOENT\n")
return validate(self)
monkeypatch.setattr(type(resource), "validate", replace_after_probe)
elif failure == "permission":
def denied(path):
raise PermissionError("access denied")
monkeypatch.setattr(adapters.os, "lstat", denied)
else:
def invalid(self):
raise ResourceIdentityError("unresolved binding")
monkeypatch.setattr(type(resource), "validate", invalid)
return original(resource, limit)
monkeypatch.setattr(adapters, "_read_whole", indeterminate)
run("read_file", {"path": "old.txt"})
history = run.journal.effects.history()
assert history.observations == ()
assert verdicts(run.journal) == [fx.EffectVerdict.UNVERIFIED]
def test_listing_is_partial_and_does_not_verify_content(run):
run("write_file", {"path": "a.txt", "content": "hello\n"})
run("ls", {"path": "."})
@@ -243,7 +302,11 @@ def test_listing_is_partial_and_does_not_verify_content(run):
{"action": "snapshot", "page": "t1"},
{"action": "evaluate", "page": "t1", "script": "1"},
])
def test_browser_page_operations_stay_fail_closed_with_effects(run, args):
def test_browser_page_operations_stay_fail_closed_with_effects(run, args, monkeypatch):
async def unexpected_dispatch(*args, **kwargs):
pytest.fail("Unsupported page operation reached execution")
monkeypatch.setattr(tool_execution, "_execute_tool_block_impl", unexpected_dispatch)
description, result = run("private_browser", args)
assert "UNSUPPORTED" in description
assert result["failure_kind"] == "browser_page_authority_unavailable" and result["executed"] is False
@@ -436,6 +436,35 @@ DISCLOSURE = ("External operation mcp__server__send_email reported success; any
"not independently verified.")
@pytest.mark.parametrize("answer", ["Here is the draft. \n\n", " \n\n"])
@pytest.mark.parametrize("final", [False, True])
async def test_streaming_external_disclosure_survives_trailing_whitespace(store, monkeypatch, answer, final):
from src.agent_runtime.completion import completion_answer, with_completion_gate
from src.agent_runtime.journal import current_journal
expected = []
@with_completion_gate
async def stream(messages):
journal = current_journal()
journal.effects = EffectLog(journal.run_id, directory=store)
remote_act(journal, monkeypatch, result={"stdout": "ok", "stderr": "", "exit_code": 0})
ledger = _ledger(journal, CompletionRequirements())
expected.append(completion_answer(answer, ledger, ledger.evaluate())[0])
yield "data: " + json.dumps({"type": "final_response", "content": answer} if final else {"delta": answer}) + "\n\n"
yield "data: " + json.dumps({"type": "metrics", "data": {"round_texts": [answer]}}) + "\n\n"
events = [json.loads(chunk[6:]) async for chunk in stream([])]
disclosure = next(event["delta"] for event in events if event.get("delta") != answer and "delta" in event)
assert disclosure == ("\n\n" if answer.strip() else "") + DISCLOSURE
visible = "".join(event.get("delta", event.get("content", "")) for event in events)
assert visible.count(DISCLOSURE) == 1
metrics = next(event["data"] for event in events if event.get("type") == "metrics")
assert metrics["round_texts"] == expected
assert expected[0].count(DISCLOSURE) == 1
assert metrics["completion_gate"]["answer_replaced"] is False
def test_reported_external_mutation_cannot_complete_as_satisfied(tmp_path, store, monkeypatch):
from src.agent_evidence import EXTERNAL_EFFECT_UNVERIFIED
from src.agent_runtime.completion import completion_answer
+16 -2
View File
@@ -615,10 +615,24 @@ async def test_approved_resource_cannot_migrate_to_another_request(tmp_path):
assert result["failure_kind"] == "resource_identity_denied"
async def test_missing_approval_resource_snapshot_cannot_be_reconstructed(tmp_path):
async def test_missing_approval_resource_snapshot_cannot_be_reconstructed(tmp_path, monkeypatch):
grant = authority(tmp_path, "read_file")
def unavailable(*args, **kwargs):
raise PermissionError("Cannot establish the proposal's resource identity")
with monkeypatch.context() as patch:
patch.setattr("src.agent_runtime.resource_binding.resolve_filesystem_operation", unavailable)
exact, security = approval(grant, "read_file", "missing")
assert exact.pending.resource_operation is None
(tmp_path / "missing").write_text("appeared after proposal")
_, result = await dispatch(grant, "read_file", "missing", exact_approval=exact, security_context=security)
assert result["failure_kind"] == "resource_identity_denied"
async def test_approved_absent_read_cannot_bind_a_file_that_appeared(tmp_path):
grant = authority(tmp_path, "read_file")
exact, security = approval(grant, "read_file", "missing")
assert exact.pending.resource_operation is None
assert exact.pending.resource_operation.bindings[0].resource.identity is None
(tmp_path / "missing").write_text("appeared after proposal")
_, result = await dispatch(grant, "read_file", "missing", exact_approval=exact, security_context=security)
assert result["failure_kind"] == "resource_identity_denied"