jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-27 08:12:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(harness): ship coherent vendor harness cluster

Browse Source 
Rebuild PR C to ship the vendor/photocraft harness as a coherent unit
(vendor implementation + vendor tests) instead of a per-file ahead/behind
mix that produced broken imports and 6+ test failures.

- Flip 36 harness impl/route/test files back to vendor (the per-file
  line-set policy misclassified user-rewritten files as 'behind').
- Flip core/models.py + src/tool_approval_scopes.py to vendor: vendor's
  harness (authority.py, turn_contract.py, clean_agent_preview.py) depends
  on vendor's older approval-scope API; upstream added
  CHAT_SESSION_APPROVAL_SIGNATURE_FIELD that vendor's core/models.py lacks.
- Add src/shutdown.py (new vendor infra) required by routes/chat_routes.py.
- Keep src/agent_tools/__init__.py at upstream (editor_tools registration
  lives in PR B).

Upstream-only security divergence preserved in C_upstream_delta.patch and
flagged for integration-time reconciliation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
pewdiepie-archdaemon
2026-10-13 13:59:04 +00:00
co-authored by Claude Opus 5.5
parent 364229f2dc
commit 9bdea753ae
39 changed files with 804 additions and 3076 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent_loop.py
+282 -909
View File
File diff suppressed because it is too large Load Diff
src/agent_runtime/resources.py
+6 -9
View File
@@ -114,9 +114,11 @@ def _control_plane_snapshot():
uploader = get_upload_handler()
if uploader is not None and isinstance(getattr(uploader, "upload_dir", None), (str, os.PathLike)):
protected.add(canonical_root(Path(uploader.upload_dir) / "uploads.json"))
# Prefix protection covers the complete runtime trees. Public policy denies
# every regular hardlink alias, so collecting all child inodes here would
# add an unbounded recursive state inventory without widening protection.
for directory in job_dirs:
jobs = Path(directory)
if jobs.exists():
# Uninspectable state fails closed; hardlinks retain object identity.
protected.update(canonical_root(p) for p in jobs.rglob("*") if p.is_file())
protected.update(canonical_root(getattr(constants, name) + suffix)
for name in ("APP_DB", "SCHEDULED_EMAILS_DB", "EMAIL_CACHE_DB")
for suffix in ("-wal", "-shm", "-journal"))
@@ -129,7 +131,7 @@ def _control_plane_snapshot():
except FileNotFoundError:
continue
identities.add((observed.st_dev, observed.st_ino))
# Effect directories are protected by prefix without inventorying children.
# Effect directories join the prefix set only after the recursive inventory.
return frozenset(job_dirs | effect_dirs), frozenset(protected), frozenset(identities)
@@ -137,11 +139,6 @@ def _control_plane_path(path, *, snapshot=None):
# A scan-local snapshot bounds repeated hardlink checks. Ordinary resource
# resolution always observes fresh state. Neither form is an atomic kernel
# access policy, and snapshots must never survive a workspace guard call.
# Public state and hardlink denial also applies to sealed resources. Lazy
# import avoids coupling inert identity definitions to dispatcher startup.
from src.tool_execution import _is_app_state_path, _is_hardlinked_regular_file
if _is_sensitive_path(path) or _is_app_state_path(path) or _is_hardlinked_regular_file(path):
return True
directories, protected, identities = _control_plane_snapshot() if snapshot is None else snapshot
if any(Path(path).is_relative_to(directory) for directory in directories) or path in protected:
return True
src/agent_tools/__init__.py
-5
View File
@@ -29,7 +29,6 @@ from .interaction_tools import AskUserTool, UpdatePlanTool
from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool
from .bg_job_tools import ManageBgJobsTool
from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool
from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool
from .admin_tools import (
ADMIN_TOOL_HANDLERS,
do_manage_endpoints, do_manage_mcp, do_manage_webhooks,
@@ -73,10 +72,6 @@ TOOL_HANDLERS = {
"list_sessions": ListSessionsTool().execute,
"send_to_session": SendToSessionTool().execute,
"manage_session": ManageSessionTool().execute,
"plan_edit": PlanEditTool().execute,
"run_editor_command": RunEditorCommandTool().execute,
"get_editor_screenshot": GetEditorScreenshotTool().execute,
"ai_edit_image": AiEditImageTool().execute,
}
# Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings).
TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS)
src/agent_tools/filesystem_tools.py
+114 -572
View File
@@ -3,9 +3,8 @@ import json
import os
import re
import difflib
import secrets
import fnmatch
import shutil
import time
import tempfile
from typing import Optional, Dict, Any, Tuple, List
@@ -19,8 +18,6 @@ _CODENAV_SKIP_DIRS = frozenset({
})
_CODENAV_MAX_HITS = 200
_CODENAV_MAX_LINE = 400
_GREP_TIMEOUT_SECONDS = 20
_GREP_STDERR_PREFIX = 20_000
_STRUCTURED_DOCUMENT_SUFFIXES = frozenset({
".doc", ".docx", ".epub", ".pdf", ".pptx", ".xls", ".xlsx",
})
@@ -91,79 +88,6 @@ def _glob_to_regex(pat: str) -> "re.Pattern":
i += 1
return re.compile("".join(out))
def _validate_grep_descriptor(descriptor):
"""Revalidate an inert parent snapshot without inherited ContextVars."""
path, identity, ancestors = descriptor
if os.path.islink(path) or os.path.realpath(path) != path:
raise ValueError("grep: resource path changed")
for parent, observed in ancestors:
info = os.stat(parent, follow_symlinks=False)
if not os.path.isdir(parent) or (info.st_dev, info.st_ino) != tuple(observed):
raise ValueError("grep: resource ancestor changed")
info = os.stat(path, follow_symlinks=False)
import stat
if (not stat.S_ISREG(info.st_mode) or info.st_nlink > 1
or (info.st_dev, info.st_ino) != tuple(identity)):
raise ValueError("grep: resource identity changed")
def _python_grep_worker(payload: dict, output_queue) -> None:
"""Spawn-safe fallback grep worker used when ripgrep is unavailable.
Keep this at module scope: a frozen Windows executable cannot safely be
relaunched as ``sys.executable -c ...``, while multiprocessing can invoke a
top-level target through its frozen-process bootstrap.
"""
try:
flags = re.IGNORECASE if payload["ignore_case"] else 0
try:
regex = re.compile(payload["pattern"], flags)
glob_regex = (
_glob_to_regex(payload["glob"].replace("\\", "/"))
if payload["glob"]
else None
)
except re.error as exc:
output_queue.put(("error", f"grep: bad pattern: {exc}"))
return
max_hits = payload["max_hits"]
hits = 0
for descriptor in payload["files"]:
if hits >= max_hits:
break
path, identity, ancestors = descriptor
_validate_grep_descriptor(descriptor)
relative = os.path.relpath(path, payload["base"]).replace(os.sep, "/")
if glob_regex and not (
glob_regex.fullmatch(relative) or glob_regex.fullmatch(os.path.basename(path))
):
continue
try:
with open(path, "r", encoding="utf-8", errors="strict") as handle:
info = os.fstat(handle.fileno())
if (info.st_dev, info.st_ino) != tuple(identity) or info.st_nlink > 1:
raise ValueError("grep: resource identity changed before read")
for number, line in enumerate(handle, 1):
if regex.search(line):
output_queue.put(("match", path, number, line.rstrip()[:_CODENAV_MAX_LINE]))
hits += 1
if hits >= max_hits:
break
except UnicodeDecodeError:
continue
except OSError as error:
output_queue.put(("error", f"grep: {error}"))
return
output_queue.put(("done",))
except BaseException as exc:
try:
output_queue.put(("error", f"grep: fallback worker failed: {exc}"))
except BaseException:
pass
def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
if old == new:
return None
@@ -344,82 +268,32 @@ class ReadFileTool:
data = data[:MAX_READ_CHARS] + f"\n... [truncated at {MAX_READ_CHARS} chars]"
return {"output": data, "exit_code": 0}
def _write_new_file_without_overwrite(path: str, body: str) -> None:
"""Publish a new file without exposing a writable placeholder at its path.
Stage beside the destination, then hard-link it into place. The link is
atomic and fails if another writer created the destination first.
"""
directory = os.path.dirname(path) or "."
temporary_path = os.path.join(
directory, f".odysseus-write-{secrets.token_hex(16)}.tmp"
)
fd = os.open(
temporary_path,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o666,
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as temporary_file:
fd = None
temporary_file.write(body)
os.link(temporary_path, path)
finally:
if fd is not None:
os.close(fd)
try:
os.unlink(temporary_path)
except FileNotFoundError:
pass
class _EmptyBodyWouldTruncate(Exception):
"""Raised inside the write thread when an undeclared empty body is about to
replace a file that holds bytes. Carries the size at risk so the caller can be
told what it would have lost (#6414)."""
def __init__(self, path: str, existing_bytes: int):
super().__init__(path)
self.path = path
self.existing_bytes = existing_bytes
def _parse_write_intent(content: str) -> tuple[str, str, bool, bool]:
"""Classify the original transport, before binding or source normalization.
The returned clear flag is derived here, never from a caller-supplied key.
"""
if not isinstance(content, str):
raise ValueError("write_file: expected string arguments")
if content.lstrip().startswith("{"):
try:
args = json.loads(content)
except (TypeError, ValueError) as error:
raise ValueError("write_file: expected valid JSON arguments") from error
if not isinstance(args, dict):
raise ValueError("write_file: expected a JSON object")
path, body = args.get("path"), args.get("content")
if not isinstance(body, str):
raise ValueError("write_file: content required and must be a string")
if not isinstance(path, str):
raise ValueError("write_file: path required and must be a string")
return path.strip(), body, True, not body.strip()
path, delimiter, body = content.partition("\n")
return path.strip(), body, bool(delimiter), False
class WriteFileTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _display_tool_path, _resolve_tool_path
from src.agent_runtime.resource_binding import active_resource_operation
bound = active_resource_operation()
original = bound.operation.input if bound is not None else content
try:
_, _, has_section, declared_clear = _parse_write_intent(original)
raw_path, body, _, _ = _parse_write_intent(content)
except ValueError as error:
return {"error": str(error), "exit_code": 1}
from src.tool_execution import _display_tool_path, _resolve_tool_path, _resolve_search_root, _truncate
lines = content.split("\n", 1)
raw_path = lines[0].strip()
body = lines[1] if len(lines) > 1 else ""
# Decode JSON-object args (the fenced inline-args shape
# ```write_file {"path": "...", "content": "..."}```), matching
# ReadFileTool above. Without this the whole JSON string becomes the
# path and the file is written under a garbage name. This is the live
# path: there is no filesystem MCP server, so write_file always runs
# here via _direct_fallback, not through _build_mcp_args.
_stripped = content.strip()
if _stripped.startswith("{"):
try:
_a = json.loads(_stripped)
if not isinstance(_a, dict):
return {"error": "write_file: expected a JSON object", "exit_code": 1}
raw_path_value = _a.get("path")
body_value = _a.get("content")
raw_path = raw_path_value.strip() if isinstance(raw_path_value, str) else ""
if not isinstance(body_value, str):
return {"error": "write_file: content required", "exit_code": 1}
body = body_value
except (json.JSONDecodeError, TypeError, ValueError):
return {"error": "write_file: expected valid JSON arguments", "exit_code": 1}
if not raw_path:
return {"error": "write_file: path required", "exit_code": 1}
try:
@@ -449,6 +323,16 @@ class WriteFileTool:
"exit_code": 1,
"artifact_format_error": True,
}
if not body:
return {
"error": (
f"write_file: {path}: content required; refusing to create an "
"empty file. Call write_file again with the exact filename and "
"non-empty content. If you need a directory, create it from "
"bash/python instead."
),
"exit_code": 1,
}
# write_file is a UTF-8 text writer. Refuse to silently destroy an
# existing PDF, image, archive, or media artifact produced by a
# format-aware tool, especially after the agent has verified it.
@@ -463,10 +347,6 @@ class WriteFileTool:
"exit_code": 1,
"binary_artifact_preserved": target_existed,
}
if not has_section:
return {"error": "write_file: content required; missing content section", "exit_code": 1}
if raw_path.endswith(("/", "\\")) or os.path.isdir(path):
return {"error": "write_file: target is a directory", "exit_code": 1}
# This writer truncates in place. Once that stage is reached, a failure
# may leave a partial file; report it so effect evidence stays honest.
attempted = []
@@ -481,67 +361,21 @@ class WriteFileTool:
d = os.path.dirname(path)
if d:
os.makedirs(d, exist_ok=True)
if not body.strip() and not declared_clear:
# Empty/whitespace-only writes to an already-empty file are no-ops.
# Avoid reopening in truncating mode: another writer may have added
# data since the read above.
if os.path.isfile(path):
existing_bytes = os.path.getsize(path)
if existing_bytes > 0:
raise _EmptyBodyWouldTruncate(path, existing_bytes)
return old, 0
try:
if body:
# Publish whitespace content atomically. Writing it
# after exclusive creation could overwrite bytes from
# a writer that filled the new placeholder meanwhile.
attempted.append(True)
_write_new_file_without_overwrite(path, body)
else:
# An exact empty body needs no staged data, so create
# the file exclusively and never write through it.
attempted.append(True)
with open(path, "x", encoding="utf-8"):
pass
except FileExistsError:
if os.path.isfile(path):
existing_bytes = os.path.getsize(path)
if existing_bytes > 0:
raise _EmptyBodyWouldTruncate(path, existing_bytes)
return old, 0
raise
return old, len(body)
attempted.append(True)
with open(path, "w", encoding="utf-8") as f:
f.write(body)
return old, len(body)
old_content, size = await asyncio.to_thread(_write)
except _EmptyBodyWouldTruncate as e:
clear_call = json.dumps({"path": raw_path, "content": ""})
return {
"error": (
f"write_file: refused to write an empty body over {e.path} — it holds "
f"{e.existing_bytes} bytes, which the write would have destroyed, so "
f"the file is unchanged. To clear it on purpose, resend with an "
f"explicit empty content: {clear_call}"
),
"exit_code": 1,
}
except PermissionError:
return {"error": f"write_file: {path}: permission denied", "exit_code": 1,
**({"mutation_attempted": True} if attempted else {})}
except OSError as e:
return {"error": f"write_file: {path}: {e}", "exit_code": 1,
**({"mutation_attempted": True} if attempted else {})}
committed_body = old_content if size == 0 and not declared_clear and not body.strip() else body
diff = _unified_diff(old_content, committed_body, path)
diff = _unified_diff(old_content, body, path)
result = {
"output": (f"Wrote {size} bytes to {_display_tool_path(path)}" if attempted else
f"No write performed for {_display_tool_path(path)} (implicit empty body)"),
"output": f"Wrote {size} bytes to {_display_tool_path(path)}",
"exit_code": 0,
**({"write_noop": True} if not attempted else {}),
}
if diff:
result["diff"] = diff
@@ -785,7 +619,7 @@ def _apply_patch_hunks(original: str, hunks: List[List[str]], label: str) -> str
class LsTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _display_tool_path, _is_denied_tool_path, _resolve_search_root, _truncate
from src.tool_execution import _display_tool_path, _resolve_search_root, _truncate
raw_path = ""
_s = (content or "").strip()
if _s.startswith("{"):
@@ -801,18 +635,15 @@ class LsTool:
return {"error": f"ls: {e}", "exit_code": 1}
def _ls():
from src.agent_runtime.resources import _control_plane_snapshot
if not os.path.isdir(root):
return None, f"ls: {root}: not a directory"
rows = []
snapshot = _control_plane_snapshot()
try:
with os.scandir(root) as it:
for entry in it:
if entry.name.startswith("."):
continue
if (_is_denied_tool_path(os.path.realpath(entry.path), snapshot=snapshot)
or not _visible_bound_resource(entry.path)):
if not _visible_bound_resource(entry.path):
continue
try:
is_dir = entry.is_dir(follow_symlinks=False)
@@ -841,8 +672,6 @@ class GlobTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import (
_SENSITIVE_BASENAMES,
_can_traverse_tool_path,
_is_denied_tool_path,
_display_tool_path,
_is_sensitive_path,
_resolve_tool_path,
@@ -867,12 +696,10 @@ class GlobTool:
return {"error": f"glob: {e}", "exit_code": 1}
def _glob():
from src.agent_runtime.resources import _control_plane_snapshot
base = os.path.abspath(root)
if not os.path.isdir(base):
return None, f"glob: {root}: not a directory"
rbase = os.path.realpath(base)
snapshot = _control_plane_snapshot()
norm_pat = pattern.replace("\\", "/")
# Fast path: literal pattern (no wildcards) → direct path lookup.
if not any(c in norm_pat for c in "*?["):
@@ -889,8 +716,7 @@ class GlobTool:
# .ssh/id_rsa, …) falls through to the walk, which skips it —
# otherwise glob would surface secret paths that read_file /
# grep already refuse to touch.
if (inside and os.path.exists(cand) and not _is_denied_tool_path(cand, snapshot=snapshot)
and _visible_bound_resource(cand)):
if inside and os.path.exists(cand) and not _is_sensitive_path(cand) and _visible_bound_resource(cand):
return [cand], None
# Literal not at exact path — fall through to walk so
# e.g. "foo.py" still matches at any depth (like rglob).
@@ -900,18 +726,13 @@ class GlobTool:
cap = _CODENAV_MAX_HITS * 5
try:
for dp, dns, fns in os.walk(base):
if not _can_traverse_tool_path(os.path.realpath(dp), snapshot=snapshot):
dns[:] = []
continue
# Prune skipped dirs before descending (unlike rglob which
# descends first then filters — fatal on large node_modules).
# Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
# never enumerates the keys/tokens inside them.
dns[:] = [
d for d in dns
if d not in _CODENAV_SKIP_DIRS
and d not in _SENSITIVE_BASENAMES
and _can_traverse_tool_path(os.path.realpath(os.path.join(dp, d)), snapshot=snapshot)
if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
]
for name in fns + dns:
full = os.path.join(dp, name)
@@ -919,8 +740,7 @@ class GlobTool:
if regex.fullmatch(rel) or regex.fullmatch(name):
# Skip deny-listed sensitive files (.env, id_rsa,
# known_hosts, …) the same way grep does.
if (_is_denied_tool_path(os.path.realpath(full), snapshot=snapshot)
or not _visible_bound_resource(full)):
if _is_sensitive_path(os.path.realpath(full)) or not _visible_bound_resource(full):
continue
try:
mtime = os.stat(full).st_mtime
@@ -947,14 +767,10 @@ class GlobTool:
class GrepTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import (
_SENSITIVE_BASENAMES,
_SENSITIVE_FILE_PATTERNS,
_agent_readable_data_subdirs,
_is_denied_tool_path,
_display_tool_path,
_can_traverse_tool_path,
_is_sensitive_path,
_path_within,
_resolve_tool_path,
_resolve_search_root,
_truncate,
)
@@ -983,357 +799,83 @@ class GrepTool:
return {"error": f"grep: {e}", "exit_code": 1}
def _grep():
import multiprocessing
import queue
import subprocess
import threading
from src.agent_runtime.resources import _control_plane_snapshot
from src.constants import DATA_DIR
rg = shutil.which("rg")
real_root = os.path.realpath(root)
deadline = time.monotonic() + _GREP_TIMEOUT_SECONDS
base = real_root if os.path.isdir(real_root) else os.path.dirname(real_root)
import re as _re
import shutil
from src.agent_runtime.resource_binding import active_resource_operation
bound = active_resource_operation()
if bound is not None:
bound.validate()
files = []
snapshot = _control_plane_snapshot()
def check_deadline():
if time.monotonic() >= deadline:
raise TimeoutError("grep: timed out")
def observe_file(path):
check_deadline()
if os.path.islink(path):
return
canonical = os.path.realpath(path)
if not _path_within(canonical, base) or _is_denied_tool_path(canonical, snapshot=snapshot):
return
if bound is not None:
try:
bound.resolve_path(canonical)
except ValueError:
# Intentionally denied publication control-plane paths
# are omitted before any producer is allowed to read.
return
info = os.stat(canonical, follow_symlinks=False)
if not os.path.isfile(canonical):
return
ancestors = []
parent = os.path.dirname(canonical)
while _path_within(parent, base):
observed = os.stat(parent, follow_symlinks=False)
ancestors.append((parent, (observed.st_dev, observed.st_ino)))
if parent == base:
break
parent = os.path.dirname(parent)
files.append((canonical, (info.st_dev, info.st_ino), tuple(ancestors)))
if len(files) > 100_000:
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
try:
if os.path.islink(root):
raise ValueError("grep: symlink search root is not allowed")
if os.path.isfile(real_root):
observe_file(real_root)
elif os.path.isdir(real_root):
pending_directories = [real_root]
enumerated = 0
while pending_directories:
check_deadline()
directory = pending_directories.pop()
if not _can_traverse_tool_path(directory, snapshot=snapshot):
continue
with os.scandir(directory) as entries:
for entry in entries:
check_deadline()
enumerated += 1
if enumerated > 100_000:
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
if entry.is_symlink():
continue
canonical = os.path.realpath(entry.path)
if not _path_within(canonical, base):
raise ValueError("grep: directory identity changed during enumeration")
if entry.is_dir(follow_symlinks=False):
if (entry.name not in _CODENAV_SKIP_DIRS
and _can_traverse_tool_path(canonical, snapshot=snapshot)
and (bound is None or _is_denied_tool_path(canonical, snapshot=snapshot)
or _visible_bound_resource(canonical))):
pending_directories.append(canonical)
else:
observe_file(entry.path)
else:
raise FileNotFoundError(f"grep: {root}: not found")
check_deadline()
except (OSError, ValueError) as error:
return None, str(error) if str(error).startswith("grep:") else f"grep: {error}"
descriptors = {record[0]: record for record in files}
targets = list(descriptors)
lines: list[str] = []
def parse_rg_result(raw: str) -> Optional[str]:
try:
record = json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
if record.get("type") != "match":
return None
data = record.get("data") or {}
path = (data.get("path") or {}).get("text")
text_value = (data.get("lines") or {}).get("text")
number = data.get("line_number")
if not isinstance(path, str) or not isinstance(text_value, str):
return None
absolute = path if os.path.isabs(path) else os.path.join(base, path)
canonical = os.path.realpath(absolute)
if canonical not in descriptors:
raise ValueError("grep: producer returned an undeclared resource")
_validate_grep_descriptor(descriptors[canonical])
if bound is not None:
bound.resolve_path(canonical)
return f"{_display_tool_path(canonical)}:{number}:{text_value.rstrip()[:_CODENAV_MAX_LINE]}"
def run_rg(cmd: list[str]) -> Optional[str]:
try:
process = subprocess.Popen(
cmd,
cwd=base,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
except Exception as exc:
return f"grep: {exc}"
output: queue.Queue[Optional[str]] = queue.Queue(maxsize=max_hits + 2)
stderr_prefix: list[str] = []
stderr_size = 0
stop_reader = threading.Event()
def enqueue_stdout(value: Optional[str]) -> bool:
# The consumer stops at the result cap or deadline. Never
# leave a producer blocked on its bounded queue afterward.
while not stop_reader.is_set():
try:
output.put(value, timeout=0.05)
return True
except queue.Full:
continue
return False
def read_stdout() -> None:
assert process.stdout is not None
try:
for line in process.stdout:
if not enqueue_stdout(line.rstrip("\n")):
break
finally:
enqueue_stdout(None)
def read_stderr() -> None:
nonlocal stderr_size
assert process.stderr is not None
while True:
chunk = process.stderr.read(4096)
if not chunk:
break
if stderr_size < _GREP_STDERR_PREFIX:
kept = chunk[:_GREP_STDERR_PREFIX - stderr_size]
stderr_prefix.append(kept)
stderr_size += len(kept)
stdout_thread = threading.Thread(target=read_stdout, daemon=True)
stderr_thread = threading.Thread(target=read_stderr, daemon=True)
stdout_thread.start()
stderr_thread.start()
timed_out = False
capped = False
try:
while len(lines) < max_hits:
remaining = deadline - time.monotonic()
if remaining <= 0:
timed_out = True
break
try:
raw = output.get(timeout=remaining)
except queue.Empty:
timed_out = True
break
if raw is None:
break
parsed = parse_rg_result(raw)
if parsed and parsed not in lines:
lines.append(parsed)
capped = len(lines) >= max_hits
finally:
stop_reader.set()
if (timed_out or capped) and process.poll() is None:
process.terminate()
try:
remaining = max(0.01, deadline - time.monotonic())
return_code = process.wait(timeout=min(1, remaining))
except subprocess.TimeoutExpired:
process.kill()
return_code = process.wait()
stdout_thread.join()
stderr_thread.join()
if timed_out:
return "grep: timed out"
if not capped and return_code not in (0, 1):
detail = "".join(stderr_prefix).strip()
return f"grep: {detail or f'process exited {return_code}'}"
return None
if not os.path.exists(root):
return None, f"grep: search target not found: {_display_tool_path(root)}"
# The pathname-only fast path scans before individual resources can
# be checked. Bound searches must validate every file before read.
rg = None if active_resource_operation() is not None else shutil.which("rg")
if rg:
cmd = [rg, "--line-number", "--with-filename", "--no-heading", "--color=never",
"--max-count", str(max_hits)]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
try:
glob_regex = _glob_to_regex(glob_pat.replace("\\", "/"))
targets = [
target for target in targets
if glob_regex.fullmatch(os.path.relpath(target, base).replace(os.sep, "/"))
or glob_regex.fullmatch(os.path.basename(target))
]
except re.error:
pass
# Validate even when policy filtering leaves no search targets.
if not targets:
cmd_probe = [rg, "--json", "--no-config"]
if glob_pat:
cmd_probe += ["--glob", glob_pat]
cmd_probe += ["--regexp", pattern, "--", "-"]
error = run_rg(cmd_probe)
return (None, error) if error else ([], None)
relative_targets = [os.path.relpath(target, base) for target in targets]
for offset in range(0, len(relative_targets), 128):
if len(lines) >= max_hits:
break
cmd = [
rg, "--json", "--no-config", "--no-follow",
"--max-count", str(max_hits - len(lines)),
"--max-columns", str(_CODENAV_MAX_LINE),
"--max-columns-preview",
]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
cmd += ["--glob", glob_pat]
for sensitive_pattern in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!{sensitive_pattern}"]
for skipped_dir in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{skipped_dir}/**"]
cmd += ["--regexp", pattern, "--", *relative_targets[offset:offset + 128]]
try:
for path in targets[offset:offset + 128]:
check_deadline()
_validate_grep_descriptor(descriptors[path])
if bound is not None:
bound.resolve_path(path)
error = run_rg(cmd)
except (OSError, ValueError) as exc:
return None, f"grep: {exc}"
if error:
return None, error
return lines, None
# This runs inside asyncio.to_thread(), so forking would clone a
# multithreaded process and can deadlock. Spawn is platform-safe and
# PyInstaller-compatible via launcher's early freeze_support().
payload = {
"root": base,
"base": base,
"files": tuple(files),
"pattern": pattern,
"ignore_case": ignore_case,
"glob": glob_pat,
"max_hits": max_hits,
"skip_dirs": tuple(_CODENAV_SKIP_DIRS),
"sensitive_names": tuple(
set(_SENSITIVE_BASENAMES) | set(_SENSITIVE_FILE_PATTERNS)
),
}
try:
context = multiprocessing.get_context("spawn")
output_queue = context.Queue(maxsize=max_hits + 2)
worker = context.Process(
target=_python_grep_worker, args=(payload, output_queue)
)
worker.start()
except Exception as exc:
cmd += ["--glob", glob_pat]
# --iglob (not --glob) so the exclusion is case-insensitive:
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
# resolve to the same secret as their lowercase forms, and the
# Python fallback below already folds case via _is_sensitive_path.
for _pat in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!*{_pat}*"]
for _d in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{_d}/**"]
cmd += ["--regexp", pattern, root]
try:
output_queue.close()
except (NameError, OSError, ValueError):
pass
return None, f"grep: could not start fallback worker: {exc}"
error = None
completed = False
import subprocess
p = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
# ripgrep: 0 = matches, 1 = no matches, 2 = failed scan.
# Do not present invalid patterns or IO failures as absence.
if p.returncode not in (0, 1):
detail = (p.stderr or '').strip()[:1200]
return None, f"grep: search failed (exit {p.returncode}): {detail or 'no diagnostic available'}"
lines = [ln for ln in (p.stdout or "").splitlines() if ln][:max_hits]
return lines, None
except subprocess.TimeoutExpired:
return None, "grep: timed out"
except Exception as _e:
return None, f"grep: {_e}"
try:
while len(lines) < max_hits:
remaining = deadline - time.monotonic()
if remaining <= 0:
error = "grep: timed out"
break
try:
# Keep queue waits short enough to observe a spawn
# worker that dies during bootstrap/import before it
# can enqueue either an error or the done sentinel.
record = output_queue.get(timeout=min(0.05, remaining))
except queue.Empty:
if worker.is_alive():
rx = _re.compile(pattern, _re.IGNORECASE if ignore_case else 0)
except _re.error as _e:
return None, f"grep: bad pattern: {_e}"
hits = []
scan_errors = []
if os.path.isfile(root):
file_iter = [root]
else:
file_iter = []
for dp, dns, fns in os.walk(root, onerror=scan_errors.append):
dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
for fn in fns:
if glob_pat and not fnmatch.fnmatch(fn, glob_pat):
continue
worker.join(timeout=0)
try:
# A multiprocessing queue's feeder can make the
# final record visible at process-exit time. Give
# that record precedence over the exit status.
remaining = deadline - time.monotonic()
record = output_queue.get(
timeout=min(0.05, max(0, remaining))
)
except queue.Empty:
error = f"grep: fallback worker exited {worker.exitcode}"
break
if record[0] == "done":
completed = True
break
if record[0] == "error":
error = record[1]
break
_, path, number, text_value = record
canonical = os.path.realpath(path)
if canonical not in descriptors:
error = "grep: fallback returned an undeclared resource"
break
try:
_validate_grep_descriptor(descriptors[canonical])
if bound is not None:
bound.resolve_path(canonical)
except (OSError, ValueError) as exc:
error = f"grep: {exc}"
break
rendered = f"{_display_tool_path(canonical)}:{number}:{text_value}"
if rendered not in lines:
lines.append(rendered)
finally:
if completed:
worker.join(timeout=min(1, max(0.01, deadline - time.monotonic())))
if worker.is_alive():
worker.terminate()
worker.join(timeout=1)
if worker.is_alive():
worker.kill()
worker.join()
output_queue.close()
if error:
return None, error
if (not completed or worker.exitcode not in (0, None)) and len(lines) < max_hits:
return None, f"grep: fallback worker exited {worker.exitcode}"
return lines, None
file_iter.append(os.path.join(dp, fn))
for fp in file_iter:
if len(hits) >= max_hits:
break
try:
resolved_file = _resolve_tool_path(fp)
except ValueError:
# Apply the same workspace/sensitive-path checks to each
# discovered file, not just the initial search directory.
continue
try:
with open(resolved_file, "r", encoding="utf-8", errors="strict") as f:
for i, line in enumerate(f, 1):
if rx.search(line):
hits.append(f"{fp}:{i}:{line.rstrip()[:_CODENAV_MAX_LINE]}")
if len(hits) >= max_hits:
break
except UnicodeDecodeError:
continue
except OSError as error:
scan_errors.append(error)
if scan_errors:
return None, "grep: search incomplete; one or more files or directories could not be read"
return hits, None
lines, err = await asyncio.to_thread(_grep)
if err:
src/agent_tools/subprocess_tools.py
+7 -16
View File
@@ -11,7 +11,7 @@ import sys
import time
import json
from typing import Optional
from urllib.parse import urlparse, urlsplit, urlunsplit
from urllib.parse import urlparse
import httpx
@@ -76,14 +76,12 @@ def _resolve_fontfile_for_text(text: str) -> str:
async def _cancel_host_shell_bridge_request(
url: str, token: str, request_id: str,
) -> None:
if not token or not is_host_shell_bridge_url_allowed(url):
return
target = host_shell_bridge_endpoint_url(url, "/cancel")
base = url.rsplit("/", 1)[0]
try:
timeout = httpx.Timeout(5.0, connect=2.0, write=2.0, pool=2.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
await client.post(
target,
f"{base}/cancel",
json={"request_id": request_id},
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
@@ -218,12 +216,6 @@ def is_host_shell_bridge_url_allowed(url: str) -> bool:
return True
def host_shell_bridge_endpoint_url(url: str, path: str) -> str:
"""Replace a validated bridge URL's path without changing its authority."""
parsed = urlsplit(url)
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def _replace_workspace_alias(content: str, cwd: str) -> str:
"""Map virtual /workspace paths without corrupting absolute host paths."""
return re.sub(
@@ -690,7 +682,6 @@ class HostShellTool:
return {"error": "host_shell: invalid bridge URL", "exit_code": 1}
if not token:
return {"error": "host_shell: bridge token missing", "exit_code": 1}
run_url = host_shell_bridge_endpoint_url(url, "/run")
job_id = str(args.get("job_id") or "").strip()
if not command and not job_id:
@@ -725,9 +716,9 @@ class HostShellTool:
request_body["request_id"] = request_id
try:
async with httpx.AsyncClient(timeout=timeout + 5, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout + 5) as client:
resp = await client.post(
run_url,
url,
json=request_body,
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
@@ -755,7 +746,7 @@ class HostShellTool:
while time.monotonic() < deadline:
await asyncio.sleep(0.25)
poll = await client.post(
run_url,
url,
json={"job_id": auto_job_id},
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
src/app_initializer.py
+2 -31
View File
@@ -2,11 +2,10 @@
"""Initialize all application components and dependencies."""
import os
import logging
import stat
from typing import Dict, Any
from src.constants import (
DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR, AGENT_WORKSPACE_DIR,
DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR,
SESSIONS_FILE, DEFAULT_HOST, OPENAI_API_KEY
)
from src.memory import MemoryManager
@@ -31,35 +30,7 @@ def create_directories():
"""Create necessary directories if they don't exist."""
for directory in (DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR):
os.makedirs(directory, exist_ok=True)
# The model-controlled workspace must be a real child of DATA_DIR. Never
# follow a pre-existing symlink here: it would silently move the default
# native-file root outside the application volume before any resolver runs.
data_root = os.path.realpath(os.path.abspath(os.path.expanduser(DATA_DIR)))
workspace = os.path.abspath(os.path.expanduser(AGENT_WORKSPACE_DIR))
expected_workspace = os.path.join(data_root, "agent_workspace")
# Validate the real parent so a supported DATA_DIR bind/symlink works, but
# require the fixed internal carve-out name and reject a link at the model-
# controlled workspace entry itself.
if (
os.path.basename(workspace) != "agent_workspace"
or os.path.realpath(os.path.dirname(workspace)) != data_root
):
raise RuntimeError("agent workspace must be the canonical child of DATA_DIR")
if os.path.lexists(workspace):
mode = os.lstat(workspace).st_mode
if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode):
raise RuntimeError("agent workspace must be a real directory")
else:
os.mkdir(workspace, 0o700)
resolved_workspace = os.path.realpath(workspace)
if resolved_workspace != expected_workspace:
raise RuntimeError("agent workspace must be the canonical child of DATA_DIR")
try:
os.chmod(workspace, 0o700)
except OSError:
pass
def initialize_managers(base_dir: str, rag_manager=None) -> Dict[str, Any]:
"""
Initialize all manager and handler instances.
src/builtin_actions.py
+9 -32
View File
@@ -2297,10 +2297,10 @@ async def action_audit_skills(owner: str, **kwargs) -> Tuple[str, bool]:
async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
"""Background note-due scanner. Fires a reminder for any note whose
`due_date` falls in the current ±90-second window and hasn't been delivered
through its configured channel in the last 25 minutes.
`due_date` falls in the current ±5-minute window and hasn't been pinged
within the last 25 minutes. Mirrors `action_ping_events` for calendar.
Per-owner state: {note_id: {at, channel}}, with legacy timestamp support. Pruned
State (`data/note_pings.json`): {note_id: iso_ts_of_last_ping}. Pruned
on each run by dropping entries for notes that are gone/archived/replied.
"""
try:
@@ -2309,11 +2309,6 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
from datetime import datetime as _dt, timezone as _tz, timedelta as _td
from pathlib import Path as _P
from core.database import SessionLocal as _SL, Note as _N
from src.settings import load_settings
channel = load_settings().get("reminder_channel", "browser")
external_channel = channel in ("email", "ntfy", "webhook")
delivery_key = f"{channel}_sent" if external_channel else "browser_sent"
# Per-owner state file so cache-pruning doesn't cross-delete other
# users' entries (review C4). Legacy path kept as fallback so a
@@ -2378,27 +2373,20 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
due = _parse_due(n.due_date)
if not due:
continue
# Inside the due window?
# Inside the ±5min window?
if abs((due - now).total_seconds()) > window.total_seconds():
continue
# Recently pinged? Skip.
last = cache.get(n.id)
browser_recent = False
if last:
try:
last_channel = None
if isinstance(last, dict):
last_channel = last.get("channel")
last = last.get("at")
last_dt = _dt.fromisoformat(str(last))
if last_dt.tzinfo is None:
last_dt = last_dt.replace(tzinfo=_tz.utc)
if last_dt >= reping_cutoff:
if not external_channel or last_channel == channel:
continue
# Browser-only receipts do not prove external
# delivery, but the fallback must not be queued twice.
browser_recent = last_channel in (None, "browser")
continue
except Exception:
pass
# Compose + dispatch.
@@ -2422,26 +2410,15 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
body = "\n\n".join(p for p in body_parts if p) or title
try:
from routes.note_routes import dispatch_reminder
result = await dispatch_reminder(
await dispatch_reminder(
title=title, note_body=body, note_id=n.id,
owner=n.owner or owner or "",
queue_browser=not browser_recent,
)
if result.get("skipped"):
continue
if result.get(delivery_key):
sent.append(title)
else:
logger.warning("ping_notes: %s delivery failed for %s", channel, n.id)
cache[n.id] = now.isoformat()
sent.append(title)
except Exception as e:
logger.warning(f"ping_notes: dispatch failed for {n.id}: {e}")
# Dispatch owns delivery receipts. Reload before pruning so this
# scanner cannot replace a fresh channel receipt with a timestamp.
try:
cache = _json.loads(STATE.read_text(encoding="utf-8")) if STATE.exists() else {}
except Exception:
pass
# Prune cache entries for notes that no longer exist.
for stale in [k for k in cache if k not in seen_ids]:
cache.pop(stale, None)
@@ -2452,7 +2429,7 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
logger.warning(f"ping_notes: cache write failed: {e}")
if not sent:
raise TaskNoop(f"scanned {len(notes)} note(s), no reminders delivered")
raise TaskNoop(f"scanned {len(notes)} note(s), none due in ±{WINDOW_SEC}s")
preview = "; ".join(sent[:3])
extra = f" (+{len(sent) - 3} more)" if len(sent) > 3 else ""
return f"Pinged {len(sent)} note(s): {preview}{extra}", True
src/llm_core.py
+3 -13
View File
@@ -3249,7 +3249,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Ollama stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
return
# ── Anthropic streaming ──
@@ -3402,7 +3402,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Anthropic stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
return
# ── OpenAI-compatible streaming ──
@@ -3872,17 +3872,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
def _stream_failure_message(error: BaseException) -> str:
"""Client-facing text for an unexpected streaming failure.
The raw exception can carry request URLs, local paths or provider internals;
callers log it server-side and stream only this generic message, like the
named transport failures above it.
"""
return f"Model stream failed ({type(error).__name__})"
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
def _summarize_stream_error(err_chunk: Optional[str]) -> str:
src/readiness.py
+2 -9
View File
@@ -6,14 +6,11 @@ writable, and storage is local-first. Served by ``GET /api/ready`` and suitable
for an orchestrator readiness probe (200 only when every critical check passes).
"""
import logging
import os
import uuid
from datetime import datetime
from typing import Dict
logger = logging.getLogger(__name__)
def check_readiness() -> Dict[str, object]:
"""Run the readiness checks and return a JSON-serialisable report.
@@ -36,10 +33,7 @@ def check_readiness() -> Dict[str, object]:
conn.execute(sql_text("SELECT 1"))
checks["database"] = {"ok": True}
except Exception as e:
# The raw driver error can carry the DB host/user/path; keep it in the
# server log and give the client only the exception type.
logger.warning("Readiness database check failed: %s", e)
checks["database"] = {"ok": False, "error_type": type(e).__name__}
checks["database"] = {"ok": False, "error": str(e)}
# Data directory present and writable — home must be able to hold its own data.
try:
@@ -50,8 +44,7 @@ def check_readiness() -> Dict[str, object]:
os.remove(probe)
checks["data_dir"] = {"ok": True, "path": DATA_DIR}
except Exception as e:
logger.warning("Readiness data_dir check failed: %s", e)
checks["data_dir"] = {"ok": False, "error_type": type(e).__name__}
checks["data_dir"] = {"ok": False, "error": str(e)}
# Local-first: storage stays on the home machine (informational, never fatal).
local_first = (
src/shutdown.py
+34
View File
@@ -0,0 +1,34 @@
"""Process-wide shutdown signalling for graceful restarts.
Server-Sent-Events / streaming responses never complete on their own, so
uvicorn's graceful shutdown would wait for an open browser stream up to the
full ``timeout_graceful_shutdown`` window before exiting. A SIGTERM/SIGINT
handler flips this flag and tells uvicorn to exit; every streaming generator
checks the flag on each loop and ends promptly, so a restart completes in
seconds while ordinary in-flight requests still finish.
"""
from __future__ import annotations
import logging
import threading
logger = logging.getLogger(__name__)
_SHUTTING_DOWN = threading.Event()
def is_shutting_down() -> bool:
"""True once a shutdown (SIGTERM/SIGINT) has been requested."""
return _SHUTTING_DOWN.is_set()
def request_shutdown() -> None:
"""Mark the process as shutting down (idempotent)."""
if not _SHUTTING_DOWN.is_set():
_SHUTTING_DOWN.set()
logger.info("shutdown requested; streaming responses will end promptly")
def reset() -> None:
"""Clear the flag (for tests)."""
_SHUTTING_DOWN.clear()
src/tool_approval_scopes.py
-125
View File
@@ -2,12 +2,7 @@
from __future__ import annotations
import hmac
import logging
from enum import Enum
from hashlib import sha256
logger = logging.getLogger(__name__)
# Keep the existing wire values so the current route and no-build frontend do
@@ -21,126 +16,6 @@ DENY_APPROVAL_DECISION = "deny"
# session history contains a matching, resolved chat-session approval.
CHAT_SESSION_APPROVAL_CONTEXT_MARKER = "_tool_approval_chat_session_granted"
# The server's proof that IT resolved this approval. More than one route
# writes caller-supplied metadata into session history, so a client can write
# the shape of a resolved card directly; only the server can produce this.
CHAT_SESSION_APPROVAL_SIGNATURE_FIELD = "_server_grant"
def _grant_key() -> bytes | None:
"""Key material for grant signatures, or None when it is unavailable.
Reuses the persistent application key so a grant survives a restart the
way the transcript holding it does.
"""
try:
from src.secret_storage import _load_or_create_key
return _load_or_create_key()
except Exception as exc:
logger.warning("Tool approval grant key unavailable: %s", exc)
return None
def sign_chat_session_grant(
session_id: object,
approval_id: object,
decision: object,
) -> str | None:
"""Return the server's signature for one resolved chat-session grant."""
key = _grant_key()
if key is None:
return None
payload = "\x00".join(
(
str(session_id or ""),
str(approval_id or ""),
str(decision or "").strip().lower(),
)
)
return hmac.new(key, payload.encode("utf-8"), sha256).hexdigest()
# Message-metadata keys the server writes and a caller never should. Both are
# read back as authority: ``tool_events`` carries the approval cards, and the
# context marker is projected onto a turn once a grant is found.
_SERVER_OWNED_METADATA_KEYS = (
"tool_events",
CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
)
def sanitize_client_message_metadata(metadata):
"""Drop server-owned keys from a caller-supplied message metadata blob.
Routes that persist a message on the caller's behalf accept this blob
verbatim, which lets a caller write the shape of a resolved approval into
its own transcript. The grant check verifies a signature, so this is not
the control that closes that path; it keeps the state out of the
transcript in the first place. Anything else in the blob is left alone.
"""
if not isinstance(metadata, dict):
return metadata
if not any(key in metadata for key in _SERVER_OWNED_METADATA_KEYS):
return metadata
return {
key: value
for key, value in metadata.items()
if key not in _SERVER_OWNED_METADATA_KEYS
}
def stamp_chat_session_grant(
ask_user: dict,
session_id: object,
decision: object,
) -> None:
"""Record the server's grant on a card it has just resolved.
Call this only from the server-side resolve path. A decision that does not
grant chat-session scope leaves no signature behind, so downgrading a
``deny`` to an ``approve`` in the transcript does not carry a usable one.
"""
if not isinstance(ask_user, dict):
return
if str(decision or "").strip().lower() != CHAT_SESSION_APPROVAL_DECISION:
ask_user.pop(CHAT_SESSION_APPROVAL_SIGNATURE_FIELD, None)
return
signature = sign_chat_session_grant(
session_id,
ask_user.get("approval_id"),
CHAT_SESSION_APPROVAL_DECISION,
)
if signature:
ask_user[CHAT_SESSION_APPROVAL_SIGNATURE_FIELD] = signature
def verify_chat_session_grant(
signature: object,
session_id: object,
approval_id: object,
decision: object,
) -> bool:
"""Whether *signature* is this server's grant for that exact approval.
Fails CLOSED: an absent, malformed, or unverifiable signature is not a
grant. Binding the session and approval ids into the payload means a
signature lifted from one chat cannot be replayed into another.
"""
# compare_digest accepts only ASCII strings. Treat arbitrary persisted
# metadata as untrusted and require the exact representation we sign.
if (
not isinstance(signature, str)
or len(signature) != sha256().digest_size * 2
or any(character not in "0123456789abcdef" for character in signature)
):
return False
expected = sign_chat_session_grant(session_id, approval_id, decision)
if expected is None:
return False
return hmac.compare_digest(signature, expected)
class ToolApprovalScope(str, Enum):
# Surfaces without a resumable chat (the skill tester, unattended audits)
src/tool_capabilities.py
+18 -76
View File
@@ -17,7 +17,7 @@ from types import MappingProxyType
from typing import Any, Iterable, Mapping
from src.tool_approval_scopes import CHAT_SESSION_APPROVAL_CONTEXT_MARKER
from src.tool_security import BUILTIN_EMAIL_TOOLS, is_public_blocked_tool
from src.tool_security import BUILTIN_EMAIL_TOOLS
class ToolEffect(str, Enum):
@@ -216,6 +216,20 @@ _register(
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"ai_edit_image"},
ToolEffect.NETWORK_EGRESS,
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"get_editor_screenshot"},
ToolEffect.READ_PRIVATE,
)
_register(
{"plan_edit", "run_editor_command"},
ToolEffect.UI_SIDE_EFFECT,
)
_register(
{
"archive_email",
@@ -501,43 +515,6 @@ def capabilities_for_action(tool_name: Any, content: Any) -> ToolCapabilities:
if not isinstance(tool_name, str):
return base
if tool_name in {"web_fetch", "pdf_extract"}:
payload = content
raw = content.strip() if isinstance(content, str) else ""
if isinstance(payload, str):
try:
payload = json.loads(raw) if raw.startswith("{") else {}
except (TypeError, ValueError):
payload = {}
if not isinstance(payload, Mapping):
payload = {}
if tool_name == "web_fetch" and "urls" in payload:
urls = payload["urls"]
sources = []
if isinstance(urls, list):
for item in urls:
source = item.get("url") if isinstance(item, Mapping) else item
sources.append(str(source or "").strip())
else:
source = payload.get("url") or (payload.get("path") if tool_name == "pdf_extract" else "")
sources = [str(source or "").strip() or raw.split("\n", 1)[0].strip()]
# Match the native readers' local selectors without resolving or
# opening files. A mixed batch retains its network effects as well.
if tool_name == "web_fetch":
local = [source.startswith("/workspace/") or source.lower().startswith("file:///workspace/")
for source in sources]
else:
local = [os.path.isabs(source) or source.lower().startswith("file://") for source in sources]
if any(local):
effects = {ToolEffect.READ_WORKSPACE}
if not all(local):
effects.update(base.effects)
return ToolCapabilities(
frozenset(effects),
ResultIntegrity.WORKSPACE_UNTRUSTED if all(local) else base.result_integrity,
known=base.known,
)
if tool_name == "extract_text":
payload = content
if isinstance(payload, str):
@@ -678,13 +655,9 @@ POST_EXTERNAL_BLOCKED_EFFECTS = frozenset(
)
# On by default: until agent processes run without network and side-effecting
# non-process tools have their own exact-approval boundary, this gate is the
# only check between injected external content and those tools. Set it to a
# falsy value to opt out.
TOOL_APPROVAL_GATE_ENABLED = (
str(os.getenv("ODYSSEUS_TOOL_APPROVAL_GATE", "1")).strip().lower()
not in {"0", "false", "no", "off"}
str(os.getenv("ODYSSEUS_TOOL_APPROVAL_GATE", "0")).strip().lower()
in {"1", "true", "yes", "on"}
)
@@ -751,18 +724,6 @@ def messages_contain_external_untrusted_context(messages: Iterable[dict]) -> boo
return False
def delegated_tool_is_blocked(tool_name: Any, content: Any = None) -> bool:
"""Filter tool names for discovery and concrete workspace reads at dispatch."""
# The bridge exposes these spellings for the same filesystem surfaces.
if isinstance(tool_name, str):
tool_name = tool_name.strip()
tool_name = {"list_dir": "ls", "find_files": "glob"}.get(tool_name, tool_name)
return is_public_blocked_tool(tool_name) or (
content is not None
and ToolEffect.READ_WORKSPACE in capabilities_for_action(tool_name, content).effects
)
@dataclass
class ToolRunSecurityContext:
"""Server-owned integrity state for one agent run."""
@@ -779,19 +740,11 @@ class ToolRunSecurityContext:
# The bypass affects only this automatic gate; current tool policy, ownership,
# workspace confinement, and execution/sandbox restrictions still apply.
approval_gate_bypassed: bool = False
# Driven by a bearer API token, not a person at a browser. Privileged
# tools are refused outright and no approval can lift that.
delegated_credential: bool = False
def observe_messages(self, messages: Iterable[dict]) -> None:
"""Apply server-owned chat scope and promote untrusted prompt context."""
message_list = list(messages or ())
if self.delegated_credential:
# A delegated run has no human to grant chat-session scope, so a
# grant sitting in this chat's history (left by the owner's own
# browser) must not be picked up by a token driving the same chat.
self.approval_gate_bypassed = False
elif any(
if any(
isinstance(message, dict)
and isinstance(message.get("metadata"), dict)
and message["metadata"].get(
@@ -816,17 +769,6 @@ class ToolRunSecurityContext:
self.external_untrusted_context_seen = True
def decision_for(self, tool_name: Any, content: Any = None) -> ToolGateDecision:
# Checked before the bypasses below, because neither may lift it, and
# kept independent of external_untrusted_context_seen so it holds on a
# run where that gate never arms and raises no prompt to bypass.
if self.delegated_credential and delegated_tool_is_blocked(tool_name, content):
return ToolGateDecision(
False,
(
f"Tool '{tool_name}' is not available to API-token callers. "
"It requires an interactive session."
),
)
if not TOOL_APPROVAL_GATE_ENABLED:
return ToolGateDecision(True)
if self.approval_gate_bypassed:
src/tool_execution.py
+39 -308
View File
@@ -15,7 +15,6 @@ import logging
import os
import pathlib
import re
import stat
import secrets
import sys
import time
@@ -37,7 +36,6 @@ from src.tool_policy import ToolPolicy
from src.client_tool_contract import TUI_ROUTED_BRIDGE_TOOL_NAMES
from src.constants import (
DATA_DIR,
AGENT_WORKSPACE_DIR,
MAX_DIFF_LINES,
MAX_OUTPUT_CHARS,
MAX_READ_CHARS,
@@ -71,11 +69,11 @@ _MISSING_TOOL_SECURITY_CONTEXT = _MissingToolSecurityContext()
NO_TOOL_SECURITY_CONTEXT = _NoToolSecurityContext()
# Persistent working directory for agent subprocesses.
# Resolves to <repo_root>/data/agent_workspace, inside the bind-mounted volume
# in Docker (/app/data), so files survive a rebuild as before. The subdirectory
# rather than data/ itself keeps agent scratch files and dotfiles out of the
# directory holding the session store and the auth database.
_AGENT_WORKDIR = AGENT_WORKSPACE_DIR
# Resolves to <repo_root>/data, which is the bind-mounted volume in Docker
# (/app/data) and the local data directory for manual installs.
# Using this as cwd and HOME prevents the agent from silently creating files
# in ephemeral container layers that are lost on the next rebuild.
_AGENT_WORKDIR = DATA_DIR
ExecutionBridgeHandler = Callable[
@@ -135,12 +133,6 @@ def get_active_execution_bridge() -> AgentExecutionBridge | None:
return _active_execution_bridge.get()
def _tui_host_bridge_endpoint_url(url: str, path: str) -> str:
"""Replace a validated bridge URL's path without changing its authority."""
from src.agent_tools.subprocess_tools import host_shell_bridge_endpoint_url
return host_shell_bridge_endpoint_url(url, path)
def _tui_host_bridge_patch_url(
client_runtime_context: Optional[Dict[str, Any]],
) -> tuple[str, str] | None:
@@ -161,7 +153,11 @@ def _tui_host_bridge_patch_url(
from src.agent_tools.subprocess_tools import is_host_shell_bridge_url_allowed
if not is_host_shell_bridge_url_allowed(url):
return None
return _tui_host_bridge_endpoint_url(url, "/patch"), token
if url.endswith("/run"):
url = url[:-4] + "/patch"
elif not url.endswith("/patch"):
url += "/patch"
return url, token
async def _apply_patch_via_tui_host_bridge(
@@ -177,7 +173,7 @@ async def _apply_patch_via_tui_host_bridge(
import httpx
timeout = httpx.Timeout(125.0, connect=5.0, write=10.0, pool=5.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(
url,
headers={"x-odysseus-tui-bridge-token": token},
@@ -237,7 +233,7 @@ async def _bridge_post(bridge: Dict, path: str, payload: Dict, *, timeout_s: flo
"error": f"{err_prefix}: invalid TUI host bridge",
"exit_code": 1,
}
target = _tui_host_bridge_endpoint_url(url, path)
base = url.rsplit("/", 1)[0] if url.endswith(("/run", "/read", "/write")) else url.rstrip("/")
try:
import httpx
timeout = httpx.Timeout(
@@ -246,9 +242,9 @@ async def _bridge_post(bridge: Dict, path: str, payload: Dict, *, timeout_s: flo
write=10.0,
pool=5.0,
)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(
target,
f"{base}{path}",
headers={"x-odysseus-tui-bridge-token": token},
json=payload,
)
@@ -304,13 +300,13 @@ async def _cancel_bridge_request(bridge: Dict, request_id: str) -> None:
from src.agent_tools.subprocess_tools import is_host_shell_bridge_url_allowed
if not token or not is_host_shell_bridge_url_allowed(url):
return
target = _tui_host_bridge_endpoint_url(url, "/cancel")
base = url.rsplit("/", 1)[0]
try:
import httpx
timeout = httpx.Timeout(5.0, connect=2.0, write=2.0, pool=2.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
await client.post(
target,
f"{base}/cancel",
headers={"x-odysseus-tui-bridge-token": token},
json={"request_id": request_id},
)
@@ -733,7 +729,7 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
path = path_value.strip() if isinstance(path_value, str) else ""
if "\n" in path or "\r" in path:
path = ""
body_value = args.get("content")
body_value = args.get("content", "")
if not isinstance(body_value, str):
return "write_file: invalid arguments", {
"error": "write_file: content must be a string",
@@ -743,7 +739,7 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
else:
path, _, body = content.partition("\n")
path = path.strip()
if not path or path.endswith(("/", "\\")):
if not path:
return "write_file: invalid arguments", {
"error": "write_file: path is required",
"exit_code": 1,
@@ -751,17 +747,6 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
rejected = _text_write_to_binary_artifact_result(content)
if rejected is not None:
return rejected
from src.agent_tools.filesystem_tools import _parse_write_intent, _unwrap_fenced_source_body
try:
_, original_body, section, explicit_clear = _parse_write_intent(content)
except ValueError as error:
return "write_file: invalid arguments", {"error": str(error), "exit_code": 1}
body = _unwrap_fenced_source_body(body, path)
if not section or (not body.strip() and not explicit_clear):
return "write_file: ambiguous content", {
"error": "write_file: bridge cannot safely enforce implicit empty content; use explicit JSON string content",
"exit_code": 1,
}
return f"write_file: {path[:80]}", await _bridge_post(
bridge,
"/write",
@@ -788,15 +773,10 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
# 1. Sensitive-subpath deny list — checked FIRST. Blocks .ssh,
# .gnupg, shell rc files, token/env files even if the root above
# them is on the allowlist.
# 2. Application-state deny (_is_app_state_path) - DATA_DIR holds the
# session store, auth database, app key and settings, so only
# _agent_readable_data_subdirs() is readable inside it.
# 3. Allowlist - only the directories the agent legitimately needs
# (its data/ workspace, user content, system tmp). $HOME is NOT on
# the default list.
# 4. Opt-in extra roots - admin can add broader roots via the
# "tool_path_extra_roots" setting. These cannot re-open DATA_DIR;
# rule 2 is independent of which root a path arrived through.
# 2. Allowlist — only the directories the agent legitimately needs
# (project data/, system tmp). $HOME is NOT on the default list.
# 3. Opt-in extra roots — admin can add broader roots via the
# "tool_path_extra_roots" setting (list of path strings).
# ---------------------------------------------------------------------------
# Compatibility exports: the same deny predicate protects tool access and
@@ -807,191 +787,6 @@ from src.agent_runtime.path_policy import (
)
def _path_within(resolved: str, root: str) -> bool:
"""True when *resolved* is *root* itself or sits underneath it.
Use the platform's path-case rules. This helper participates in allow
decisions, so unconditional case-folding would let a distinct ``/DATA``
tree masquerade as a descendant of ``/data`` on case-sensitive systems.
"""
resolved, root = os.path.normcase(resolved), os.path.normcase(root)
if resolved == root:
return True
try:
if os.path.commonpath([resolved, root]) == root:
return True
except ValueError:
return False
# normcase is intentionally conservative about assumptions (notably on
# POSIX), so consult the filesystem when paths exist. This recognizes a
# case alias on a case-insensitive volume without treating distinct
# case-sensitive paths as the same allow root.
if os.path.exists(root):
candidate = resolved
while True:
try:
if os.path.exists(candidate) and os.path.samefile(candidate, root):
return True
except OSError:
pass
parent = os.path.dirname(candidate)
if parent == candidate:
break
candidate = parent
return False
def _path_within_conservative(resolved: str, root: str) -> bool:
"""Containment for deny decisions, folding case to fail closed."""
resolved, root = resolved.casefold(), root.casefold()
if resolved == root:
return True
try:
return os.path.commonpath([resolved, root]) == root
except ValueError:
return False
def _agent_readable_data_subdirs() -> tuple[str, ...]:
"""The only parts of DATA_DIR the agent's file tools may reach.
The agent's own scratch folder, plus the directories of user content whose
paths the application itself gives to the model, which it would then be
unable to open. These normally live under DATA_DIR; the documented mail
attachment override may instead name a disjoint external directory:
UPLOAD_DIR the chat upload manifest renders "path=<p>" and
says to read it with read_file (agent_loop.py)
MAIL_ATTACHMENTS_DIR download_attachment returns the path and its own
description tells the model to read it
PERSONAL_DIR GET /api/personal returns a path per file and is
reachable through the app_api tool; RUNBOOK_DIR
nests under it
PERSONAL_UPLOADS_DIR indexed as a personal-docs directory, which
manage_rag lists as an absolute path
Order matters: the first entry is roots[0], which _resolve_search_root uses
when grep/glob/ls are called with no path.
"""
from src.constants import (
DATA_DIR,
MAIL_ATTACHMENTS_DIR,
PERSONAL_DIR,
PERSONAL_UPLOADS_DIR,
UPLOAD_DIR,
)
configured = (
(AGENT_WORKSPACE_DIR, "agent_workspace", False),
(UPLOAD_DIR, "uploads", False),
# This has a documented environment override and may legitimately
# live outside DATA_DIR, but it must never equal/contain DATA_DIR.
(MAIL_ATTACHMENTS_DIR, "mail-attachments", True),
(PERSONAL_DIR, "personal_docs", False),
(PERSONAL_UPLOADS_DIR, "personal_uploads", False),
)
configured_data_dir = os.path.abspath(os.path.expanduser(str(DATA_DIR)))
data_dir = os.path.realpath(configured_data_dir)
safe: list[str] = []
for raw, internal_name, external_ok in configured:
value = str(raw or "").strip()
# These paths are security-policy roots, not ordinary allowlist
# entries. Internal roles may inherit a relative DATA_DIR, but must
# still resolve to their exact canonical child below. External mail
# overrides require an absolute, disjoint directory.
if not value:
continue
expanded = os.path.abspath(os.path.expanduser(value))
# A policy root must not acquire an exemption by redirecting its final
# path component to protected state or to an unrelated external tree.
if os.path.islink(expanded):
continue
resolved = os.path.realpath(expanded)
if os.path.exists(resolved) and not os.path.isdir(resolved):
continue
expected_internal = os.path.join(data_dir, internal_name)
expected_configured = os.path.join(configured_data_dir, internal_name)
inside_data = (
os.path.normcase(expanded)
in {
os.path.normcase(expected_configured),
os.path.normcase(expected_internal),
}
and resolved == expected_internal
)
external_safe = (
external_ok
and os.path.isabs(os.path.expanduser(value))
and resolved != data_dir
and os.path.dirname(resolved) != resolved
and not _path_within(data_dir, resolved)
and not _path_within(resolved, data_dir)
)
if not (inside_data or external_safe) or _is_sensitive_path(resolved):
continue
safe.append(resolved)
return tuple(safe)
def _is_app_state_path(resolved: str) -> bool:
"""True for anything under DATA_DIR that is not agent-readable.
DATA_DIR holds the session store, the auth database, the app encryption key
and the settings file. A model-supplied path must not reach those through
any root, so this is checked in both resolvers rather than expressed as an
absence from the allowlist: a workspace bound at or above the data
directory, or an opt-in tool_path_extra_roots entry covering it, would
otherwise put them back in reach.
A containment rule rather than a filename deny list, so state files added
later are covered without anyone remembering to list them. The composed
sensitive-name policy also denies settings.json and app.db in workspaces.
"""
from src.constants import DATA_DIR
if not _path_within_conservative(resolved, os.path.realpath(DATA_DIR)):
return False
return not any(
_path_within(resolved, d)
for d in _agent_readable_data_subdirs()
)
def _is_hardlinked_regular_file(resolved: str) -> bool:
"""Reject inode aliases that can smuggle DATA_DIR state into an allow root."""
try:
target = os.stat(resolved, follow_symlinks=False)
except OSError:
return False
return stat.S_ISREG(target.st_mode) and getattr(target, "st_nlink", 1) > 1
def _is_denied_tool_path(resolved: str, *, snapshot=None) -> bool:
"""Apply every path deny to a canonical traversal result.
Directory scans pass one ``_control_plane_snapshot()`` for the whole walk;
rebuilding it per entry made grep/ls/glob linear in protected-state stats.
"""
from src.agent_runtime.resources import _control_plane_path
return (
_is_sensitive_path(resolved)
or _is_app_state_path(resolved)
or _is_hardlinked_regular_file(resolved)
or _control_plane_path(resolved, snapshot=snapshot)
)
def _can_traverse_tool_path(resolved: str, *, snapshot=None) -> bool:
"""Allow walking a denied state parent only to reach safe carve-outs."""
if _is_sensitive_path(resolved):
return False
if not _is_app_state_path(resolved):
from src.agent_runtime.resources import _control_plane_path
return not _control_plane_path(resolved, snapshot=snapshot)
return any(
_path_within(readable, resolved)
for readable in _agent_readable_data_subdirs()
)
def _tool_path_roots() -> list[str]:
"""Return the list of directory roots that read_file / write_file
may touch. Default: project data/ + system temp dirs. Extra roots
@@ -999,9 +794,9 @@ def _tool_path_roots() -> list[str]:
"""
roots: list[str] = []
# The agent's workspace plus the user-content directories inside data/.
# The rest of DATA_DIR is denied by _is_app_state_path.
roots.extend(_agent_readable_data_subdirs())
# Project data directory — the agent's primary workspace.
from src.constants import DATA_DIR
roots.append(DATA_DIR)
# /tmp (and its macOS realpath /private/tmp).
roots.append("/tmp")
@@ -1058,10 +853,7 @@ def _resolve_tool_path(raw_path: str) -> str:
"""
resource_operation = active_resource_operation()
if resource_operation is not None:
resolved = resource_operation.resolve_path(raw_path)
if _is_denied_tool_path(resolved):
raise ValueError("bound path is sensitive, application state or a hard-linked file")
return resolved
return resource_operation.resolve_path(raw_path)
ws = get_active_workspace()
if ws:
return _resolve_tool_path_in_workspace(ws, raw_path)
@@ -1070,20 +862,11 @@ def _resolve_tool_path(raw_path: str) -> str:
expanded = os.path.expanduser(str(raw_path).strip())
resolved = os.path.realpath(expanded)
if _is_app_state_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside the application state directory"
)
if _is_sensitive_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside a sensitive directory "
f"(e.g. .ssh, .gnupg) or matches a sensitive filename"
)
if _is_hardlinked_regular_file(resolved):
raise ValueError(f"path '{raw_path}' is a hard-linked file")
if _is_denied_tool_path(resolved):
raise ValueError("path addresses sensitive execution control-plane state")
for root in _tool_path_roots():
if is_inside(root, resolved):
@@ -1120,19 +903,11 @@ def _resolve_tool_path_in_workspace(workspace: str, raw_path: str) -> str:
# Confinement says "inside the root"; the deny list says "allowed". They
# are separate questions and this one stays here, with the policy that
# owns it.
if _is_app_state_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside the application state directory"
)
if _is_sensitive_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside a sensitive directory "
f"(e.g. .ssh, .gnupg) or matches a sensitive filename"
)
if _is_hardlinked_regular_file(resolved):
raise ValueError(f"path '{raw_path}' is a hard-linked file")
if _is_denied_tool_path(resolved):
raise ValueError("path addresses sensitive execution control-plane state")
return resolved
@@ -1191,10 +966,6 @@ def vet_workspace(raw: str) -> Optional[str]:
resolved = os.path.realpath(os.path.expanduser(raw))
if not os.path.isdir(resolved) or _is_sensitive_path(resolved):
return None
# Refuse the bind rather than binding a workspace where every subsequent
# tool call would fail on the same deny list.
if _is_denied_tool_path(resolved):
return None
# Reject filesystem roots: binding / (or a Windows drive/UNC root) as the
# workspace would make every absolute path "inside" it, collapsing the
# confinement into host-wide file access. A root is its own dirname, which
@@ -1206,25 +977,11 @@ def vet_workspace(raw: str) -> Optional[str]:
def agent_cwd() -> str:
"""Working directory for agent subprocesses (bash/python/background jobs):
a validated sealed launch root, active workspace, or agent workspace."""
the active workspace when set, else the persistent data dir."""
from src.agent_runtime.process_resources import active_process_operation
bound = active_process_operation()
if bound is not None and bound.launch is not None:
bound.launch.scope.validate()
root = bound.launch.scope.root.path
if vet_workspace(root) != root:
raise RuntimeError("sealed process root is not a safe real directory")
return root
workspace = get_active_workspace()
if workspace:
vetted = vet_workspace(workspace)
if vetted is None:
raise RuntimeError("active workspace is not a safe real directory")
return vetted
resolved = os.path.realpath(_AGENT_WORKDIR)
if resolved not in _agent_readable_data_subdirs() or vet_workspace(resolved) != resolved:
raise RuntimeError("agent workspace is not a safe real directory")
return resolved
return (bound.launch.scope.root.path if bound is not None and bound.launch is not None
else get_active_workspace() or _AGENT_WORKDIR)
def get_mcp_manager():
@@ -1239,28 +996,19 @@ def _resolve_search_root(raw_path: str) -> str:
With a workspace active, the workspace folder is the root and a supplied
path is confined inside it. Otherwise an empty path defaults to the agent's
primary root (its workspace under the project data dir) and a supplied path
is confined by the global allowlist + sensitive-file policy.
primary root (project data dir) and a supplied path is confined by the
global allowlist + sensitive-file policy.
"""
resource_operation = active_resource_operation()
if resource_operation is not None:
resolved = resource_operation.resolve_path(raw_path, search=True)
if _is_denied_tool_path(resolved):
raise ValueError("bound search root is sensitive, application state or a hard-linked file")
return resolved
return resource_operation.resolve_path(raw_path, search=True)
raw = (raw_path or "").strip()
ws = get_active_workspace()
if ws:
# Resolve the empty case as the workspace path rather than returning
# it directly: returned unchecked it skipped both deny lists, so a
# bare ls listed whatever the workspace was bound to.
return _resolve_tool_path_in_workspace(ws, raw or ws)
return os.path.realpath(ws) if not raw else _resolve_tool_path_in_workspace(ws, raw)
if not raw:
roots = _tool_path_roots()
default_root = os.path.realpath(AGENT_WORKSPACE_DIR)
if default_root in roots and not _is_denied_tool_path(default_root):
return default_root
raise ValueError("default agent workspace is not a safe readable data subdirectory")
return roots[0] if roots else os.path.realpath(".")
return _resolve_tool_path(raw)
logger = logging.getLogger(__name__)
@@ -1651,14 +1399,6 @@ async def execute_tool_block(
"NO_TOOL_SECURITY_CONTEXT"
)
# Credential denial precedes authority, exact approval claim and bridges.
if (isinstance(security_context, ToolRunSecurityContext)
and security_context.delegated_credential):
from src.tool_capabilities import delegated_tool_is_blocked
if delegated_tool_is_blocked(getattr(block, "tool_type", None), getattr(block, "content", None)):
decision = security_context.decision_for(block.tool_type, block.content)
return blocked_tool_result(block.tool_type, decision.reason)
authority = active_request_authority() if request_authority is MISSING_AUTHORITY else request_authority
parent = active_request_authority()
if isinstance(authority, RequestAuthority) and parent is not None and authority is not parent:
@@ -2082,17 +1822,6 @@ async def _execute_tool_block_impl(
mark_authorized()
if bridge_owns_tool:
if tool == "write_file":
from src.agent_tools.filesystem_tools import _parse_write_intent, _unwrap_fenced_source_body
try:
path, body, section, explicit_clear = _parse_write_intent(content)
if not path or path.endswith(("/", "\\")) or "\n" in path or "\r" in path:
raise ValueError("write_file: valid file path required")
body = _unwrap_fenced_source_body(body, path)
if not section or (not body.strip() and not explicit_clear):
raise ValueError("write_file: bridge cannot safely enforce implicit empty content; use explicit JSON string content")
except ValueError as error:
return "write_file: invalid arguments", {"error": str(error), "exit_code": 1}
try:
return await dispatched(execution_bridge.route_tool(
tool,
@@ -2164,7 +1893,8 @@ async def _execute_tool_block_impl(
if tool == "generate_image":
from src.ai_interaction import do_generate_image
desc = "generate_image"
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner, **({'incognito': True} if image_privacy else {})))
elif (tool in NATIVE_FILESYSTEM_TOOLS
and (active_resource_operation() is not None or tool != "apply_patch"
or not _tui_host_bridge_patch_url(client_runtime_context))):
@@ -2324,7 +2054,8 @@ async def _execute_tool_block_impl(
result = await dispatched(do_list_cookbook_servers(content, owner=owner))
elif tool == "edit_image":
desc = "edit_image"
result = await dispatched(do_edit_image(content, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_edit_image(content, owner=owner, **({'incognito': True} if image_privacy else {})))
elif tool == "edit_file":
result = await dispatched(_direct_fallback(tool, content)) or {"error": "edit failed", "exit_code": 1}
desc = result.get("output") or result.get("error") or "edit_file"
src/tool_parsing.py
+25 -163
View File
@@ -194,33 +194,10 @@ _TOOL_CODE_OPEN_RE = re.compile(r"<tool_code>\s*\{", re.IGNORECASE)
_TOOL_CODE_CLOSE_RE = re.compile(r"\}\s*</tool_code>", re.IGNORECASE)
# Pattern 4b: Gemma-style <|tool_call|> call:tool_name{args} <tool_call|>
_GEMMA_TOOL_CALL_OPEN_RE = re.compile(
r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*\{",
_GEMMA_TOOL_CALL_RE = re.compile(
r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*(\{[\s\S]*?\})\s*<\|?tool_call\|?>",
re.IGNORECASE,
)
_GEMMA_TOOL_CALL_CLOSE_RE = re.compile(
r"\}\s*<\|?tool_call\|?>",
re.IGNORECASE,
)
# Native Qwen markup shares the same non-nesting delimiter grammar as the
# XML helpers. Literal closers keep whitespace and opener floods linear;
# values are stripped by the caller, as in the original regex path.
_QWEN_FUNCTION_OPEN_RE = re.compile(r"<function=([A-Za-z_][\w:.-]*)>\s*")
_QWEN_FUNCTION_CLOSE_RE = re.compile(r"</function>")
_QWEN_PARAMETER_OPEN_RE = re.compile(r"<parameter=([A-Za-z_]\w*)>\s*")
_QWEN_PARAMETER_CLOSE_RE = re.compile(r"</parameter>")
_QWEN_PYTHON_ARG_KEY_RE = re.compile(r"[A-Za-z_]\w*")
_QWEN_PYTHON_ARG_VALUE_RE = re.compile(r"\s*=\s*(['\"].*?['\"]|[^,]+)")
_ANGLE_TAG_OPEN_RE = re.compile(r"<")
_NONEMPTY_ANGLE_TAG_OPEN_RE = re.compile(r"<(?=[^>])")
_ANGLE_TAG_CLOSE_RE = re.compile(r">")
_EMAIL_LOCAL_RE = re.compile(r"[\w.+-]+")
_EMAIL_ADDRESS_RE = re.compile(r"[\w.+-]+@[\w.-]+\.\w+")
_ASCII_EMAIL_LOCAL_RE = re.compile(r"[A-Za-z0-9.!#$%&\x27*+/=?^_`{|}~-]+")
_ASCII_EMAIL_ADDRESS_RE = re.compile(
r"[A-Za-z0-9.!#$%&\x27*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
)
# Pattern 4c: Open-function wrapper emitted by some local MLX/Exo models.
# Example:
@@ -565,34 +542,6 @@ _PLAIN_UI_OPEN_PANEL_RE = re.compile(
r"((?:\s+(?:day|week|month|year|agenda)(?:\s+view)?(?:\s+\d{4}-\d{2}(?:-\d{2})?)?)?)"
r"\s*(?:`{1,3})?\s*$"
)
_PLAIN_UI_CANDIDATE_RE = re.compile(r"ui_control", re.IGNORECASE)
def _iter_plain_ui_open_panel(text: str):
"""Yield line-anchored UI commands without retrying every blank line."""
pos = 0
while candidate := _PLAIN_UI_CANDIDATE_RE.search(text, pos):
line_start = text.rfind("\n", 0, candidate.start()) + 1
match = _PLAIN_UI_OPEN_PANEL_RE.match(text, line_start)
if match is not None:
yield match
pos = match.end()
continue
newline = text.find("\n", candidate.end())
pos = len(text) if newline < 0 else newline + 1
def _strip_plain_ui_open_panel(text: str) -> str:
matches = list(_iter_plain_ui_open_panel(text))
if not matches:
return text
out = []
pos = 0
for match in matches:
out.append(text[pos:match.start()])
pos = match.end()
out.append(text[pos:])
return "".join(out)
# ---------------------------------------------------------------------------
@@ -909,10 +858,7 @@ def _raw_openai_tool_call_to_block(value) -> Optional[ToolBlock]:
elif tool_type in ("grep", "glob", "ls", "edit_file"):
content = json.dumps(args) if args else "{}"
elif tool_type == "write_file":
# Keep raw OpenAI JSON on the canonical path so explicit empty content
# remains distinguishable from a missing body.
from src.tool_schemas import function_call_to_tool_block
return function_call_to_tool_block(name, json.dumps(args))
content = args.get("path", "") + "\n" + args.get("content", "")
elif tool_type == "create_document":
parts = [args.get("title", "Untitled")]
if args.get("language"):
@@ -1043,43 +989,6 @@ def _strip_raw_openai_tool_call_json(text: str) -> str:
return "".join(pieces)
def iter_email_addresses(text: str, *, ascii_only: bool = False):
"""Find legacy email-shaped strings without retrying every word suffix.
Match the address only at the start of each maximal local-part token.
When that attempt fails, every suffix has the same '@'/domain boundary
and must fail too. Local/domain character runs are each scanned a bounded
number of times, including malformed text with no '@' or domain dot.
"""
local_re = _ASCII_EMAIL_LOCAL_RE if ascii_only else _EMAIL_LOCAL_RE
address_re = _ASCII_EMAIL_ADDRESS_RE if ascii_only else _EMAIL_ADDRESS_RE
pos = 0
while token := local_re.search(text, pos):
address = address_re.match(text, token.start())
if address is not None:
yield address.group(0)
pos = address.end()
else:
pos = token.end()
def _iter_qwen_python_args(raw_args: str):
"""Match each maximal key once, then its value at that fixed position.
If a word has no following equals sign, none of its suffixes can have one
either. Advancing past it avoids the old unanchored regex's O(n^2) retries
on a long malformed key, while preserving permissive quoted/bare values.
"""
pos = 0
while key := _QWEN_PYTHON_ARG_KEY_RE.search(raw_args, pos):
value = _QWEN_PYTHON_ARG_VALUE_RE.match(raw_args, key.end())
if value is not None:
yield key.group(0), value.group(1)
pos = value.end()
else:
pos = key.end()
def _parse_qwen3_native_text_call(
text: str,
additional_tool_names: Optional[Iterable[str]] = None,
@@ -1106,14 +1015,13 @@ def _parse_qwen3_native_text_call(
# Qwen native text rendering:
# <tool_call><function=manage_notes><parameter=action>list</parameter>...
fn_match = next(_iter_named_blocks(
text, _QWEN_FUNCTION_OPEN_RE, _QWEN_FUNCTION_CLOSE_RE
), None)
fn_match = re.search(r"<function=([A-Za-z_][\w:.-]*)>\s*([\s\S]*?)\s*</function>", text)
if fn_match:
name, raw_body = fn_match
name, raw_body = fn_match.groups()
args = {}
for key, raw_value in _iter_named_blocks(
raw_body.rstrip(), _QWEN_PARAMETER_OPEN_RE, _QWEN_PARAMETER_CLOSE_RE,
for key, raw_value in re.findall(
r"<parameter=([A-Za-z_]\w*)>\s*([\s\S]*?)\s*</parameter>",
raw_body,
):
value = raw_value.strip()
if value and value[0] in "[{\"":
@@ -1174,7 +1082,7 @@ def _parse_qwen3_native_text_call(
or normalized_name in declared_names
):
args = {}
for key, raw_value in _iter_qwen_python_args(raw_args):
for key, raw_value in re.findall(r"([A-Za-z_]\w*)\s*=\s*(['\"].*?['\"]|[^,]+)", raw_args):
try:
args[key] = ast.literal_eval(raw_value.strip())
except (ValueError, SyntaxError):
@@ -1515,45 +1423,6 @@ def _parse_tool_code_block(raw: str) -> Optional[ToolBlock]:
return ToolBlock(tool_name, content.strip())
return None
_GEMMA_FALLBACK_KEY_RE = re.compile(r"\w++")
_GEMMA_FALLBACK_COLON_RE = re.compile(r"\s*+:")
_GEMMA_FALLBACK_BOUNDARY_RE = re.compile(r"(?<!\s)(\s*+)(?:,\s*+\w++\s*+:|})")
def _iter_gemma_fallback_items(body):
"""Preserve permissive k:v recovery with one pass over keys/boundaries."""
boundaries = list(_GEMMA_FALLBACK_BOUNDARY_RE.finditer(body))
boundary_index = 0
pos = 0
newline = body.find("\n")
while key := _GEMMA_FALLBACK_KEY_RE.search(body, pos):
colon = _GEMMA_FALLBACK_COLON_RE.match(body, key.end())
if colon is None:
pos = key.end()
continue
start = colon.end()
while start < len(body) and body[start].isspace():
start += 1
if start < len(body) and body[start] in "\"'":
start += 1
while boundary_index < len(boundaries) and boundaries[boundary_index].end(1) < start:
boundary_index += 1
if boundary_index == len(boundaries):
return
boundary = boundaries[boundary_index]
end = max(start, boundary.start())
if 0 <= newline < start:
newline = body.find("\n", start)
if newline >= 0 and newline < end:
pos = key.end()
continue
capture_end = end
if capture_end > start and body[capture_end - 1] in "\"'":
capture_end -= 1
yield key.group(0), body[start:capture_end].strip()
pos = end
def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
"""Parse a Gemma-style call:tool_name{...} block into a ToolBlock."""
tool_name = tool_name.strip().lower().replace("-", "_")
@@ -1578,7 +1447,12 @@ def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
if not isinstance(params, dict):
params = {}
except Exception:
params = dict(_iter_gemma_fallback_items(body))
# Simple regex key-value extraction fallback
params = {}
for m in re.finditer(r'(\w+)\s*:\s*["\']?(.*?)["\']?(?=\s*,\s*\w+\s*:|\s*\})', body):
k = m.group(1)
v = m.group(2).strip()
params[k] = v
from src.tool_schemas import function_call_to_tool_block
return function_call_to_tool_block(tool_name, json.dumps(params))
@@ -1632,9 +1506,9 @@ def _iter_delimited(text, open_re, close_re):
pos = cm.end()
def _strip_delimited(text: str, open_re, close_re, replacement: str = "") -> str:
"""Replace every ``open_re ... close_re`` span (forward-only; see
_iter_delimited). Equivalent to ``open_re([\\s\\S]*?)close_re`` substitution
def _strip_delimited(text: str, open_re, close_re) -> str:
"""Remove every ``open_re ... close_re`` span (forward-only; see
_iter_delimited). Equivalent to ``open_re([\\s\\S]*?)close_re`` ``re.sub('')``
for these delimiters, without the O(n^2) rescan on unclosed openers."""
spans = list(_iter_delimited(text, open_re, close_re))
if not spans:
@@ -1643,23 +1517,11 @@ def _strip_delimited(text: str, open_re, close_re, replacement: str = "") -> str
last = 0
for match_start, _inner_start, _inner_end, match_end in spans:
out.append(text[last:match_start])
out.append(replacement)
last = match_end
out.append(text[last:])
return "".join(out)
def strip_angle_tags(text: str, replacement: str = "", *, allow_empty: bool = False) -> str:
"""Linear equivalent of replacing ``<[^>]+>`` (or ``<[^>]*>``).
This preserves the existing flat text cleanup, including nested '<' and
malformed tails; it does not interpret HTML. Stop once no '>' is reachable
instead of retrying a suffix scan at every '<' in untrusted text.
"""
opener = _ANGLE_TAG_OPEN_RE if allow_empty else _NONEMPTY_ANGLE_TAG_OPEN_RE
return _strip_delimited(text, opener, _ANGLE_TAG_CLOSE_RE, replacement)
def _iter_named_blocks(text, open_re, close_re):
"""Forward-only equivalent of ``open_re([\\s\\S]*?)close_re`` finditer where
open_re captures a name in group 1: yield ``(name, body)``, pairing each
@@ -2093,10 +1955,10 @@ def parse_tool_blocks(
# Pattern 4b: Gemma-style <|tool_call|> blocks
if not blocks:
for tool_name, body in _iter_named_blocks(
text, _GEMMA_TOOL_CALL_OPEN_RE, _GEMMA_TOOL_CALL_CLOSE_RE
):
block = _parse_gemma_tool_call(tool_name, "{" + body + "}")
for m in _GEMMA_TOOL_CALL_RE.finditer(text):
tool_name = m.group(1)
body = m.group(2)
block = _parse_gemma_tool_call(tool_name, body)
if block:
blocks.append(block)
@@ -2133,7 +1995,7 @@ def parse_tool_blocks(
# from weaker native-tool models after reading the tool docs but failing to
# emit the actual structured call.
if not blocks:
m = next(_iter_plain_ui_open_panel(text), None)
m = _PLAIN_UI_OPEN_PANEL_RE.search(text)
if m:
blocks.append(ToolBlock("ui_control", f"open_panel {m.group(1).lower()}{m.group(2).lower()}".strip()))
@@ -2176,7 +2038,7 @@ def strip_tool_blocks(
cleaned = _strip_delimited(cleaned, _XML_TOOL_CALL_OPEN_RE, _XML_TOOL_CALL_CLOSE_RE)
cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned)
cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE)
cleaned = _strip_delimited(cleaned, _GEMMA_TOOL_CALL_OPEN_RE, _GEMMA_TOOL_CALL_CLOSE_RE)
cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned)
cleaned = _strip_delimited(cleaned, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE)
cleaned = _strip_raw_openai_tool_call_json(cleaned)
declared_xml_calls = _parse_declared_direct_xml_calls(
@@ -2195,7 +2057,7 @@ def strip_tool_blocks(
if raw_web_json:
_, (start, end) = raw_web_json
cleaned = cleaned[:start] + cleaned[end:]
cleaned = _strip_plain_ui_open_panel(cleaned)
cleaned = _PLAIN_UI_OPEN_PANEL_RE.sub("", cleaned)
# Strip bare <invoke> blocks not wrapped in <tool_call>
cleaned = _strip_bare_invoke_markup(cleaned)
cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)
src/tool_security.py
+2 -13
View File
@@ -127,6 +127,7 @@ PLAN_MODE_READONLY_TOOLS = {
"resolve_contact",
"chat_with_model",
"ask_teacher",
"get_editor_screenshot",
}
@@ -163,6 +164,7 @@ _PLAN_MODE_KNOWN_MUTATORS = {
"download_model", "serve_model",
"stop_served_model", "cancel_download", "adopt_served_model", "serve_preset",
"generate_image", "edit_image", "trigger_research", "manage_research",
"plan_edit", "run_editor_command", "ai_edit_image",
# Shell is never read-only-safe; block it explicitly so it stays out of plan
# mode even if the schema list fails to load.
"bash", "python",
@@ -274,16 +276,3 @@ def blocked_tools_for_owner(owner: Optional[str]) -> Set[str]:
if owner_is_admin_or_single_user(owner):
return set()
return set(NON_ADMIN_BLOCKED_TOOLS)
def delegated_credential_blocked_tools() -> Set[str]:
"""Tools an agent run driven by a bearer API token must not reach.
Deliberately not owner-dependent. ``blocked_tools_for_owner`` asks whether
the OWNER is an admin, and for a token that question is always answered
yes: minting a token is an admin-only action, so the empty set comes back
for every token in existence. A token is a long-lived credential the owner
hands to a third party, so it is capped at the non-admin policy no matter
who minted it.
"""
return set(NON_ADMIN_BLOCKED_TOOLS)
src/turn_contract.py
+146 -269
View File
@@ -17,7 +17,6 @@ from typing import Iterable, Mapping
from src.action_intents import classify_tool_intent
from src.tool_policy import ToolPolicy
from src.text_scanning import has_prefixed_token_match
FAMILY_TOOLS = {
@@ -37,6 +36,7 @@ FAMILY_TOOLS = {
"sessions": frozenset({"list_sessions", "manage_session", "create_session", "send_to_session", "chat_with_model", "pipeline"}),
"image_generation": frozenset({"generate_image"}),
"image_editing": frozenset({"edit_image"}),
"photo_editor": frozenset({"plan_edit", "run_editor_command", "get_editor_screenshot", "ai_edit_image", "generate_image"}),
"transcription": frozenset({"transcribe_media"}),
"media_inspection": frozenset({"inspect_media"}),
"ocr": frozenset({"extract_text"}),
@@ -48,62 +48,6 @@ FAMILY_TOOLS = {
CONTRACT_CORE_TOOLS = frozenset({
"bash", "python", "read_file", "web_search", "web_fetch", "ask_user",
})
_WORKSPACE_PREFIX_RE = re.compile(r"/workspace/", re.I)
_WORKSPACE_ARTIFACT_RE = re.compile(
r"/workspace/[^\s`\"']+\.(?:csv|html?|json|md|svg|txt)\b", re.I
)
_WORKSPACE_OUTPUT_PREFIX_RE = re.compile(r"/workspace/(?!input/)", re.I)
_WORKSPACE_OUTPUT_RE = re.compile(
r"/workspace/(?!input/)[^\s`\"']+\."
r"(?:csv|html?|json|md|svg|txt|avif|bmp|gif|jpe?g|png|webp|pdf|mp4|webm)\b",
re.I,
)
_WORKSPACE_TOKEN_TAIL_RE = re.compile(r"[^\s`\"']*")
def _mentions_workspace_artifact(text: str) -> bool:
return has_prefixed_token_match(
str(text or ""),
_WORKSPACE_PREFIX_RE,
_WORKSPACE_ARTIFACT_RE,
_WORKSPACE_TOKEN_TAIL_RE,
)
def _mentions_workspace_output(text: str) -> bool:
return has_prefixed_token_match(
str(text or ""),
_WORKSPACE_OUTPUT_PREFIX_RE,
_WORKSPACE_OUTPUT_RE,
_WORKSPACE_TOKEN_TAIL_RE,
)
def _mentions_under_budget(text: str) -> bool:
value = str(text or "")
for under in re.finditer(r"\bunder", value, re.I):
cursor = under.end()
if cursor >= len(value) or not value[cursor].isspace():
continue
while cursor < len(value) and value[cursor].isspace():
cursor += 1
if cursor < len(value) and value[cursor] in "¥$€£":
cursor += 1
while cursor < len(value) and value[cursor].isspace():
cursor += 1
digit_start = cursor
while cursor < len(value) and value[cursor].isdecimal():
cursor += 1
if cursor == digit_start:
continue
if cursor == len(value) or not (value[cursor].isalnum() or value[cursor] == "_"):
return True
if value[cursor:cursor + 3].casefold() == "yen":
cursor += 3
if cursor == len(value) or not (value[cursor].isalnum() or value[cursor] == "_"):
return True
return False
_FAMILY_WORDS = {
"calendar": r"\b(?:calendar|calender|events?|appointments?|meetings?|agenda)\b",
"notes": r"\b(?:notes?|checklists?|groceries|remind\s+me)\b",
@@ -179,7 +123,13 @@ def _normalize_request_lead(value: str) -> str:
text,
flags=re.I,
)
text = _strip_cancelled_request_lead(text)
text = re.sub(
r"^(?:never\s*mind|scratch\s+that)\s*[,;:—–-]?\s*"
r"(?=(?:open|show|list|read|search|find|check|switch|go)\b)",
"",
text,
flags=re.I,
)
text = re.sub(r"^k(?:ay)?\s*[,!]?\s+(?=\S)", "", text, flags=re.I)
text = re.sub(
r"^(?:(?:great|nice|cool)\s*[,!.]|thanks?\s*[.!])\s+"
@@ -214,21 +164,6 @@ def _normalize_request_lead(value: str) -> str:
text = re.sub(r"^((?:can|could|would|will)\s+)u\b", r"\1you", text, flags=re.I)
text = re.sub(r"\boffical\b", "official", text, flags=re.I)
return text
def _strip_cancelled_request_lead(text: str) -> str:
lead = re.match(r"^(?:never\s*mind|scratch\s+that)", text, re.I)
if lead is None:
return text
cursor = lead.end()
while cursor < len(text) and text[cursor].isspace():
cursor += 1
if cursor < len(text) and text[cursor] in ",;:—–-":
cursor += 1
while cursor < len(text) and text[cursor].isspace():
cursor += 1
action = re.match(r"(?:open|show|list|read|search|find|check|switch|go)\b", text[cursor:], re.I)
return text[cursor:] if action is not None else text
_MISSPELLED_RESEARCH_ACTION = re.compile(
r"^\s*" + _REQUEST_PREFIX + r"(?:reserch|reasearch|reseach)\b",
re.I,
@@ -261,21 +196,6 @@ _PURE_ACTION_PROHIBITION = re.compile(
r"[^.;\n]*[.!?]*\s*$",
re.I,
)
def _is_pure_action_prohibition(text: str) -> bool:
value = str(text or "").strip()
prefix = re.match(
r"(?:read[- ]only(?:\s+and)?\s+)?(?:do\s+not|don['’]?t|never)\s+"
r"(?:add|create|make|write|draft|edit|change|update|delete|remove|send|reply|"
r"run|execute|download|serve|open|save|schedule|transcribe|inspect)\b",
value,
re.I,
)
if prefix is None:
return False
tail = value[prefix.end():].rstrip(".!?")
return not any(char in ".;\n" for char in tail)
_RETURN_TO_ACTION = re.compile(r"^\s*" + _REQUEST_PREFIX + r"return\s+to\b", re.I)
_PANEL_NAVIGATION = re.compile(
r"^\s*" + _REQUEST_PREFIX
@@ -487,7 +407,8 @@ _BOUND_EDITOR_WRITE = re.compile(
re.I,
)
_BOUND_EDITOR_IMPLICIT_REVISION = re.compile(
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|"
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|go\s+more\s+into|"
r"g[aå]\s+mer\s+(?:in|ut)\s+p[aå]|"
r"clean\s+(?:this|it|the\s+(?:text|draft|document|doc))\s+up|"
r"give\s+(?:me\s+)?(?:feedback|a\s+critique|suggestions?|comments?)|"
r"remove\b[^.;\n]{0,100}\b(?:mistakes?|errors?|inaccurac(?:y|ies)|misinformation)|"
@@ -528,111 +449,11 @@ _WARM_RECALL_WITH_FOLLOWUP = re.compile(
r"(?P<followup>(?:what(?:['’]?s|\s+is)?|which|who|where|when|how|show|open|read|list|find|search)\b[\s\S]{0,180})$",
re.I,
)
_WARM_TARGET_RE = re.compile(
r"calendar|emails?|inbox|notes?|tasks?|skills?|memories|memory|"
r"documents?|docs?|web|browser|cookbook|files?|shell",
re.I,
)
_WARM_FOLLOWUP_RE = re.compile(
r"(?:what(?:['’]?s|\s+is)?|which|who|where|when|how|show|open|read|list|find|search)"
r"\b[\s\S]{0,180}\Z",
re.I,
)
def _consume_space(value: str, cursor: int, *, required: bool = False) -> int | None:
start = cursor
while cursor < len(value) and value[cursor].isspace():
cursor += 1
return None if required and cursor == start else cursor
def _phrase_end(value: str, cursor: int, phrase: str) -> int | None:
for index, word in enumerate(phrase.split(" ")):
if value[cursor:cursor + len(word)].casefold() != word:
return None
cursor += len(word)
if index + 1 < len(phrase.split(" ")):
cursor = _consume_space(value, cursor, required=True)
if cursor is None:
return None
return cursor
def _warm_recall_parts(value: str, *, with_followup: bool = False) -> tuple[str, str] | None:
value = str(value or "")
cursor = _consume_space(value, 0) or 0
states = [cursor]
discourse = re.match(r"(?:ok(?:ay)?|and|then)", value[cursor:], re.I)
if discourse is not None:
after = _consume_space(value, cursor + discourse.end(), required=True)
if after is not None:
states.insert(0, after)
action_phrases = ("back to", "return to", "what about", "check", "show", "open")
action_states: list[int] = []
for state in states:
if not with_followup:
action_states.append(state)
for phrase in action_phrases:
end = _phrase_end(value, state, phrase)
if end is None:
continue
if with_followup:
end = _consume_space(value, end, required=True)
if end is None:
continue
action_states.append(end)
for state in dict.fromkeys(action_states):
state = _consume_space(value, state) or 0
possessive_states = [state]
for possessive in ("my", "the"):
if value[state:state + len(possessive)].casefold() == possessive:
possessive_states.insert(0, state + len(possessive))
for target_state in possessive_states:
target_state = _consume_space(value, target_state) or 0
target = _WARM_TARGET_RE.match(value, target_state)
if target is None:
continue
target_text = target.group(0)
suffix = target.end()
if with_followup:
if suffix < len(value) and (value[suffix].isalnum() or value[suffix] == "_"):
continue
separator_states = []
spaced = _consume_space(value, suffix) or 0
if spaced > suffix:
separator_states.append(spaced)
if spaced < len(value) and value[spaced] in "-—,:;":
separator_states.append(_consume_space(value, spaced + 1) or 0)
for conjunction in ("and", "then"):
end = spaced + len(conjunction)
if (value[spaced:end].casefold() == conjunction
and (end == len(value) or not (value[end].isalnum() or value[end] == "_"))):
separator_states.append(_consume_space(value, end) or 0)
for followup_start in dict.fromkeys(separator_states):
followup = _WARM_FOLLOWUP_RE.match(value, followup_start)
if followup is not None:
return target_text, followup.group(0)
continue
suffix = _consume_space(value, suffix) or 0
suffix_states = [suffix]
for word in ("again", "now"):
if value[suffix:suffix + len(word)].casefold() == word:
suffix_states.insert(0, suffix + len(word))
for end in suffix_states:
while end < len(value) and value[end] in ".!?":
end += 1
end = _consume_space(value, end) or 0
if end == len(value):
return target_text, ""
return None
_REQUIRED_TOOLS = {
"calendar": "manage_calendar", "notes": "manage_notes",
"tasks": "manage_tasks", "skills": "manage_skills",
"image_generation": "generate_image", "image_editing": "edit_image",
"photo_editor": "plan_edit",
"transcription": "transcribe_media", "media_inspection": "inspect_media", "ocr": "extract_text",
}
@@ -895,6 +716,60 @@ def scheduled_automation_request(message: str) -> bool:
))
_TASK_OBJECT = (
r'(?:tasks?|to-?dos?|to-?do\s+lists?|task\s+lists?|reminders?|automations?|'
r'(?:schedul\w*|recurring|scheduled)\s+jobs?)'
)
_TASK_MANAGEMENT_VERB = (
r'(?:add|create|make|set\s+up|schedule|new|start|track|turn|convert|save|store|'
r'delete|remove|complete|finish|close|pause|resume|restart|re-?run|run|edit|'
r'update|rename|archive|cancel|list|show|view|see|check|manage|assign|mark|'
r'look|peek|pull\s+up|open|find|keep)'
)
def explicit_task_management(message: str) -> bool:
"""True only when the user explicitly asks to manage background tasks.
A bare project noun ("30-day task + protocol doc") describes work to do, not
a request to create or read a scheduled task, so it must not select the
tasks family or offer manage_tasks.
"""
text = _normalize_request_lead(message)
if not text:
return False
if scheduled_automation_request(text):
return True
# Possessive / near-noun references: "my tasks", "our reminders", "my jobs".
if re.search(
r'\b(?:my|our|the|any|all)\b[^.!?\n]{0,24}?\b' + _TASK_OBJECT + r'\b',
text, re.I,
):
return True
# An explicit management verb near a task noun: "add a task", "delete my todo".
if re.search(
r'\b' + _TASK_MANAGEMENT_VERB + r'\b[^.!?\n]{0,40}?\b' + _TASK_OBJECT + r'\b',
text, re.I,
):
return True
# A task noun with an interrogative/state tail: "tasks do I have", "tasks titled".
if re.search(
r'\b' + _TASK_OBJECT + r'\b[^.!?\n]{0,32}?'
r'\b(?:do\s+i\s+have|have\s+i|are\s+(?:set\s+up|scheduled|active)|'
r'titled|named|called|set\s+up|scheduled|active|due|overdue)\b',
text, re.I,
):
return True
# A task noun tied to a list/tracker surface.
if re.search(
r'\b' + _TASK_OBJECT + r'\b[^.!?\n]{0,24}?\b'
r'(?:list|tracker|manager|dashboard|schedule|inbox|library|board)\b',
text, re.I,
):
return True
return False
def creation_container_tool(message: str) -> str | None:
"""The explicitly created container owns its content, not vice versa."""
text = _normalize_request_lead(message)
@@ -1033,7 +908,11 @@ def selected_tools_for_request(message: str) -> frozenset[str] | None:
tools = {"inspect_media"}
if (
re.search(r"\b(?:create|write|save|build|produce)\b", raw_text, re.I)
and _mentions_workspace_artifact(raw_text)
and re.search(
r"(?:file://)?/workspace/[^\s`\"']+\.(?:csv|html?|json|md|svg|txt)\b",
raw_text,
re.I,
)
):
tools.update({"write_file", "read_file"})
if re.search(r"\b(?:preview|render|open)\b[^.\n]{0,100}\b(?:page|html|browser)\b", raw_text, re.I):
@@ -2296,34 +2175,6 @@ _READ_PRESENTATION_SUFFIX = re.compile(
)
def _terminal_clause_match(text: str, core_pattern: str) -> re.Match[str] | None:
"""Match a terminal clause after removing its ambiguous punctuation tail."""
end = len(text)
while end and text[end - 1].isspace():
end -= 1
while end and text[end - 1] in ".!?":
end -= 1
possessive = core_pattern.replace(r"\s+", r"\s++").replace(r"\s*", r"\s*+")
# An optional leading " and" branch must start at a whitespace boundary;
# otherwise search retries the whole possessive run at every suffix.
possessive = possessive.replace(r"|\s++and", r"|(?<!\s)\s++and")
return re.search(possessive + r"$", text[:end], re.I)
def _strip_terminal_but(text: str) -> str:
end = len(text)
while end and text[end - 1].isspace():
end -= 1
if end < 3 or text[end - 3:end].casefold() != "but":
return text
start = end - 3
if start == 0 or not text[start - 1].isspace():
return text
while start and text[start - 1].isspace():
start -= 1
return text[:start]
def _read_request_and_limit(message: str) -> tuple[str, int | None]:
"""Strip only whole, known presentation/safety suffixes, never actions."""
text = _normalize_request_lead(message)
@@ -2373,11 +2224,11 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
if keep_few_suffix:
maximum = 3
text = text[:keep_few_suffix.start()].strip()
few_suffix = _terminal_clause_match(
text,
few_suffix = re.search(
r"[,.;?]\s*(?:(?:only|just)\s+)?(?:(?:list|show)\s+(?:me\s+)?)?a\s+few"
r"(?:\s+(?:task\s+)?(?:names?|items?|results?|entries?))?"
r"(?:\s+and\s+(?:whether|if)\s+[^.;\n]+)?",
r"(?:\s+and\s+(?:whether|if)\s+[^.;\n]+)?[.!?]*\s*$",
text, re.I,
)
if few_suffix:
maximum = 3
@@ -2389,43 +2240,42 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
text,
flags=re.I,
).strip()
terminal_read_only = _terminal_clause_match(
text,
text = re.sub(
r"[.;]\s*read[- ]only(?:\s+(?:please|pls|plz))?\s*,?\s*"
r"(?:(?:and\s+)?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:change|edit|modify)(?:\s+or\s+send)?\s+(?:anything|data))?",
)
if terminal_read_only:
text = text[:terminal_read_only.start()].strip()
# Explanatory/safety tails do not alter a preceding exact read request.
safety_tail = _terminal_clause_match(
r"(?:change|edit|modify)(?:\s+or\s+send)?\s+(?:anything|data))?"
r"[.!?]*\s*$",
"",
text,
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:touch|change|edit|modify)(?:\s+(?:anything|data|them))?(?:\s+yet)?",
)
if safety_tail:
text = text[:safety_tail.start()].strip()
flags=re.I,
).strip()
# Explanatory/safety tails do not alter a preceding exact read request.
text = re.sub(
r"[.!?]\s*+read[- ]only(?:\s++(?:please|pls|plz))?\s*+(?:,\s*+)?"
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:touch|change|edit|modify)(?:\s+(?:anything|data|them))?(?:\s+yet)?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[.!?]\s*read[- ]only(?:\s+(?:please|pls|plz))?\s*,?\s*"
r"(?:do\s+not|don['’]?t|dont)\s+(?:change|edit|modify)\s+"
r"(?:or\s+send\s+)?anything[.!?]*\s*$",
"", text, flags=re.I,
).strip()
for terminal_core in (
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+changes?",
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+edits?",
):
terminal_match = _terminal_clause_match(text, terminal_core)
if terminal_match:
text = text[:terminal_match.start()].strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+changes?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+edits?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[,;]\s*no\s+edits?[.!?]*\s*$", "", text, flags=re.I,
).strip()
terminal_no_write = _terminal_clause_match(
text, r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+writes?"
)
if terminal_no_write:
text = text[:terminal_no_write.start()].strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+writes?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[.!?]\s*(?:i['’]?m|i\s+am)\s+(?:just\s+)?checking\b[^\n]*$",
"", text, flags=re.I,
@@ -2443,11 +2293,12 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
text,
flags=re.I,
).strip()
short_lines = _terminal_clause_match(
text, r"[,;]\s*(?:keep\s+(?:them|it)\s+)?short\s+lines?\s*,?"
)
if short_lines:
text = text[:short_lines.start()].strip()
text = re.sub(
r"[,;]\s*(?:keep\s+(?:them|it)\s+)?short\s+lines?\s*,?[.!?]*\s*$",
"",
text,
flags=re.I,
).strip()
text = re.sub(
r"[,.;]\s*keep\s+(?:the\s+answer|it|them)\s+short[.!?]*\s*$",
"",
@@ -2472,11 +2323,11 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
maximum = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
text = text[:approximate_limit.start()].strip()
natural_limit = re.search(
r"(?:[,.;?]|[—–-]|(?<!\s)\s++but\s++|(?<!\s)\s++)\s*+(?:"
r"(?:i\s++)?only\s++(?:need|want|show(?:\s++me)?)?\s*+(?:(?:the\s++)?first\s++)?"
r"|just\s++(?:(?:the\s++)?first\s++)?|(?:show\s++me\s++)?like\s++|no\s++more\s++than\s++"
r"|cap(?:\s++(?:it|them|the\s++(?:answer|list)))?\s++at\s++"
r"|(?:maybe\s++)?(?:first|same)\s++)"
r"(?:[,.;?]|[—–-]|\s+but\s+|\s+)\s*(?:"
r"(?:i\s+)?only\s+(?:need|want|show(?:\s+me)?)?\s*(?:(?:the\s+)?first\s+)?"
r"|just\s+(?:(?:the\s+)?first\s+)?|(?:show\s+me\s+)?like\s+|no\s+more\s+than\s+"
r"|cap(?:\s+(?:it|them|the\s+(?:answer|list)))?\s+at\s+"
r"|(?:maybe\s+)?(?:first|same)\s+)"
r"(" + _READ_COUNT + r")"
r"(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?|bits?|things?))?"
r"(?:\s*(?:and|\+)\s+(?:their\s+)?(?:status(?:es)?|states?))?"
@@ -2514,7 +2365,7 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
maximum = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
text = text[:compact_limit.start()].strip()
conversational_limit = re.search(
r"(?:[,.;?]\s*|(?<!\s)\s++)(?:maybe\s+)?(?:keep\s+it\s+to\s+|stick\s+to\s+|(?:first|top)\s+)"
r"(?:[,.;?]\s*|\s+)(?:maybe\s+)?(?:keep\s+it\s+to\s+|stick\s+to\s+|(?:first|top)\s+)"
r"(" + _READ_COUNT + r")(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?))?"
r"[.!?]*\s*$",
text, re.I,
@@ -2524,7 +2375,7 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
value = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
maximum = value if maximum is None else min(maximum, value)
text = text[:conversational_limit.start()].rstrip(' ,.;?')
text = _strip_terminal_but(text)
text = re.sub(r"\s+but\s*$", "", text, flags=re.I)
need_limit = re.search(
r"[.!?]\s*(?:i\s+)?only\s+need\s+(" + _READ_COUNT + r")"
r"(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?))?"
@@ -4149,13 +4000,21 @@ def _families_for_tool(tool: str) -> frozenset[str]:
if bare in {"list_sessions", "manage_session", "create_session", "send_to_session",
"chat_with_model", "pipeline"}:
return frozenset({"sessions"})
return frozenset(family for family, tools in FAMILY_TOOLS.items() if bare in tools)
# The photo_editor family is context-only: it is only ever selected by an
# explicit editor request at the route, never inferred from message text.
# Overlapping tools such as generate_image must still resolve to their real
# product family, so exclude photo_editor rather than short-circuiting.
return frozenset(
family
for family, tools in FAMILY_TOOLS.items()
if bare in tools and family != "photo_editor"
)
def _clause_capabilities(text: str) -> set[str]:
# A prohibition constrains authority; it must never grant the family named
# only as the forbidden side effect (for example, "do not create a file").
if _is_pure_action_prohibition(text):
if _PURE_ACTION_PROHIBITION.fullmatch(text):
return set()
container_tool = creation_container_tool(text)
if container_tool:
@@ -4299,6 +4158,11 @@ def _clause_capabilities(text: str) -> set[str]:
if intent.reason == "terse calendar follow-up action":
return set()
words = {f for f, pattern in _FAMILY_WORDS.items() if re.search(pattern, text, re.I)}
if "tasks" in words and not explicit_task_management(text):
# A bare mention of "task"/"todo"/"automation" in a project description
# is content, not a request to create or read a scheduled task. Only an
# explicit task-management request may select the tasks family.
words.discard("tasks")
personal_stores = words & {
"calendar", "notes", "tasks", "skills", "memory", "documents", "email",
}
@@ -4406,6 +4270,14 @@ def _clause_capabilities(text: str) -> set[str]:
mapped = "tasks"
if mapped in FAMILY_TOOLS:
return {mapped}
if "shell_files" in words and re.search(
r"\b(?:bash|shell|terminal|zsh|command\s+line|cmd)\b", text, re.I,
):
# An explicitly named shell is a request to use the shell even when the
# rest of the message is a bare or typo'd command ("Bash commabd Ls la").
# Grant the shell family so the always-offered bash tool is not denied
# by request authority (offer/admission alignment).
return {"shell_files"}
# Families absent from action_intents still need a generic action gate;
# merely discussing a domain must not offer its mutation tools.
return words if _has_action_signal(text) or personal_lookup else set()
@@ -4816,7 +4688,12 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
raw_text,
re.I,
)
and _mentions_workspace_output(raw_text)
and re.search(
r"(?:file://)?/workspace/(?!input/)[^\s`\"']+\."
r"(?:csv|html?|json|md|svg|txt|avif|bmp|gif|jpe?g|png|webp|pdf|mp4|webm)\b",
raw_text,
re.I,
)
):
families.add("shell_files")
if re.search(
@@ -5275,7 +5152,7 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
and re.search(r"\b(?:ones?|top|apps?|services?|providers?)\b", text, re.I)
and re.search(r"\b(?:price|cheap|under|dimensions?|quote|trustworthy|app)\b", text, re.I)
)
or _mentions_under_budget(text)
or re.search(r"\bunder\s+[¥$€£]?\s*\d+(?:[.,]\d+)?(?:\s*yen)?\b", text, re.I)
):
return frozenset({"search_browser"})
if recent_family == ("search_browser",) and re.fullmatch(
@@ -6212,8 +6089,8 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
elif (recent == ("search_browser",) and families == {"cookbook_admin"}
and re.match(r"^\s*" + _REQUEST_PREFIX + r"(?:search|find)\s+(?:those|them|these)\b", text, re.I)):
families = {"search_browser"}
if not families and (recall := _warm_recall_parts(text)):
target = recall[0].lower()
if not families and (recall := _WARM_RECALL.fullmatch(text)):
target = recall["target"].lower()
family = {
"email": "email", "emails": "email", "inbox": "email",
"note": "notes", "notes": "notes", "task": "tasks", "tasks": "tasks",
@@ -6225,8 +6102,8 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
}[target]
if family in recently_executed_families(history):
families.add(family)
if not families and (recall := _warm_recall_parts(text, with_followup=True)):
target = recall[0].lower()
if not families and (recall := _WARM_RECALL_WITH_FOLLOWUP.fullmatch(text)):
target = recall["target"].lower()
family = {
"email": "email", "emails": "email", "inbox": "email",
"note": "notes", "notes": "notes", "task": "tasks", "tasks": "tasks",
@@ -6325,7 +6202,7 @@ def resolve_full_inventory_contract(*, schemas: Iterable[dict], policy: ToolPoli
"""Experimental trained inventory: permissions filter offers; model chooses actions."""
families = frozenset({"calendar", "notes", "tasks", "skills", "memory", "documents",
"email", "search_browser", "shell_files", "cookbook_admin",
"image_editing", "image_generation"})
"image_editing", "image_generation", "photo_editor"})
# ``ui_control`` is the executable bridge for explicit client-interface
# requests (for example, opening the gallery). It is not one of the ten
# persisted-data families, but omitting it here makes the full-inventory
src/url_safety.py
+2 -55
View File
@@ -12,7 +12,7 @@ break the primary use case. What it *always* rejects:
- a non-HTTP(S) scheme (``file://``, ``gopher://``, ``ftp://`` …), and
- the link-local range (``169.254.0.0/16`` / ``fe80::/10``), i.e. the cloud
instance-metadata SSRF credential-exfil vector — nobody serves embeddings
there, plus multicast / non-loopback reserved / unspecified addresses.
there — plus multicast / reserved / unspecified addresses.
For exposed multi-tenant deployments, set ``EMBEDDING_BLOCK_PRIVATE_IPS=true`` to
additionally reject all private and loopback targets (full SSRF lockdown).
@@ -94,15 +94,11 @@ def _classify(ip: ipaddress._BaseAddress, *, block_private: bool) -> Optional[st
return None
if ip.is_link_local:
return f"link-local address blocked (SSRF metadata risk): {ip}"
# IPv6 loopback is also reserved; its policy must match IPv4 loopback.
if ip.is_loopback:
if block_private:
return f"private/shared/loopback address blocked: {ip}"
return None
if ip.is_multicast or ip.is_reserved or ip.is_unspecified:
return f"disallowed address: {ip}"
if block_private and (
ip.is_private
or ip.is_loopback
or (isinstance(ip, ipaddress.IPv4Address) and ip in _SHARED_ADDRESS_SPACE_V4)
):
return f"private/shared/loopback address blocked: {ip}"
@@ -158,52 +154,3 @@ def check_outbound_url(
if not saw_ip:
return False, "host does not resolve to an IP"
return True, "ok"
class OutboundAddressBlocked(PermissionError):
"""A non-HTTP outbound host resolves into a disallowed address range."""
def connect_outbound_tcp(
host: str,
port: int,
*,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
block_private: bool = False,
source_address=None,
resolver: Optional[Callable[..., list]] = None,
) -> socket.socket:
"""Open a TCP connection to *host* under the outbound address policy.
For raw TCP clients (IMAP/SMTP). The host is resolved exactly once, every
resolved address is judged with the same policy as check_outbound_url, and
the socket connects only to those already-judged addresses. A second
lookup at connect time (what socket.create_connection(host) does) would let
a rebinding name pass the check with a public answer and then connect to
metadata/private space. TLS callers keep wrapping the returned socket with
``server_hostname=host``, so SNI and certificate checks are unchanged.
"""
resolve = resolver or socket.getaddrinfo
infos = resolve(host, port, 0, socket.SOCK_STREAM)
for _family, _type, _proto, _canon, sockaddr in infos:
ip = ipaddress.ip_address(str(sockaddr[0]).split("%")[0])
reason = _classify(ip, block_private=block_private)
if reason:
raise OutboundAddressBlocked(reason)
last_error: Optional[OSError] = None
for family, socktype, proto, _canon, sockaddr in infos:
sock = socket.socket(family, socktype, proto)
try:
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: # same contract as create_connection
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
sock.connect(sockaddr)
return sock
except OSError as exc:
last_error = exc
sock.close()
if last_error is not None:
raise last_error
raise socket.gaierror(socket.EAI_NONAME, f"{host} did not resolve to an address")