jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-15 19:32:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(harness): ship coherent vendor harness cluster

Browse Source 
Rebuild PR C to ship the vendor/photocraft harness as a coherent unit
(vendor implementation + vendor tests) instead of a per-file ahead/behind
mix that produced broken imports and 6+ test failures.

- Flip 36 harness impl/route/test files back to vendor (the per-file
  line-set policy misclassified user-rewritten files as 'behind').
- Flip core/models.py + src/tool_approval_scopes.py to vendor: vendor's
  harness (authority.py, turn_contract.py, clean_agent_preview.py) depends
  on vendor's older approval-scope API; upstream added
  CHAT_SESSION_APPROVAL_SIGNATURE_FIELD that vendor's core/models.py lacks.
- Add src/shutdown.py (new vendor infra) required by routes/chat_routes.py.
- Keep src/agent_tools/__init__.py at upstream (editor_tools registration
  lives in PR B).

Upstream-only security divergence preserved in C_upstream_delta.patch and
flagged for integration-time reconciliation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
pewdiepie-archdaemon
2026-10-13 13:59:04 +00:00
co-authored by Claude Opus 5.5
parent 364229f2dc
commit 9bdea753ae
39 changed files with 804 additions and 3076 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/models.py
-10
View File
@@ -11,8 +11,6 @@ from typing import Dict, List, Any, Optional, TYPE_CHECKING
from src.tool_approval_scopes import (
CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
CHAT_SESSION_APPROVAL_DECISION,
CHAT_SESSION_APPROVAL_SIGNATURE_FIELD,
verify_chat_session_grant,
)
if TYPE_CHECKING:
@@ -62,14 +60,6 @@ def _history_grants_chat_session_approval(
ask_user.get("kind") == "tool_approval"
and ask_user.get("resolved") == CHAT_SESSION_APPROVAL_DECISION
and str(ask_user.get("session_id") or "") == expected_session
# Shape proves nothing here: routes that accept a
# caller-supplied metadata blob write into this same history.
and verify_chat_session_grant(
ask_user.get(CHAT_SESSION_APPROVAL_SIGNATURE_FIELD),
expected_session,
ask_user.get("approval_id"),
CHAT_SESSION_APPROVAL_DECISION,
)
):
return True
return False
routes/assistant_routes.py
+1 -8
View File
@@ -16,7 +16,6 @@ from pydantic import BaseModel
from core.database import SessionLocal, CrewMember, ScheduledTask
from src.auth_helpers import get_current_user
from src.endpoint_resolver import resolve_owner_registered_endpoint_url
from src.owner_identity import REQUEST_SENTINEL_OWNERS
from src.task_scheduler import compute_next_run
@@ -179,13 +178,7 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
if payload.model is not None:
crew_db.model = payload.model or None
if payload.endpoint_url is not None:
try:
crew_db.endpoint_url = (
resolve_owner_registered_endpoint_url(db, payload.endpoint_url, owner)
if payload.endpoint_url else None
)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
crew_db.endpoint_url = payload.endpoint_url or None
if payload.timezone is not None:
crew_db.timezone = payload.timezone or None
routes/chat_helpers.py
+3 -9
View File
@@ -98,9 +98,7 @@ def clean_repeated_assistant_content(text: object) -> str:
"",
value,
).strip()
# No leading `\s*`: .strip() removes that whitespace anyway, and scanning
# it from every offset of a long whitespace run was quadratic (ReDoS).
value = re.sub(r"(?is)</\s*think\s*>\s*$", "", value).strip()
value = re.sub(r"(?is)\s*</\s*think\s*>\s*$", "", value).strip()
return value
_CASUAL_OPENING_RE = re.compile(
@@ -1326,17 +1324,13 @@ def _normalize_thinking(text: str) -> str:
# Handle garbled <think> tags: reasoning text followed by <think> as separator
# e.g. "The user said...I should respond.\n<think>Hey! What's up?"
# Linear form of `^([\s\S]+?)\n*<think>\s*([\s\S]*?)(?:</think>)?\s*$`:
# the lookbehind stops the lazy prefix re-scanning a newline run from every
# offset, and the optional trailing closer is dropped after the match
# instead of being retried at every body offset (both were quadratic).
garbled = re.match(
r'^([\s\S]+?)(?<![\s\S]\n)\n*<think(?:ing)?>\s*([\s\S]*)$',
r'^([\s\S]+?)\n*<think(?:ing)?>\s*([\s\S]*?)(?:</think(?:ing)?>)?\s*$',
text, re.IGNORECASE
)
if garbled:
before = garbled.group(1).strip()
after = re.sub(r'</think(?:ing)?>$', '', garbled.group(2).rstrip(), flags=re.IGNORECASE).strip()
after = garbled.group(2).strip()
# Only treat as garbled if the part before <think> looks like reasoning
reasoning_starts = (
'The user ', 'I need ', 'I should ', 'I will ',
routes/cookbook_routes.py
+7 -11
View File
@@ -701,17 +701,13 @@ def setup_cookbook_routes() -> APIRouter:
return cmd
repo_id = "cyankiwi/MiniMax-M3-AWQ-INT4"
hf_home = Path(os.environ.get("HF_HOME", str(Path.home() / ".cache" / "huggingface")))
hf_cache = Path(os.environ.get("HUGGINGFACE_HUB_CACHE", str(hf_home / "hub")))
snapshot_root = hf_cache / "models--cyankiwi--MiniMax-M3-AWQ-INT4" / "snapshots"
if body[serve_i + 1] == repo_id and snapshot_root.is_dir():
installed_snapshots = sorted(
(p for p in snapshot_root.iterdir() if p.is_dir()),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
if installed_snapshots:
body[serve_i + 1] = str(installed_snapshots[0])
snapshot = (
"/home/pewds/.cache/huggingface/hub/"
"models--cyankiwi--MiniMax-M3-AWQ-INT4/"
"snapshots/4082acbbec1236d21828d55b6bb0fe02ade4ab5b"
)
if body[serve_i + 1] == repo_id:
body[serve_i + 1] = snapshot
def add_env(key: str, value: str) -> None:
if not any(p.startswith(f"{key}=") for p in env_parts):
routes/history/history_routes.py
+4 -8
View File
@@ -7,14 +7,13 @@ import logging
import re
from typing import Dict, Any, Optional
from fastapi import APIRouter, Request, HTTPException, Depends
from fastapi import APIRouter, Request, HTTPException
from core.models import ChatMessage
from core.database import SessionLocal, ChatMessage as DbChatMessage, Session as DbSession
from src.auth_helpers import effective_user, require_chat_api_token_scope
from src.auth_helpers import effective_user
from src.topic_analyzer import analyze_topics
from src.upload_handler import reserve_message_upload_references
from src.tool_approval_scopes import sanitize_client_message_metadata
from routes.session_routes import (
_message_role,
_message_text,
@@ -238,10 +237,7 @@ def _keep_count_before_message(db_messages, before_msg_id: str | None) -> int |
def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
router = APIRouter(
tags=["history"],
dependencies=[Depends(require_chat_api_token_scope)],
)
router = APIRouter(tags=["history"])
def _reserve_message_uploads(
request: Request,
@@ -436,7 +432,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
content = body.get("content", "")
if not content:
raise HTTPException(400, "content is required")
metadata = sanitize_client_message_metadata(body.get("metadata"))
metadata = body.get("metadata")
_reserve_message_uploads(request, content, metadata)
msg = ChatMessage(role=role, content=content, metadata=metadata)
session_manager.add_message(session_id, msg)
routes/search/search_routes.py
+4 -17
View File
@@ -43,22 +43,11 @@ def setup_search_routes(config) -> APIRouter:
router = APIRouter(tags=["search"])
@router.get("/search/web", response_class=HTMLResponse)
async def web_search_page(request: Request, q: str = Query("", min_length=0)) -> HTMLResponse:
async def web_search_page(q: str = Query("", min_length=0)) -> HTMLResponse:
"""Browser-facing search results page for clickable agent web_search rows."""
# The site CSP allows inline script only with the per-request nonce
# (SecurityHeadersMiddleware); without it this page's script never ran.
nonce = html.escape(getattr(request.state, "csp_nonce", "") or "", quote=True)
safe_q = str(q or "").strip()
title = html.escape(safe_q or "Web search")
# json.dumps leaves `<`, `>` and `&` alone, so q=</script><script ...>
# would close this inline <script> and inject markup (reflected XSS).
# Unicode-escape them; the JS string value is unchanged.
q_json = (
json.dumps(safe_q)
.replace("<", "\\u003c")
.replace(">", "\\u003e")
.replace("&", "\\u0026")
)
q_json = json.dumps(safe_q)
page = f"""<!doctype html>
<html>
<head>
@@ -90,7 +79,7 @@ def setup_search_routes(config) -> APIRouter:
<div class="status" id="status">Loading...</div>
<div id="results"></div>
</main>
<script nonce="{nonce}">
<script>
const initialQuery = {q_json};
const input = document.getElementById('query');
const statusEl = document.getElementById('status');
@@ -114,9 +103,7 @@ def setup_search_routes(config) -> APIRouter:
}}
statusEl.textContent = sources.length ? `${{sources.length}} results` : 'No results';
resultsEl.innerHTML = sources.map(s => {{
const rawUrl = String(s.url || s.link || '');
// Result URLs come from third-party search results: only http(s) may become a link.
const url = /^https?:[/][/]/i.test(rawUrl) ? rawUrl : '';
const url = s.url || s.link || '';
const title = s.title || url || 'Untitled';
const snippet = s.snippet || s.content || '';
return `<a class="result" href="${{esc(url)}}" target="_blank" rel="noopener noreferrer">
routes/session_routes.py
+8 -57
View File
@@ -6,24 +6,17 @@ import uuid
import time
from pathlib import Path
from datetime import datetime
from fastapi import APIRouter, Form, HTTPException, Response, Request, Depends, Query
from fastapi import APIRouter, Form, HTTPException, Response, Request, Query
import logging
from core.session_manager import SessionManager
from core.models import ChatMessage
from src.request_models import SessionResponse
from core.database import Session as DbSession, SessionLocal, Document, GalleryImage, utcnow_naive
from src.auth_helpers import (
effective_user,
_auth_disabled,
owner_filter,
is_delegated_credential,
require_chat_api_token_scope,
)
from src.auth_helpers import effective_user, _auth_disabled, owner_filter
from src.session_image_cleanup import _generated_image_path_for_cleanup, session_image_refs
from src.session_actions import is_session_recently_active
from src.upload_handler import reserve_message_upload_references
from src.tool_approval_scopes import sanitize_client_message_metadata
def _sanitize_export_filename(name: str) -> str:
@@ -241,15 +234,9 @@ def _verify_session_owner(request: Request, session_id: str, session_manager=Non
logger = logging.getLogger(__name__)
router = APIRouter(
prefix="/api",
tags=["sessions"],
dependencies=[Depends(require_chat_api_token_scope)],
)
router = APIRouter(prefix="/api", tags=["sessions"])
def _current_user_is_admin(request: Request, user: str | None) -> bool:
if is_delegated_credential(request):
return False
if not user:
return False
auth_mgr = getattr(request.app.state, "auth_manager", None)
@@ -280,22 +267,6 @@ def _reject_raw_endpoint_url_for_non_admin(
raise HTTPException(403, "Choose a registered model endpoint")
def _reject_delegated_session_options(
request: Request,
*,
skip_validation: bool = False,
api_key: str | None = None,
) -> None:
"""Keep bearer credentials from exercising interactive-admin options."""
if is_delegated_credential(request) and (
skip_validation or bool((api_key or "").strip())
):
raise HTTPException(
403,
"API tokens cannot supply endpoint credentials or skip endpoint validation",
)
def _persist_session_headers(session_id: str, headers: dict | None) -> bool:
"""Persist endpoint auth headers for DB-backed session metadata."""
delays = (0.05, 0.15, 0.35)
@@ -530,11 +501,6 @@ def setup_session_routes(
):
skip_val = str(skip_validation).lower() == "true"
user = effective_user(request)
_reject_delegated_session_options(
request,
skip_validation=skip_val,
api_key=api_key,
)
endpoint_api_key = ""
endpoint_base_url = ""
_reject_raw_endpoint_url_for_non_admin(request, user, endpoint_id, endpoint_url)
@@ -791,11 +757,7 @@ def setup_session_routes(
except (AttributeError, TypeError, ValueError) as exc:
raise HTTPException(400, "Invalid message attachment metadata") from exc
for m in messages:
sess.add_message(ChatMessage(
m["role"],
m["content"],
metadata=sanitize_client_message_metadata(m.get("metadata")),
))
sess.add_message(ChatMessage(m["role"], m["content"], metadata=m.get("metadata")))
session_manager.save_sessions()
return {"ok": True, "count": len(messages)}
@@ -1156,8 +1118,6 @@ def setup_session_routes(
model: str = Form("gpt-4o"),
rag: str = Form(None)
):
if is_delegated_credential(request):
raise HTTPException(403, "This session type requires an interactive session")
if not OPENAI_API_KEY:
raise HTTPException(400, "Server missing OPENAI_API_KEY")
sid = str(uuid.uuid4())
@@ -1601,13 +1561,7 @@ def setup_session_routes(
raise HTTPException(404, "Session not found")
skills = _context_info_skill_inventory(skills_manager, owner=owner)
tools = _context_info_tool_inventory()
# `cwd` is caller-chosen. Walking it for AGENTS.md and reporting whether
# it exists confirms paths on the host, which /api/workspace/browse and
# /vet reserve for admins (or single-user mode); keep that boundary here.
from src.tool_security import owner_is_admin_or_single_user
_host_paths_visible = owner_is_admin_or_single_user(owner)
agents_md = _context_info_agents_md_inventory(cwd) if _host_paths_visible else []
agents_md = _context_info_agents_md_inventory(cwd)
# Workspace visibility: lets the TUI answer "can the backend actually
# see this directory?" (mounted vs bridge-only) without probing.
from src.workspace_paths import backend_workspace_path, workspace_mount_pairs
@@ -1617,22 +1571,19 @@ def setup_session_routes(
# Server-side tool policy: non-admin owners silently lose the computer
# tools (src/tool_security); surface that so the TUI can show it.
try:
from src.tool_security import blocked_tools_for_owner, delegated_credential_blocked_tools
from src.tool_security import blocked_tools_for_owner
_blocked = blocked_tools_for_owner(owner)
if is_delegated_credential(request):
_blocked.update(delegated_credential_blocked_tools())
except Exception:
_blocked = set()
_computer = {"bash", "python", "read_file", "write_file", "host_shell"}
_policy = {
"computer_tools": "restricted" if _computer & _blocked else "full",
"reason": "API-token caller" if is_delegated_credential(request) else
"non-admin owner" if _blocked else "single-user or admin",
"reason": "non-admin owner" if _blocked else "single-user or admin",
}
_workspace = {
"backend_path": _backend_cwd,
"exists_in_backend": _host_paths_visible and bool(_backend_cwd) and Path(_backend_cwd).is_dir(),
"exists_in_backend": bool(_backend_cwd) and Path(_backend_cwd).is_dir(),
"mount_configured": bool(workspace_mount_pairs()),
"via_mount": bool(_raw_cwd) and backend_workspace_path(_raw_cwd) != _raw_cwd,
}
routes/skills_routes.py
+15 -22
View File
@@ -19,7 +19,7 @@ from pydantic import BaseModel, Field
from services.memory.skills import SkillsManager
from src.auth_helpers import get_current_user
from src.prompt_security import untrusted_context_message
from src.text_helpers import strip_closed_think_blocks
from src.model_profiles import uses_odysseus_progressive_thinking
from core.middleware import require_admin
logger = logging.getLogger(__name__)
@@ -251,7 +251,7 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
# Strip closed think blocks. If a <think> was opened but never closed
# (the model ran out of budget mid-reasoning), drop everything from it
# onward so its stray braces don't poison JSON extraction.
text = strip_closed_think_blocks(text)
text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', text, flags=_re.I)
text = _re.sub(r'<think(?:ing)?>[\s\S]*$', '', text, flags=_re.I).strip()
def _coerce(d):
@@ -348,6 +348,8 @@ async def _eval_skill_run(skill_md: str, task: str, transcript: str,
url, model, msgs,
temperature=0.1, max_tokens=32768, headers=headers, timeout=180,
workload=workload,
thinking_mode=('on' if attempt == 0 else 'off')
if uses_odysseus_progressive_thinking(model) else None,
)
except Exception as e:
# Don't give up on a transient first-attempt error — let the second
@@ -400,11 +402,12 @@ async def _eval_skill_necessity(skill_md: str, others: list, url: str, model: st
[{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}],
temperature=0.1, max_tokens=8192, headers=headers, timeout=120,
workload=workload,
thinking_mode='on' if uses_odysseus_progressive_thinking(model) else None,
)
except Exception as e:
logger.warning(f"Necessity check failed: {e}")
return None
text = strip_closed_think_blocks(raw or '')
text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', (raw or ''), flags=_re.I)
text = _re.sub(r'<think(?:ing)?>[\s\S]*$', '', text, flags=_re.I).strip()
data = None
a, b = text.find('{'), text.rfind('}')
@@ -490,11 +493,12 @@ async def _eval_skill_retrieval_precision(skill_md: str, others: list,
[{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_msg}],
temperature=0.1, max_tokens=4096, headers=headers, timeout=90,
workload=workload,
thinking_mode='on' if uses_odysseus_progressive_thinking(model) else None,
)
except Exception as e:
logger.warning(f"Retrieval precision check failed: {e}")
return None
text = strip_closed_think_blocks(raw or '')
text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', (raw or ''), flags=_re.I)
text = _re.sub(r'<think(?:ing)?>[\s\S]*$', '', text, flags=_re.I).strip()
data = None
a, b = text.find('{'), text.rfind('}')
@@ -1185,11 +1189,12 @@ async def _improve_skill_md(skill_md: str, verdict: dict, transcript: str, url,
[{"role": "system", "content": sys_prompt},
{"role": "user", "content": user_msg}],
temperature=0.2, max_tokens=16384, headers=headers, timeout=180,
workload=workload)
workload=workload,
thinking_mode='on' if uses_odysseus_progressive_thinking(model) else None)
except Exception as e:
logger.warning(f"Audit: improve call failed: {e}")
return None
text = strip_closed_think_blocks(raw or '')
text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', (raw or ''), flags=_re.I)
text = _re.sub(r'<think(?:ing)?>[\s\S]*$', '', text, flags=_re.I)
text = _re.sub(r'</think(?:ing)?>', '', text, flags=_re.I).strip()
if text.startswith("```"):
@@ -2004,11 +2009,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
it untouched). It never changes the skill's published/draft STATUS."""
import time as _time
import asyncio as _asyncio
from core.database import SessionLocal
from src.endpoint_resolver import (
build_chat_url, build_headers, resolve_endpoint,
resolve_endpoint_runtime, resolve_owner_registered_endpoint,
)
from src.endpoint_resolver import resolve_endpoint
user = _owner(request)
body = await request.json()
@@ -2032,18 +2033,10 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter:
# session's model. Fall back to the caller's session model only if unset.
url, model, headers = resolve_endpoint("utility", owner=user)
if not url or not model:
if not url and body.get("endpoint_url") is not None:
db = SessionLocal()
try:
endpoint = resolve_owner_registered_endpoint(db, body["endpoint_url"], user)
base, api_key = resolve_endpoint_runtime(endpoint, owner=user)
url = build_chat_url(base)
headers = build_headers(api_key, base)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
finally:
db.close()
url = url or ((body.get("endpoint_url") or "").strip() or None)
model = model or ((body.get("model") or "").strip() or None)
if headers is None and isinstance(body.get("headers"), dict):
headers = body.get("headers")
if not url or not model:
raise HTTPException(400, "No model configured — set a Default or Utility model in Settings.")
routes/task/task_routes.py
+8 -24
View File
@@ -14,7 +14,6 @@ from core.database import SessionLocal, ScheduledTask, TaskRun, NotificationLog
from core.constants import internal_api_base
from src.auth_helpers import get_current_user
from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
from src.endpoint_resolver import resolve_owner_registered_endpoint_url
from src.task_action_policy import (
ADMIN_ONLY_TASK_ACTIONS,
is_admin_only_task_action,
@@ -520,12 +519,6 @@ def setup_task_routes(task_scheduler) -> APIRouter:
db = SessionLocal()
try:
then_task_id = _validate_then_task_id(db, req.then_task_id, user)
endpoint_url = None
if req.endpoint_url:
try:
endpoint_url = resolve_owner_registered_endpoint_url(db, req.endpoint_url, user)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
notifications_enabled = (
False if req.task_type == "action" and req.notifications_enabled is None
else bool(req.notifications_enabled) if req.notifications_enabled is not None
@@ -562,7 +555,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
status="active" if (req.trigger_type in ("event", "webhook") or next_run) else "completed",
output_target=req.output_target,
model=req.model or None,
endpoint_url=endpoint_url,
endpoint_url=req.endpoint_url or None,
then_task_id=then_task_id,
webhook_token=webhook_token,
notifications_enabled=notifications_enabled,
@@ -618,8 +611,12 @@ def setup_task_routes(task_scheduler) -> APIRouter:
async def create_notification_log(request: Request):
"""Persist an in-app toast so Settings can show notification history."""
user = _owner(request)
# Toast logging is best-effort. With auth disabled (or no user) there is
# nothing meaningful to persist, so return success rather than a 401 —
# a 401 here would trip the global fetch redirect-to-/login handler and
# yank users out of the editor mid-edit.
if not user:
raise HTTPException(401, "Authentication required")
return {"success": False}
body = await request.json()
message = str(body.get("body") or "").strip()[:2000]
if not message:
@@ -762,14 +759,7 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if req.model is not None:
task.model = req.model or None
if req.endpoint_url is not None:
try:
# An empty override restores the existing default-model workflow.
task.endpoint_url = (
resolve_owner_registered_endpoint_url(db, req.endpoint_url, user)
if req.endpoint_url else None
)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
task.endpoint_url = req.endpoint_url or None
if req.trigger_type is not None:
# Generate webhook token when switching to webhook trigger
if req.trigger_type == "webhook" and not task.webhook_token:
@@ -1252,14 +1242,8 @@ def setup_task_routes(task_scheduler) -> APIRouter:
if not out.get("prompt"):
return {"success": False, "message": "Could not extract a task instruction"}
return {"success": True, "draft": out}
except HTTPException as e:
# llm_core raises HTTPException with curated upstream text.
logger.error(f"parse_task failed: {e.detail}")
return {"success": False, "message": str(e.detail)}
except Exception as e:
# Anything else (JSON/parse/runtime errors) can carry internals;
# log it and keep the client message generic.
logger.error(f"parse_task failed: {e}")
return {"success": False, "message": f"Could not draft a task ({type(e).__name__})"}
return {"success": False, "message": str(e)}
return router
services/memory/memory_extractor.py
+3 -8
View File
@@ -610,9 +610,7 @@ async def update_persona_memory(
)
updated = strip_think(str(raw or ""), prose=True, prompt_echo=True).strip()
# No leading `\s*` before the closing fence: .strip() drops that
# whitespace anyway, and scanning it from every offset was quadratic.
updated = re.sub(r"^```(?:text|markdown)?\s*|```$", "", updated, flags=re.I | re.S).strip()
updated = re.sub(r"^```(?:text|markdown)?\s*|\s*```$", "", updated, flags=re.I | re.S).strip()
if len(updated) > 6000:
updated = updated[:6000].rstrip()
if updated == existing_memory:
@@ -696,9 +694,8 @@ async def audit_memories(
# Parse the JSON list, tolerating reasoning-model noise: <think> blocks,
# markdown fences, leading prose, and trailing commas.
import re as _re
from src.text_helpers import strip_closed_think_blocks
text = (raw or "").strip()
text = strip_closed_think_blocks(text).strip()
text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', text, flags=_re.I).strip()
def _loads_list(s):
if not s:
@@ -714,9 +711,7 @@ async def audit_memories(
cleaned = _loads_list(text)
if cleaned is None:
# Possessive `\s*+`: handing fence whitespace back to the body on a
# missing closing fence only rescanned the same tail (ReDoS).
_m = _re.search(r'```(?:json)?\s*+\n?([\s\S]*?)```', text)
_m = _re.search(r'```(?:json)?\s*\n?([\s\S]*?)```', text)
if _m:
cleaned = _loads_list(_m.group(1).strip())
if cleaned is None:
src/agent_loop.py
+282 -909
View File
File diff suppressed because it is too large Load Diff
src/agent_runtime/resources.py
+6 -9
View File
@@ -114,9 +114,11 @@ def _control_plane_snapshot():
uploader = get_upload_handler()
if uploader is not None and isinstance(getattr(uploader, "upload_dir", None), (str, os.PathLike)):
protected.add(canonical_root(Path(uploader.upload_dir) / "uploads.json"))
# Prefix protection covers the complete runtime trees. Public policy denies
# every regular hardlink alias, so collecting all child inodes here would
# add an unbounded recursive state inventory without widening protection.
for directory in job_dirs:
jobs = Path(directory)
if jobs.exists():
# Uninspectable state fails closed; hardlinks retain object identity.
protected.update(canonical_root(p) for p in jobs.rglob("*") if p.is_file())
protected.update(canonical_root(getattr(constants, name) + suffix)
for name in ("APP_DB", "SCHEDULED_EMAILS_DB", "EMAIL_CACHE_DB")
for suffix in ("-wal", "-shm", "-journal"))
@@ -129,7 +131,7 @@ def _control_plane_snapshot():
except FileNotFoundError:
continue
identities.add((observed.st_dev, observed.st_ino))
# Effect directories are protected by prefix without inventorying children.
# Effect directories join the prefix set only after the recursive inventory.
return frozenset(job_dirs | effect_dirs), frozenset(protected), frozenset(identities)
@@ -137,11 +139,6 @@ def _control_plane_path(path, *, snapshot=None):
# A scan-local snapshot bounds repeated hardlink checks. Ordinary resource
# resolution always observes fresh state. Neither form is an atomic kernel
# access policy, and snapshots must never survive a workspace guard call.
# Public state and hardlink denial also applies to sealed resources. Lazy
# import avoids coupling inert identity definitions to dispatcher startup.
from src.tool_execution import _is_app_state_path, _is_hardlinked_regular_file
if _is_sensitive_path(path) or _is_app_state_path(path) or _is_hardlinked_regular_file(path):
return True
directories, protected, identities = _control_plane_snapshot() if snapshot is None else snapshot
if any(Path(path).is_relative_to(directory) for directory in directories) or path in protected:
return True
src/agent_tools/__init__.py
-5
View File
@@ -29,7 +29,6 @@ from .interaction_tools import AskUserTool, UpdatePlanTool
from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool
from .bg_job_tools import ManageBgJobsTool
from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool
from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool
from .admin_tools import (
ADMIN_TOOL_HANDLERS,
do_manage_endpoints, do_manage_mcp, do_manage_webhooks,
@@ -73,10 +72,6 @@ TOOL_HANDLERS = {
"list_sessions": ListSessionsTool().execute,
"send_to_session": SendToSessionTool().execute,
"manage_session": ManageSessionTool().execute,
"plan_edit": PlanEditTool().execute,
"run_editor_command": RunEditorCommandTool().execute,
"get_editor_screenshot": GetEditorScreenshotTool().execute,
"ai_edit_image": AiEditImageTool().execute,
}
# Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings).
TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS)
src/agent_tools/filesystem_tools.py
+114 -572
View File
@@ -3,9 +3,8 @@ import json
import os
import re
import difflib
import secrets
import fnmatch
import shutil
import time
import tempfile
from typing import Optional, Dict, Any, Tuple, List
@@ -19,8 +18,6 @@ _CODENAV_SKIP_DIRS = frozenset({
})
_CODENAV_MAX_HITS = 200
_CODENAV_MAX_LINE = 400
_GREP_TIMEOUT_SECONDS = 20
_GREP_STDERR_PREFIX = 20_000
_STRUCTURED_DOCUMENT_SUFFIXES = frozenset({
".doc", ".docx", ".epub", ".pdf", ".pptx", ".xls", ".xlsx",
})
@@ -91,79 +88,6 @@ def _glob_to_regex(pat: str) -> "re.Pattern":
i += 1
return re.compile("".join(out))
def _validate_grep_descriptor(descriptor):
"""Revalidate an inert parent snapshot without inherited ContextVars."""
path, identity, ancestors = descriptor
if os.path.islink(path) or os.path.realpath(path) != path:
raise ValueError("grep: resource path changed")
for parent, observed in ancestors:
info = os.stat(parent, follow_symlinks=False)
if not os.path.isdir(parent) or (info.st_dev, info.st_ino) != tuple(observed):
raise ValueError("grep: resource ancestor changed")
info = os.stat(path, follow_symlinks=False)
import stat
if (not stat.S_ISREG(info.st_mode) or info.st_nlink > 1
or (info.st_dev, info.st_ino) != tuple(identity)):
raise ValueError("grep: resource identity changed")
def _python_grep_worker(payload: dict, output_queue) -> None:
"""Spawn-safe fallback grep worker used when ripgrep is unavailable.
Keep this at module scope: a frozen Windows executable cannot safely be
relaunched as ``sys.executable -c ...``, while multiprocessing can invoke a
top-level target through its frozen-process bootstrap.
"""
try:
flags = re.IGNORECASE if payload["ignore_case"] else 0
try:
regex = re.compile(payload["pattern"], flags)
glob_regex = (
_glob_to_regex(payload["glob"].replace("\\", "/"))
if payload["glob"]
else None
)
except re.error as exc:
output_queue.put(("error", f"grep: bad pattern: {exc}"))
return
max_hits = payload["max_hits"]
hits = 0
for descriptor in payload["files"]:
if hits >= max_hits:
break
path, identity, ancestors = descriptor
_validate_grep_descriptor(descriptor)
relative = os.path.relpath(path, payload["base"]).replace(os.sep, "/")
if glob_regex and not (
glob_regex.fullmatch(relative) or glob_regex.fullmatch(os.path.basename(path))
):
continue
try:
with open(path, "r", encoding="utf-8", errors="strict") as handle:
info = os.fstat(handle.fileno())
if (info.st_dev, info.st_ino) != tuple(identity) or info.st_nlink > 1:
raise ValueError("grep: resource identity changed before read")
for number, line in enumerate(handle, 1):
if regex.search(line):
output_queue.put(("match", path, number, line.rstrip()[:_CODENAV_MAX_LINE]))
hits += 1
if hits >= max_hits:
break
except UnicodeDecodeError:
continue
except OSError as error:
output_queue.put(("error", f"grep: {error}"))
return
output_queue.put(("done",))
except BaseException as exc:
try:
output_queue.put(("error", f"grep: fallback worker failed: {exc}"))
except BaseException:
pass
def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
if old == new:
return None
@@ -344,82 +268,32 @@ class ReadFileTool:
data = data[:MAX_READ_CHARS] + f"\n... [truncated at {MAX_READ_CHARS} chars]"
return {"output": data, "exit_code": 0}
def _write_new_file_without_overwrite(path: str, body: str) -> None:
"""Publish a new file without exposing a writable placeholder at its path.
Stage beside the destination, then hard-link it into place. The link is
atomic and fails if another writer created the destination first.
"""
directory = os.path.dirname(path) or "."
temporary_path = os.path.join(
directory, f".odysseus-write-{secrets.token_hex(16)}.tmp"
)
fd = os.open(
temporary_path,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o666,
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as temporary_file:
fd = None
temporary_file.write(body)
os.link(temporary_path, path)
finally:
if fd is not None:
os.close(fd)
try:
os.unlink(temporary_path)
except FileNotFoundError:
pass
class _EmptyBodyWouldTruncate(Exception):
"""Raised inside the write thread when an undeclared empty body is about to
replace a file that holds bytes. Carries the size at risk so the caller can be
told what it would have lost (#6414)."""
def __init__(self, path: str, existing_bytes: int):
super().__init__(path)
self.path = path
self.existing_bytes = existing_bytes
def _parse_write_intent(content: str) -> tuple[str, str, bool, bool]:
"""Classify the original transport, before binding or source normalization.
The returned clear flag is derived here, never from a caller-supplied key.
"""
if not isinstance(content, str):
raise ValueError("write_file: expected string arguments")
if content.lstrip().startswith("{"):
try:
args = json.loads(content)
except (TypeError, ValueError) as error:
raise ValueError("write_file: expected valid JSON arguments") from error
if not isinstance(args, dict):
raise ValueError("write_file: expected a JSON object")
path, body = args.get("path"), args.get("content")
if not isinstance(body, str):
raise ValueError("write_file: content required and must be a string")
if not isinstance(path, str):
raise ValueError("write_file: path required and must be a string")
return path.strip(), body, True, not body.strip()
path, delimiter, body = content.partition("\n")
return path.strip(), body, bool(delimiter), False
class WriteFileTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _display_tool_path, _resolve_tool_path
from src.agent_runtime.resource_binding import active_resource_operation
bound = active_resource_operation()
original = bound.operation.input if bound is not None else content
try:
_, _, has_section, declared_clear = _parse_write_intent(original)
raw_path, body, _, _ = _parse_write_intent(content)
except ValueError as error:
return {"error": str(error), "exit_code": 1}
from src.tool_execution import _display_tool_path, _resolve_tool_path, _resolve_search_root, _truncate
lines = content.split("\n", 1)
raw_path = lines[0].strip()
body = lines[1] if len(lines) > 1 else ""
# Decode JSON-object args (the fenced inline-args shape
# ```write_file {"path": "...", "content": "..."}```), matching
# ReadFileTool above. Without this the whole JSON string becomes the
# path and the file is written under a garbage name. This is the live
# path: there is no filesystem MCP server, so write_file always runs
# here via _direct_fallback, not through _build_mcp_args.
_stripped = content.strip()
if _stripped.startswith("{"):
try:
_a = json.loads(_stripped)
if not isinstance(_a, dict):
return {"error": "write_file: expected a JSON object", "exit_code": 1}
raw_path_value = _a.get("path")
body_value = _a.get("content")
raw_path = raw_path_value.strip() if isinstance(raw_path_value, str) else ""
if not isinstance(body_value, str):
return {"error": "write_file: content required", "exit_code": 1}
body = body_value
except (json.JSONDecodeError, TypeError, ValueError):
return {"error": "write_file: expected valid JSON arguments", "exit_code": 1}
if not raw_path:
return {"error": "write_file: path required", "exit_code": 1}
try:
@@ -449,6 +323,16 @@ class WriteFileTool:
"exit_code": 1,
"artifact_format_error": True,
}
if not body:
return {
"error": (
f"write_file: {path}: content required; refusing to create an "
"empty file. Call write_file again with the exact filename and "
"non-empty content. If you need a directory, create it from "
"bash/python instead."
),
"exit_code": 1,
}
# write_file is a UTF-8 text writer. Refuse to silently destroy an
# existing PDF, image, archive, or media artifact produced by a
# format-aware tool, especially after the agent has verified it.
@@ -463,10 +347,6 @@ class WriteFileTool:
"exit_code": 1,
"binary_artifact_preserved": target_existed,
}
if not has_section:
return {"error": "write_file: content required; missing content section", "exit_code": 1}
if raw_path.endswith(("/", "\\")) or os.path.isdir(path):
return {"error": "write_file: target is a directory", "exit_code": 1}
# This writer truncates in place. Once that stage is reached, a failure
# may leave a partial file; report it so effect evidence stays honest.
attempted = []
@@ -481,67 +361,21 @@ class WriteFileTool:
d = os.path.dirname(path)
if d:
os.makedirs(d, exist_ok=True)
if not body.strip() and not declared_clear:
# Empty/whitespace-only writes to an already-empty file are no-ops.
# Avoid reopening in truncating mode: another writer may have added
# data since the read above.
if os.path.isfile(path):
existing_bytes = os.path.getsize(path)
if existing_bytes > 0:
raise _EmptyBodyWouldTruncate(path, existing_bytes)
return old, 0
try:
if body:
# Publish whitespace content atomically. Writing it
# after exclusive creation could overwrite bytes from
# a writer that filled the new placeholder meanwhile.
attempted.append(True)
_write_new_file_without_overwrite(path, body)
else:
# An exact empty body needs no staged data, so create
# the file exclusively and never write through it.
attempted.append(True)
with open(path, "x", encoding="utf-8"):
pass
except FileExistsError:
if os.path.isfile(path):
existing_bytes = os.path.getsize(path)
if existing_bytes > 0:
raise _EmptyBodyWouldTruncate(path, existing_bytes)
return old, 0
raise
return old, len(body)
attempted.append(True)
with open(path, "w", encoding="utf-8") as f:
f.write(body)
return old, len(body)
old_content, size = await asyncio.to_thread(_write)
except _EmptyBodyWouldTruncate as e:
clear_call = json.dumps({"path": raw_path, "content": ""})
return {
"error": (
f"write_file: refused to write an empty body over {e.path} — it holds "
f"{e.existing_bytes} bytes, which the write would have destroyed, so "
f"the file is unchanged. To clear it on purpose, resend with an "
f"explicit empty content: {clear_call}"
),
"exit_code": 1,
}
except PermissionError:
return {"error": f"write_file: {path}: permission denied", "exit_code": 1,
**({"mutation_attempted": True} if attempted else {})}
except OSError as e:
return {"error": f"write_file: {path}: {e}", "exit_code": 1,
**({"mutation_attempted": True} if attempted else {})}
committed_body = old_content if size == 0 and not declared_clear and not body.strip() else body
diff = _unified_diff(old_content, committed_body, path)
diff = _unified_diff(old_content, body, path)
result = {
"output": (f"Wrote {size} bytes to {_display_tool_path(path)}" if attempted else
f"No write performed for {_display_tool_path(path)} (implicit empty body)"),
"output": f"Wrote {size} bytes to {_display_tool_path(path)}",
"exit_code": 0,
**({"write_noop": True} if not attempted else {}),
}
if diff:
result["diff"] = diff
@@ -785,7 +619,7 @@ def _apply_patch_hunks(original: str, hunks: List[List[str]], label: str) -> str
class LsTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _display_tool_path, _is_denied_tool_path, _resolve_search_root, _truncate
from src.tool_execution import _display_tool_path, _resolve_search_root, _truncate
raw_path = ""
_s = (content or "").strip()
if _s.startswith("{"):
@@ -801,18 +635,15 @@ class LsTool:
return {"error": f"ls: {e}", "exit_code": 1}
def _ls():
from src.agent_runtime.resources import _control_plane_snapshot
if not os.path.isdir(root):
return None, f"ls: {root}: not a directory"
rows = []
snapshot = _control_plane_snapshot()
try:
with os.scandir(root) as it:
for entry in it:
if entry.name.startswith("."):
continue
if (_is_denied_tool_path(os.path.realpath(entry.path), snapshot=snapshot)
or not _visible_bound_resource(entry.path)):
if not _visible_bound_resource(entry.path):
continue
try:
is_dir = entry.is_dir(follow_symlinks=False)
@@ -841,8 +672,6 @@ class GlobTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import (
_SENSITIVE_BASENAMES,
_can_traverse_tool_path,
_is_denied_tool_path,
_display_tool_path,
_is_sensitive_path,
_resolve_tool_path,
@@ -867,12 +696,10 @@ class GlobTool:
return {"error": f"glob: {e}", "exit_code": 1}
def _glob():
from src.agent_runtime.resources import _control_plane_snapshot
base = os.path.abspath(root)
if not os.path.isdir(base):
return None, f"glob: {root}: not a directory"
rbase = os.path.realpath(base)
snapshot = _control_plane_snapshot()
norm_pat = pattern.replace("\\", "/")
# Fast path: literal pattern (no wildcards) → direct path lookup.
if not any(c in norm_pat for c in "*?["):
@@ -889,8 +716,7 @@ class GlobTool:
# .ssh/id_rsa, …) falls through to the walk, which skips it —
# otherwise glob would surface secret paths that read_file /
# grep already refuse to touch.
if (inside and os.path.exists(cand) and not _is_denied_tool_path(cand, snapshot=snapshot)
and _visible_bound_resource(cand)):
if inside and os.path.exists(cand) and not _is_sensitive_path(cand) and _visible_bound_resource(cand):
return [cand], None
# Literal not at exact path — fall through to walk so
# e.g. "foo.py" still matches at any depth (like rglob).
@@ -900,18 +726,13 @@ class GlobTool:
cap = _CODENAV_MAX_HITS * 5
try:
for dp, dns, fns in os.walk(base):
if not _can_traverse_tool_path(os.path.realpath(dp), snapshot=snapshot):
dns[:] = []
continue
# Prune skipped dirs before descending (unlike rglob which
# descends first then filters — fatal on large node_modules).
# Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
# never enumerates the keys/tokens inside them.
dns[:] = [
d for d in dns
if d not in _CODENAV_SKIP_DIRS
and d not in _SENSITIVE_BASENAMES
and _can_traverse_tool_path(os.path.realpath(os.path.join(dp, d)), snapshot=snapshot)
if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
]
for name in fns + dns:
full = os.path.join(dp, name)
@@ -919,8 +740,7 @@ class GlobTool:
if regex.fullmatch(rel) or regex.fullmatch(name):
# Skip deny-listed sensitive files (.env, id_rsa,
# known_hosts, …) the same way grep does.
if (_is_denied_tool_path(os.path.realpath(full), snapshot=snapshot)
or not _visible_bound_resource(full)):
if _is_sensitive_path(os.path.realpath(full)) or not _visible_bound_resource(full):
continue
try:
mtime = os.stat(full).st_mtime
@@ -947,14 +767,10 @@ class GlobTool:
class GrepTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import (
_SENSITIVE_BASENAMES,
_SENSITIVE_FILE_PATTERNS,
_agent_readable_data_subdirs,
_is_denied_tool_path,
_display_tool_path,
_can_traverse_tool_path,
_is_sensitive_path,
_path_within,
_resolve_tool_path,
_resolve_search_root,
_truncate,
)
@@ -983,357 +799,83 @@ class GrepTool:
return {"error": f"grep: {e}", "exit_code": 1}
def _grep():
import multiprocessing
import queue
import subprocess
import threading
from src.agent_runtime.resources import _control_plane_snapshot
from src.constants import DATA_DIR
rg = shutil.which("rg")
real_root = os.path.realpath(root)
deadline = time.monotonic() + _GREP_TIMEOUT_SECONDS
base = real_root if os.path.isdir(real_root) else os.path.dirname(real_root)
import re as _re
import shutil
from src.agent_runtime.resource_binding import active_resource_operation
bound = active_resource_operation()
if bound is not None:
bound.validate()
files = []
snapshot = _control_plane_snapshot()
def check_deadline():
if time.monotonic() >= deadline:
raise TimeoutError("grep: timed out")
def observe_file(path):
check_deadline()
if os.path.islink(path):
return
canonical = os.path.realpath(path)
if not _path_within(canonical, base) or _is_denied_tool_path(canonical, snapshot=snapshot):
return
if bound is not None:
try:
bound.resolve_path(canonical)
except ValueError:
# Intentionally denied publication control-plane paths
# are omitted before any producer is allowed to read.
return
info = os.stat(canonical, follow_symlinks=False)
if not os.path.isfile(canonical):
return
ancestors = []
parent = os.path.dirname(canonical)
while _path_within(parent, base):
observed = os.stat(parent, follow_symlinks=False)
ancestors.append((parent, (observed.st_dev, observed.st_ino)))
if parent == base:
break
parent = os.path.dirname(parent)
files.append((canonical, (info.st_dev, info.st_ino), tuple(ancestors)))
if len(files) > 100_000:
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
try:
if os.path.islink(root):
raise ValueError("grep: symlink search root is not allowed")
if os.path.isfile(real_root):
observe_file(real_root)
elif os.path.isdir(real_root):
pending_directories = [real_root]
enumerated = 0
while pending_directories:
check_deadline()
directory = pending_directories.pop()
if not _can_traverse_tool_path(directory, snapshot=snapshot):
continue
with os.scandir(directory) as entries:
for entry in entries:
check_deadline()
enumerated += 1
if enumerated > 100_000:
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
if entry.is_symlink():
continue
canonical = os.path.realpath(entry.path)
if not _path_within(canonical, base):
raise ValueError("grep: directory identity changed during enumeration")
if entry.is_dir(follow_symlinks=False):
if (entry.name not in _CODENAV_SKIP_DIRS
and _can_traverse_tool_path(canonical, snapshot=snapshot)
and (bound is None or _is_denied_tool_path(canonical, snapshot=snapshot)
or _visible_bound_resource(canonical))):
pending_directories.append(canonical)
else:
observe_file(entry.path)
else:
raise FileNotFoundError(f"grep: {root}: not found")
check_deadline()
except (OSError, ValueError) as error:
return None, str(error) if str(error).startswith("grep:") else f"grep: {error}"
descriptors = {record[0]: record for record in files}
targets = list(descriptors)
lines: list[str] = []
def parse_rg_result(raw: str) -> Optional[str]:
try:
record = json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
if record.get("type") != "match":
return None
data = record.get("data") or {}
path = (data.get("path") or {}).get("text")
text_value = (data.get("lines") or {}).get("text")
number = data.get("line_number")
if not isinstance(path, str) or not isinstance(text_value, str):
return None
absolute = path if os.path.isabs(path) else os.path.join(base, path)
canonical = os.path.realpath(absolute)
if canonical not in descriptors:
raise ValueError("grep: producer returned an undeclared resource")
_validate_grep_descriptor(descriptors[canonical])
if bound is not None:
bound.resolve_path(canonical)
return f"{_display_tool_path(canonical)}:{number}:{text_value.rstrip()[:_CODENAV_MAX_LINE]}"
def run_rg(cmd: list[str]) -> Optional[str]:
try:
process = subprocess.Popen(
cmd,
cwd=base,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
except Exception as exc:
return f"grep: {exc}"
output: queue.Queue[Optional[str]] = queue.Queue(maxsize=max_hits + 2)
stderr_prefix: list[str] = []
stderr_size = 0
stop_reader = threading.Event()
def enqueue_stdout(value: Optional[str]) -> bool:
# The consumer stops at the result cap or deadline. Never
# leave a producer blocked on its bounded queue afterward.
while not stop_reader.is_set():
try:
output.put(value, timeout=0.05)
return True
except queue.Full:
continue
return False
def read_stdout() -> None:
assert process.stdout is not None
try:
for line in process.stdout:
if not enqueue_stdout(line.rstrip("\n")):
break
finally:
enqueue_stdout(None)
def read_stderr() -> None:
nonlocal stderr_size
assert process.stderr is not None
while True:
chunk = process.stderr.read(4096)
if not chunk:
break
if stderr_size < _GREP_STDERR_PREFIX:
kept = chunk[:_GREP_STDERR_PREFIX - stderr_size]
stderr_prefix.append(kept)
stderr_size += len(kept)
stdout_thread = threading.Thread(target=read_stdout, daemon=True)
stderr_thread = threading.Thread(target=read_stderr, daemon=True)
stdout_thread.start()
stderr_thread.start()
timed_out = False
capped = False
try:
while len(lines) < max_hits:
remaining = deadline - time.monotonic()
if remaining <= 0:
timed_out = True
break
try:
raw = output.get(timeout=remaining)
except queue.Empty:
timed_out = True
break
if raw is None:
break
parsed = parse_rg_result(raw)
if parsed and parsed not in lines:
lines.append(parsed)
capped = len(lines) >= max_hits
finally:
stop_reader.set()
if (timed_out or capped) and process.poll() is None:
process.terminate()
try:
remaining = max(0.01, deadline - time.monotonic())
return_code = process.wait(timeout=min(1, remaining))
except subprocess.TimeoutExpired:
process.kill()
return_code = process.wait()
stdout_thread.join()
stderr_thread.join()
if timed_out:
return "grep: timed out"
if not capped and return_code not in (0, 1):
detail = "".join(stderr_prefix).strip()
return f"grep: {detail or f'process exited {return_code}'}"
return None
if not os.path.exists(root):
return None, f"grep: search target not found: {_display_tool_path(root)}"
# The pathname-only fast path scans before individual resources can
# be checked. Bound searches must validate every file before read.
rg = None if active_resource_operation() is not None else shutil.which("rg")
if rg:
cmd = [rg, "--line-number", "--with-filename", "--no-heading", "--color=never",
"--max-count", str(max_hits)]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
try:
glob_regex = _glob_to_regex(glob_pat.replace("\\", "/"))
targets = [
target for target in targets
if glob_regex.fullmatch(os.path.relpath(target, base).replace(os.sep, "/"))
or glob_regex.fullmatch(os.path.basename(target))
]
except re.error:
pass
# Validate even when policy filtering leaves no search targets.
if not targets:
cmd_probe = [rg, "--json", "--no-config"]
if glob_pat:
cmd_probe += ["--glob", glob_pat]
cmd_probe += ["--regexp", pattern, "--", "-"]
error = run_rg(cmd_probe)
return (None, error) if error else ([], None)
relative_targets = [os.path.relpath(target, base) for target in targets]
for offset in range(0, len(relative_targets), 128):
if len(lines) >= max_hits:
break
cmd = [
rg, "--json", "--no-config", "--no-follow",
"--max-count", str(max_hits - len(lines)),
"--max-columns", str(_CODENAV_MAX_LINE),
"--max-columns-preview",
]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
cmd += ["--glob", glob_pat]
for sensitive_pattern in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!{sensitive_pattern}"]
for skipped_dir in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{skipped_dir}/**"]
cmd += ["--regexp", pattern, "--", *relative_targets[offset:offset + 128]]
try:
for path in targets[offset:offset + 128]:
check_deadline()
_validate_grep_descriptor(descriptors[path])
if bound is not None:
bound.resolve_path(path)
error = run_rg(cmd)
except (OSError, ValueError) as exc:
return None, f"grep: {exc}"
if error:
return None, error
return lines, None
# This runs inside asyncio.to_thread(), so forking would clone a
# multithreaded process and can deadlock. Spawn is platform-safe and
# PyInstaller-compatible via launcher's early freeze_support().
payload = {
"root": base,
"base": base,
"files": tuple(files),
"pattern": pattern,
"ignore_case": ignore_case,
"glob": glob_pat,
"max_hits": max_hits,
"skip_dirs": tuple(_CODENAV_SKIP_DIRS),
"sensitive_names": tuple(
set(_SENSITIVE_BASENAMES) | set(_SENSITIVE_FILE_PATTERNS)
),
}
try:
context = multiprocessing.get_context("spawn")
output_queue = context.Queue(maxsize=max_hits + 2)
worker = context.Process(
target=_python_grep_worker, args=(payload, output_queue)
)
worker.start()
except Exception as exc:
cmd += ["--glob", glob_pat]
# --iglob (not --glob) so the exclusion is case-insensitive:
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
# resolve to the same secret as their lowercase forms, and the
# Python fallback below already folds case via _is_sensitive_path.
for _pat in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!*{_pat}*"]
for _d in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{_d}/**"]
cmd += ["--regexp", pattern, root]
try:
output_queue.close()
except (NameError, OSError, ValueError):
pass
return None, f"grep: could not start fallback worker: {exc}"
error = None
completed = False
import subprocess
p = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
# ripgrep: 0 = matches, 1 = no matches, 2 = failed scan.
# Do not present invalid patterns or IO failures as absence.
if p.returncode not in (0, 1):
detail = (p.stderr or '').strip()[:1200]
return None, f"grep: search failed (exit {p.returncode}): {detail or 'no diagnostic available'}"
lines = [ln for ln in (p.stdout or "").splitlines() if ln][:max_hits]
return lines, None
except subprocess.TimeoutExpired:
return None, "grep: timed out"
except Exception as _e:
return None, f"grep: {_e}"
try:
while len(lines) < max_hits:
remaining = deadline - time.monotonic()
if remaining <= 0:
error = "grep: timed out"
break
try:
# Keep queue waits short enough to observe a spawn
# worker that dies during bootstrap/import before it
# can enqueue either an error or the done sentinel.
record = output_queue.get(timeout=min(0.05, remaining))
except queue.Empty:
if worker.is_alive():
rx = _re.compile(pattern, _re.IGNORECASE if ignore_case else 0)
except _re.error as _e:
return None, f"grep: bad pattern: {_e}"
hits = []
scan_errors = []
if os.path.isfile(root):
file_iter = [root]
else:
file_iter = []
for dp, dns, fns in os.walk(root, onerror=scan_errors.append):
dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
for fn in fns:
if glob_pat and not fnmatch.fnmatch(fn, glob_pat):
continue
worker.join(timeout=0)
try:
# A multiprocessing queue's feeder can make the
# final record visible at process-exit time. Give
# that record precedence over the exit status.
remaining = deadline - time.monotonic()
record = output_queue.get(
timeout=min(0.05, max(0, remaining))
)
except queue.Empty:
error = f"grep: fallback worker exited {worker.exitcode}"
break
if record[0] == "done":
completed = True
break
if record[0] == "error":
error = record[1]
break
_, path, number, text_value = record
canonical = os.path.realpath(path)
if canonical not in descriptors:
error = "grep: fallback returned an undeclared resource"
break
try:
_validate_grep_descriptor(descriptors[canonical])
if bound is not None:
bound.resolve_path(canonical)
except (OSError, ValueError) as exc:
error = f"grep: {exc}"
break
rendered = f"{_display_tool_path(canonical)}:{number}:{text_value}"
if rendered not in lines:
lines.append(rendered)
finally:
if completed:
worker.join(timeout=min(1, max(0.01, deadline - time.monotonic())))
if worker.is_alive():
worker.terminate()
worker.join(timeout=1)
if worker.is_alive():
worker.kill()
worker.join()
output_queue.close()
if error:
return None, error
if (not completed or worker.exitcode not in (0, None)) and len(lines) < max_hits:
return None, f"grep: fallback worker exited {worker.exitcode}"
return lines, None
file_iter.append(os.path.join(dp, fn))
for fp in file_iter:
if len(hits) >= max_hits:
break
try:
resolved_file = _resolve_tool_path(fp)
except ValueError:
# Apply the same workspace/sensitive-path checks to each
# discovered file, not just the initial search directory.
continue
try:
with open(resolved_file, "r", encoding="utf-8", errors="strict") as f:
for i, line in enumerate(f, 1):
if rx.search(line):
hits.append(f"{fp}:{i}:{line.rstrip()[:_CODENAV_MAX_LINE]}")
if len(hits) >= max_hits:
break
except UnicodeDecodeError:
continue
except OSError as error:
scan_errors.append(error)
if scan_errors:
return None, "grep: search incomplete; one or more files or directories could not be read"
return hits, None
lines, err = await asyncio.to_thread(_grep)
if err:
src/agent_tools/subprocess_tools.py
+7 -16
View File
@@ -11,7 +11,7 @@ import sys
import time
import json
from typing import Optional
from urllib.parse import urlparse, urlsplit, urlunsplit
from urllib.parse import urlparse
import httpx
@@ -76,14 +76,12 @@ def _resolve_fontfile_for_text(text: str) -> str:
async def _cancel_host_shell_bridge_request(
url: str, token: str, request_id: str,
) -> None:
if not token or not is_host_shell_bridge_url_allowed(url):
return
target = host_shell_bridge_endpoint_url(url, "/cancel")
base = url.rsplit("/", 1)[0]
try:
timeout = httpx.Timeout(5.0, connect=2.0, write=2.0, pool=2.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
await client.post(
target,
f"{base}/cancel",
json={"request_id": request_id},
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
@@ -218,12 +216,6 @@ def is_host_shell_bridge_url_allowed(url: str) -> bool:
return True
def host_shell_bridge_endpoint_url(url: str, path: str) -> str:
"""Replace a validated bridge URL's path without changing its authority."""
parsed = urlsplit(url)
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
def _replace_workspace_alias(content: str, cwd: str) -> str:
"""Map virtual /workspace paths without corrupting absolute host paths."""
return re.sub(
@@ -690,7 +682,6 @@ class HostShellTool:
return {"error": "host_shell: invalid bridge URL", "exit_code": 1}
if not token:
return {"error": "host_shell: bridge token missing", "exit_code": 1}
run_url = host_shell_bridge_endpoint_url(url, "/run")
job_id = str(args.get("job_id") or "").strip()
if not command and not job_id:
@@ -725,9 +716,9 @@ class HostShellTool:
request_body["request_id"] = request_id
try:
async with httpx.AsyncClient(timeout=timeout + 5, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout + 5) as client:
resp = await client.post(
run_url,
url,
json=request_body,
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
@@ -755,7 +746,7 @@ class HostShellTool:
while time.monotonic() < deadline:
await asyncio.sleep(0.25)
poll = await client.post(
run_url,
url,
json={"job_id": auto_job_id},
headers={"X-Odysseus-TUI-Bridge-Token": token},
)
src/app_initializer.py
+2 -31
View File
@@ -2,11 +2,10 @@
"""Initialize all application components and dependencies."""
import os
import logging
import stat
from typing import Dict, Any
from src.constants import (
DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR, AGENT_WORKSPACE_DIR,
DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR,
SESSIONS_FILE, DEFAULT_HOST, OPENAI_API_KEY
)
from src.memory import MemoryManager
@@ -31,35 +30,7 @@ def create_directories():
"""Create necessary directories if they don't exist."""
for directory in (DATA_DIR, PERSONAL_DIR, RUNBOOK_DIR, UPLOAD_DIR):
os.makedirs(directory, exist_ok=True)
# The model-controlled workspace must be a real child of DATA_DIR. Never
# follow a pre-existing symlink here: it would silently move the default
# native-file root outside the application volume before any resolver runs.
data_root = os.path.realpath(os.path.abspath(os.path.expanduser(DATA_DIR)))
workspace = os.path.abspath(os.path.expanduser(AGENT_WORKSPACE_DIR))
expected_workspace = os.path.join(data_root, "agent_workspace")
# Validate the real parent so a supported DATA_DIR bind/symlink works, but
# require the fixed internal carve-out name and reject a link at the model-
# controlled workspace entry itself.
if (
os.path.basename(workspace) != "agent_workspace"
or os.path.realpath(os.path.dirname(workspace)) != data_root
):
raise RuntimeError("agent workspace must be the canonical child of DATA_DIR")
if os.path.lexists(workspace):
mode = os.lstat(workspace).st_mode
if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode):
raise RuntimeError("agent workspace must be a real directory")
else:
os.mkdir(workspace, 0o700)
resolved_workspace = os.path.realpath(workspace)
if resolved_workspace != expected_workspace:
raise RuntimeError("agent workspace must be the canonical child of DATA_DIR")
try:
os.chmod(workspace, 0o700)
except OSError:
pass
def initialize_managers(base_dir: str, rag_manager=None) -> Dict[str, Any]:
"""
Initialize all manager and handler instances.
src/builtin_actions.py
+9 -32
View File
@@ -2297,10 +2297,10 @@ async def action_audit_skills(owner: str, **kwargs) -> Tuple[str, bool]:
async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
"""Background note-due scanner. Fires a reminder for any note whose
`due_date` falls in the current ±90-second window and hasn't been delivered
through its configured channel in the last 25 minutes.
`due_date` falls in the current ±5-minute window and hasn't been pinged
within the last 25 minutes. Mirrors `action_ping_events` for calendar.
Per-owner state: {note_id: {at, channel}}, with legacy timestamp support. Pruned
State (`data/note_pings.json`): {note_id: iso_ts_of_last_ping}. Pruned
on each run by dropping entries for notes that are gone/archived/replied.
"""
try:
@@ -2309,11 +2309,6 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
from datetime import datetime as _dt, timezone as _tz, timedelta as _td
from pathlib import Path as _P
from core.database import SessionLocal as _SL, Note as _N
from src.settings import load_settings
channel = load_settings().get("reminder_channel", "browser")
external_channel = channel in ("email", "ntfy", "webhook")
delivery_key = f"{channel}_sent" if external_channel else "browser_sent"
# Per-owner state file so cache-pruning doesn't cross-delete other
# users' entries (review C4). Legacy path kept as fallback so a
@@ -2378,27 +2373,20 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
due = _parse_due(n.due_date)
if not due:
continue
# Inside the due window?
# Inside the ±5min window?
if abs((due - now).total_seconds()) > window.total_seconds():
continue
# Recently pinged? Skip.
last = cache.get(n.id)
browser_recent = False
if last:
try:
last_channel = None
if isinstance(last, dict):
last_channel = last.get("channel")
last = last.get("at")
last_dt = _dt.fromisoformat(str(last))
if last_dt.tzinfo is None:
last_dt = last_dt.replace(tzinfo=_tz.utc)
if last_dt >= reping_cutoff:
if not external_channel or last_channel == channel:
continue
# Browser-only receipts do not prove external
# delivery, but the fallback must not be queued twice.
browser_recent = last_channel in (None, "browser")
continue
except Exception:
pass
# Compose + dispatch.
@@ -2422,26 +2410,15 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
body = "\n\n".join(p for p in body_parts if p) or title
try:
from routes.note_routes import dispatch_reminder
result = await dispatch_reminder(
await dispatch_reminder(
title=title, note_body=body, note_id=n.id,
owner=n.owner or owner or "",
queue_browser=not browser_recent,
)
if result.get("skipped"):
continue
if result.get(delivery_key):
sent.append(title)
else:
logger.warning("ping_notes: %s delivery failed for %s", channel, n.id)
cache[n.id] = now.isoformat()
sent.append(title)
except Exception as e:
logger.warning(f"ping_notes: dispatch failed for {n.id}: {e}")
# Dispatch owns delivery receipts. Reload before pruning so this
# scanner cannot replace a fresh channel receipt with a timestamp.
try:
cache = _json.loads(STATE.read_text(encoding="utf-8")) if STATE.exists() else {}
except Exception:
pass
# Prune cache entries for notes that no longer exist.
for stale in [k for k in cache if k not in seen_ids]:
cache.pop(stale, None)
@@ -2452,7 +2429,7 @@ async def action_ping_notes(owner: str, **kwargs) -> Tuple[str, bool]:
logger.warning(f"ping_notes: cache write failed: {e}")
if not sent:
raise TaskNoop(f"scanned {len(notes)} note(s), no reminders delivered")
raise TaskNoop(f"scanned {len(notes)} note(s), none due in ±{WINDOW_SEC}s")
preview = "; ".join(sent[:3])
extra = f" (+{len(sent) - 3} more)" if len(sent) > 3 else ""
return f"Pinged {len(sent)} note(s): {preview}{extra}", True
src/llm_core.py
+3 -13
View File
@@ -3249,7 +3249,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Ollama stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
return
# ── Anthropic streaming ──
@@ -3402,7 +3402,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Anthropic stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
return
# ── OpenAI-compatible streaming ──
@@ -3872,17 +3872,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat
yield f'event: error\ndata: {json.dumps({"error": "Network error", "status": 502, "fallback_eligible": False})}\n\n'
except Exception as e:
logger.error(f"Stream error: {e}")
yield f'event: error\ndata: {json.dumps({"error": _stream_failure_message(e), "status": 502, "fallback_eligible": False})}\n\n'
def _stream_failure_message(error: BaseException) -> str:
"""Client-facing text for an unexpected streaming failure.
The raw exception can carry request URLs, local paths or provider internals;
callers log it server-side and stream only this generic message, like the
named transport failures above it.
"""
return f"Model stream failed ({type(error).__name__})"
yield f'event: error\ndata: {json.dumps({"error": str(e), "status": 502, "fallback_eligible": False})}\n\n'
def _summarize_stream_error(err_chunk: Optional[str]) -> str:
src/readiness.py
+2 -9
View File
@@ -6,14 +6,11 @@ writable, and storage is local-first. Served by ``GET /api/ready`` and suitable
for an orchestrator readiness probe (200 only when every critical check passes).
"""
import logging
import os
import uuid
from datetime import datetime
from typing import Dict
logger = logging.getLogger(__name__)
def check_readiness() -> Dict[str, object]:
"""Run the readiness checks and return a JSON-serialisable report.
@@ -36,10 +33,7 @@ def check_readiness() -> Dict[str, object]:
conn.execute(sql_text("SELECT 1"))
checks["database"] = {"ok": True}
except Exception as e:
# The raw driver error can carry the DB host/user/path; keep it in the
# server log and give the client only the exception type.
logger.warning("Readiness database check failed: %s", e)
checks["database"] = {"ok": False, "error_type": type(e).__name__}
checks["database"] = {"ok": False, "error": str(e)}
# Data directory present and writable — home must be able to hold its own data.
try:
@@ -50,8 +44,7 @@ def check_readiness() -> Dict[str, object]:
os.remove(probe)
checks["data_dir"] = {"ok": True, "path": DATA_DIR}
except Exception as e:
logger.warning("Readiness data_dir check failed: %s", e)
checks["data_dir"] = {"ok": False, "error_type": type(e).__name__}
checks["data_dir"] = {"ok": False, "error": str(e)}
# Local-first: storage stays on the home machine (informational, never fatal).
local_first = (
src/shutdown.py
+34
View File
@@ -0,0 +1,34 @@
"""Process-wide shutdown signalling for graceful restarts.
Server-Sent-Events / streaming responses never complete on their own, so
uvicorn's graceful shutdown would wait for an open browser stream up to the
full ``timeout_graceful_shutdown`` window before exiting. A SIGTERM/SIGINT
handler flips this flag and tells uvicorn to exit; every streaming generator
checks the flag on each loop and ends promptly, so a restart completes in
seconds while ordinary in-flight requests still finish.
"""
from __future__ import annotations
import logging
import threading
logger = logging.getLogger(__name__)
_SHUTTING_DOWN = threading.Event()
def is_shutting_down() -> bool:
"""True once a shutdown (SIGTERM/SIGINT) has been requested."""
return _SHUTTING_DOWN.is_set()
def request_shutdown() -> None:
"""Mark the process as shutting down (idempotent)."""
if not _SHUTTING_DOWN.is_set():
_SHUTTING_DOWN.set()
logger.info("shutdown requested; streaming responses will end promptly")
def reset() -> None:
"""Clear the flag (for tests)."""
_SHUTTING_DOWN.clear()
src/tool_approval_scopes.py
-125
View File
@@ -2,12 +2,7 @@
from __future__ import annotations
import hmac
import logging
from enum import Enum
from hashlib import sha256
logger = logging.getLogger(__name__)
# Keep the existing wire values so the current route and no-build frontend do
@@ -21,126 +16,6 @@ DENY_APPROVAL_DECISION = "deny"
# session history contains a matching, resolved chat-session approval.
CHAT_SESSION_APPROVAL_CONTEXT_MARKER = "_tool_approval_chat_session_granted"
# The server's proof that IT resolved this approval. More than one route
# writes caller-supplied metadata into session history, so a client can write
# the shape of a resolved card directly; only the server can produce this.
CHAT_SESSION_APPROVAL_SIGNATURE_FIELD = "_server_grant"
def _grant_key() -> bytes | None:
"""Key material for grant signatures, or None when it is unavailable.
Reuses the persistent application key so a grant survives a restart the
way the transcript holding it does.
"""
try:
from src.secret_storage import _load_or_create_key
return _load_or_create_key()
except Exception as exc:
logger.warning("Tool approval grant key unavailable: %s", exc)
return None
def sign_chat_session_grant(
session_id: object,
approval_id: object,
decision: object,
) -> str | None:
"""Return the server's signature for one resolved chat-session grant."""
key = _grant_key()
if key is None:
return None
payload = "\x00".join(
(
str(session_id or ""),
str(approval_id or ""),
str(decision or "").strip().lower(),
)
)
return hmac.new(key, payload.encode("utf-8"), sha256).hexdigest()
# Message-metadata keys the server writes and a caller never should. Both are
# read back as authority: ``tool_events`` carries the approval cards, and the
# context marker is projected onto a turn once a grant is found.
_SERVER_OWNED_METADATA_KEYS = (
"tool_events",
CHAT_SESSION_APPROVAL_CONTEXT_MARKER,
)
def sanitize_client_message_metadata(metadata):
"""Drop server-owned keys from a caller-supplied message metadata blob.
Routes that persist a message on the caller's behalf accept this blob
verbatim, which lets a caller write the shape of a resolved approval into
its own transcript. The grant check verifies a signature, so this is not
the control that closes that path; it keeps the state out of the
transcript in the first place. Anything else in the blob is left alone.
"""
if not isinstance(metadata, dict):
return metadata
if not any(key in metadata for key in _SERVER_OWNED_METADATA_KEYS):
return metadata
return {
key: value
for key, value in metadata.items()
if key not in _SERVER_OWNED_METADATA_KEYS
}
def stamp_chat_session_grant(
ask_user: dict,
session_id: object,
decision: object,
) -> None:
"""Record the server's grant on a card it has just resolved.
Call this only from the server-side resolve path. A decision that does not
grant chat-session scope leaves no signature behind, so downgrading a
``deny`` to an ``approve`` in the transcript does not carry a usable one.
"""
if not isinstance(ask_user, dict):
return
if str(decision or "").strip().lower() != CHAT_SESSION_APPROVAL_DECISION:
ask_user.pop(CHAT_SESSION_APPROVAL_SIGNATURE_FIELD, None)
return
signature = sign_chat_session_grant(
session_id,
ask_user.get("approval_id"),
CHAT_SESSION_APPROVAL_DECISION,
)
if signature:
ask_user[CHAT_SESSION_APPROVAL_SIGNATURE_FIELD] = signature
def verify_chat_session_grant(
signature: object,
session_id: object,
approval_id: object,
decision: object,
) -> bool:
"""Whether *signature* is this server's grant for that exact approval.
Fails CLOSED: an absent, malformed, or unverifiable signature is not a
grant. Binding the session and approval ids into the payload means a
signature lifted from one chat cannot be replayed into another.
"""
# compare_digest accepts only ASCII strings. Treat arbitrary persisted
# metadata as untrusted and require the exact representation we sign.
if (
not isinstance(signature, str)
or len(signature) != sha256().digest_size * 2
or any(character not in "0123456789abcdef" for character in signature)
):
return False
expected = sign_chat_session_grant(session_id, approval_id, decision)
if expected is None:
return False
return hmac.compare_digest(signature, expected)
class ToolApprovalScope(str, Enum):
# Surfaces without a resumable chat (the skill tester, unattended audits)
src/tool_capabilities.py
+18 -76
View File
@@ -17,7 +17,7 @@ from types import MappingProxyType
from typing import Any, Iterable, Mapping
from src.tool_approval_scopes import CHAT_SESSION_APPROVAL_CONTEXT_MARKER
from src.tool_security import BUILTIN_EMAIL_TOOLS, is_public_blocked_tool
from src.tool_security import BUILTIN_EMAIL_TOOLS
class ToolEffect(str, Enum):
@@ -216,6 +216,20 @@ _register(
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"ai_edit_image"},
ToolEffect.NETWORK_EGRESS,
ToolEffect.WRITE_PRIVATE,
result_integrity=ResultIntegrity.EXTERNAL_UNTRUSTED,
)
_register(
{"get_editor_screenshot"},
ToolEffect.READ_PRIVATE,
)
_register(
{"plan_edit", "run_editor_command"},
ToolEffect.UI_SIDE_EFFECT,
)
_register(
{
"archive_email",
@@ -501,43 +515,6 @@ def capabilities_for_action(tool_name: Any, content: Any) -> ToolCapabilities:
if not isinstance(tool_name, str):
return base
if tool_name in {"web_fetch", "pdf_extract"}:
payload = content
raw = content.strip() if isinstance(content, str) else ""
if isinstance(payload, str):
try:
payload = json.loads(raw) if raw.startswith("{") else {}
except (TypeError, ValueError):
payload = {}
if not isinstance(payload, Mapping):
payload = {}
if tool_name == "web_fetch" and "urls" in payload:
urls = payload["urls"]
sources = []
if isinstance(urls, list):
for item in urls:
source = item.get("url") if isinstance(item, Mapping) else item
sources.append(str(source or "").strip())
else:
source = payload.get("url") or (payload.get("path") if tool_name == "pdf_extract" else "")
sources = [str(source or "").strip() or raw.split("\n", 1)[0].strip()]
# Match the native readers' local selectors without resolving or
# opening files. A mixed batch retains its network effects as well.
if tool_name == "web_fetch":
local = [source.startswith("/workspace/") or source.lower().startswith("file:///workspace/")
for source in sources]
else:
local = [os.path.isabs(source) or source.lower().startswith("file://") for source in sources]
if any(local):
effects = {ToolEffect.READ_WORKSPACE}
if not all(local):
effects.update(base.effects)
return ToolCapabilities(
frozenset(effects),
ResultIntegrity.WORKSPACE_UNTRUSTED if all(local) else base.result_integrity,
known=base.known,
)
if tool_name == "extract_text":
payload = content
if isinstance(payload, str):
@@ -678,13 +655,9 @@ POST_EXTERNAL_BLOCKED_EFFECTS = frozenset(
)
# On by default: until agent processes run without network and side-effecting
# non-process tools have their own exact-approval boundary, this gate is the
# only check between injected external content and those tools. Set it to a
# falsy value to opt out.
TOOL_APPROVAL_GATE_ENABLED = (
str(os.getenv("ODYSSEUS_TOOL_APPROVAL_GATE", "1")).strip().lower()
not in {"0", "false", "no", "off"}
str(os.getenv("ODYSSEUS_TOOL_APPROVAL_GATE", "0")).strip().lower()
in {"1", "true", "yes", "on"}
)
@@ -751,18 +724,6 @@ def messages_contain_external_untrusted_context(messages: Iterable[dict]) -> boo
return False
def delegated_tool_is_blocked(tool_name: Any, content: Any = None) -> bool:
"""Filter tool names for discovery and concrete workspace reads at dispatch."""
# The bridge exposes these spellings for the same filesystem surfaces.
if isinstance(tool_name, str):
tool_name = tool_name.strip()
tool_name = {"list_dir": "ls", "find_files": "glob"}.get(tool_name, tool_name)
return is_public_blocked_tool(tool_name) or (
content is not None
and ToolEffect.READ_WORKSPACE in capabilities_for_action(tool_name, content).effects
)
@dataclass
class ToolRunSecurityContext:
"""Server-owned integrity state for one agent run."""
@@ -779,19 +740,11 @@ class ToolRunSecurityContext:
# The bypass affects only this automatic gate; current tool policy, ownership,
# workspace confinement, and execution/sandbox restrictions still apply.
approval_gate_bypassed: bool = False
# Driven by a bearer API token, not a person at a browser. Privileged
# tools are refused outright and no approval can lift that.
delegated_credential: bool = False
def observe_messages(self, messages: Iterable[dict]) -> None:
"""Apply server-owned chat scope and promote untrusted prompt context."""
message_list = list(messages or ())
if self.delegated_credential:
# A delegated run has no human to grant chat-session scope, so a
# grant sitting in this chat's history (left by the owner's own
# browser) must not be picked up by a token driving the same chat.
self.approval_gate_bypassed = False
elif any(
if any(
isinstance(message, dict)
and isinstance(message.get("metadata"), dict)
and message["metadata"].get(
@@ -816,17 +769,6 @@ class ToolRunSecurityContext:
self.external_untrusted_context_seen = True
def decision_for(self, tool_name: Any, content: Any = None) -> ToolGateDecision:
# Checked before the bypasses below, because neither may lift it, and
# kept independent of external_untrusted_context_seen so it holds on a
# run where that gate never arms and raises no prompt to bypass.
if self.delegated_credential and delegated_tool_is_blocked(tool_name, content):
return ToolGateDecision(
False,
(
f"Tool '{tool_name}' is not available to API-token callers. "
"It requires an interactive session."
),
)
if not TOOL_APPROVAL_GATE_ENABLED:
return ToolGateDecision(True)
if self.approval_gate_bypassed:
src/tool_execution.py
+39 -308
View File
@@ -15,7 +15,6 @@ import logging
import os
import pathlib
import re
import stat
import secrets
import sys
import time
@@ -37,7 +36,6 @@ from src.tool_policy import ToolPolicy
from src.client_tool_contract import TUI_ROUTED_BRIDGE_TOOL_NAMES
from src.constants import (
DATA_DIR,
AGENT_WORKSPACE_DIR,
MAX_DIFF_LINES,
MAX_OUTPUT_CHARS,
MAX_READ_CHARS,
@@ -71,11 +69,11 @@ _MISSING_TOOL_SECURITY_CONTEXT = _MissingToolSecurityContext()
NO_TOOL_SECURITY_CONTEXT = _NoToolSecurityContext()
# Persistent working directory for agent subprocesses.
# Resolves to <repo_root>/data/agent_workspace, inside the bind-mounted volume
# in Docker (/app/data), so files survive a rebuild as before. The subdirectory
# rather than data/ itself keeps agent scratch files and dotfiles out of the
# directory holding the session store and the auth database.
_AGENT_WORKDIR = AGENT_WORKSPACE_DIR
# Resolves to <repo_root>/data, which is the bind-mounted volume in Docker
# (/app/data) and the local data directory for manual installs.
# Using this as cwd and HOME prevents the agent from silently creating files
# in ephemeral container layers that are lost on the next rebuild.
_AGENT_WORKDIR = DATA_DIR
ExecutionBridgeHandler = Callable[
@@ -135,12 +133,6 @@ def get_active_execution_bridge() -> AgentExecutionBridge | None:
return _active_execution_bridge.get()
def _tui_host_bridge_endpoint_url(url: str, path: str) -> str:
"""Replace a validated bridge URL's path without changing its authority."""
from src.agent_tools.subprocess_tools import host_shell_bridge_endpoint_url
return host_shell_bridge_endpoint_url(url, path)
def _tui_host_bridge_patch_url(
client_runtime_context: Optional[Dict[str, Any]],
) -> tuple[str, str] | None:
@@ -161,7 +153,11 @@ def _tui_host_bridge_patch_url(
from src.agent_tools.subprocess_tools import is_host_shell_bridge_url_allowed
if not is_host_shell_bridge_url_allowed(url):
return None
return _tui_host_bridge_endpoint_url(url, "/patch"), token
if url.endswith("/run"):
url = url[:-4] + "/patch"
elif not url.endswith("/patch"):
url += "/patch"
return url, token
async def _apply_patch_via_tui_host_bridge(
@@ -177,7 +173,7 @@ async def _apply_patch_via_tui_host_bridge(
import httpx
timeout = httpx.Timeout(125.0, connect=5.0, write=10.0, pool=5.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(
url,
headers={"x-odysseus-tui-bridge-token": token},
@@ -237,7 +233,7 @@ async def _bridge_post(bridge: Dict, path: str, payload: Dict, *, timeout_s: flo
"error": f"{err_prefix}: invalid TUI host bridge",
"exit_code": 1,
}
target = _tui_host_bridge_endpoint_url(url, path)
base = url.rsplit("/", 1)[0] if url.endswith(("/run", "/read", "/write")) else url.rstrip("/")
try:
import httpx
timeout = httpx.Timeout(
@@ -246,9 +242,9 @@ async def _bridge_post(bridge: Dict, path: str, payload: Dict, *, timeout_s: flo
write=10.0,
pool=5.0,
)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(
target,
f"{base}{path}",
headers={"x-odysseus-tui-bridge-token": token},
json=payload,
)
@@ -304,13 +300,13 @@ async def _cancel_bridge_request(bridge: Dict, request_id: str) -> None:
from src.agent_tools.subprocess_tools import is_host_shell_bridge_url_allowed
if not token or not is_host_shell_bridge_url_allowed(url):
return
target = _tui_host_bridge_endpoint_url(url, "/cancel")
base = url.rsplit("/", 1)[0]
try:
import httpx
timeout = httpx.Timeout(5.0, connect=2.0, write=2.0, pool=2.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
async with httpx.AsyncClient(timeout=timeout) as client:
await client.post(
target,
f"{base}/cancel",
headers={"x-odysseus-tui-bridge-token": token},
json={"request_id": request_id},
)
@@ -733,7 +729,7 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
path = path_value.strip() if isinstance(path_value, str) else ""
if "\n" in path or "\r" in path:
path = ""
body_value = args.get("content")
body_value = args.get("content", "")
if not isinstance(body_value, str):
return "write_file: invalid arguments", {
"error": "write_file: content must be a string",
@@ -743,7 +739,7 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
else:
path, _, body = content.partition("\n")
path = path.strip()
if not path or path.endswith(("/", "\\")):
if not path:
return "write_file: invalid arguments", {
"error": "write_file: path is required",
"exit_code": 1,
@@ -751,17 +747,6 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
rejected = _text_write_to_binary_artifact_result(content)
if rejected is not None:
return rejected
from src.agent_tools.filesystem_tools import _parse_write_intent, _unwrap_fenced_source_body
try:
_, original_body, section, explicit_clear = _parse_write_intent(content)
except ValueError as error:
return "write_file: invalid arguments", {"error": str(error), "exit_code": 1}
body = _unwrap_fenced_source_body(body, path)
if not section or (not body.strip() and not explicit_clear):
return "write_file: ambiguous content", {
"error": "write_file: bridge cannot safely enforce implicit empty content; use explicit JSON string content",
"exit_code": 1,
}
return f"write_file: {path[:80]}", await _bridge_post(
bridge,
"/write",
@@ -788,15 +773,10 @@ async def _route_tool_via_bridge(tool: str, content: str, session_id: Optional[s
# 1. Sensitive-subpath deny list — checked FIRST. Blocks .ssh,
# .gnupg, shell rc files, token/env files even if the root above
# them is on the allowlist.
# 2. Application-state deny (_is_app_state_path) - DATA_DIR holds the
# session store, auth database, app key and settings, so only
# _agent_readable_data_subdirs() is readable inside it.
# 3. Allowlist - only the directories the agent legitimately needs
# (its data/ workspace, user content, system tmp). $HOME is NOT on
# the default list.
# 4. Opt-in extra roots - admin can add broader roots via the
# "tool_path_extra_roots" setting. These cannot re-open DATA_DIR;
# rule 2 is independent of which root a path arrived through.
# 2. Allowlist — only the directories the agent legitimately needs
# (project data/, system tmp). $HOME is NOT on the default list.
# 3. Opt-in extra roots — admin can add broader roots via the
# "tool_path_extra_roots" setting (list of path strings).
# ---------------------------------------------------------------------------
# Compatibility exports: the same deny predicate protects tool access and
@@ -807,191 +787,6 @@ from src.agent_runtime.path_policy import (
)
def _path_within(resolved: str, root: str) -> bool:
"""True when *resolved* is *root* itself or sits underneath it.
Use the platform's path-case rules. This helper participates in allow
decisions, so unconditional case-folding would let a distinct ``/DATA``
tree masquerade as a descendant of ``/data`` on case-sensitive systems.
"""
resolved, root = os.path.normcase(resolved), os.path.normcase(root)
if resolved == root:
return True
try:
if os.path.commonpath([resolved, root]) == root:
return True
except ValueError:
return False
# normcase is intentionally conservative about assumptions (notably on
# POSIX), so consult the filesystem when paths exist. This recognizes a
# case alias on a case-insensitive volume without treating distinct
# case-sensitive paths as the same allow root.
if os.path.exists(root):
candidate = resolved
while True:
try:
if os.path.exists(candidate) and os.path.samefile(candidate, root):
return True
except OSError:
pass
parent = os.path.dirname(candidate)
if parent == candidate:
break
candidate = parent
return False
def _path_within_conservative(resolved: str, root: str) -> bool:
"""Containment for deny decisions, folding case to fail closed."""
resolved, root = resolved.casefold(), root.casefold()
if resolved == root:
return True
try:
return os.path.commonpath([resolved, root]) == root
except ValueError:
return False
def _agent_readable_data_subdirs() -> tuple[str, ...]:
"""The only parts of DATA_DIR the agent's file tools may reach.
The agent's own scratch folder, plus the directories of user content whose
paths the application itself gives to the model, which it would then be
unable to open. These normally live under DATA_DIR; the documented mail
attachment override may instead name a disjoint external directory:
UPLOAD_DIR the chat upload manifest renders "path=<p>" and
says to read it with read_file (agent_loop.py)
MAIL_ATTACHMENTS_DIR download_attachment returns the path and its own
description tells the model to read it
PERSONAL_DIR GET /api/personal returns a path per file and is
reachable through the app_api tool; RUNBOOK_DIR
nests under it
PERSONAL_UPLOADS_DIR indexed as a personal-docs directory, which
manage_rag lists as an absolute path
Order matters: the first entry is roots[0], which _resolve_search_root uses
when grep/glob/ls are called with no path.
"""
from src.constants import (
DATA_DIR,
MAIL_ATTACHMENTS_DIR,
PERSONAL_DIR,
PERSONAL_UPLOADS_DIR,
UPLOAD_DIR,
)
configured = (
(AGENT_WORKSPACE_DIR, "agent_workspace", False),
(UPLOAD_DIR, "uploads", False),
# This has a documented environment override and may legitimately
# live outside DATA_DIR, but it must never equal/contain DATA_DIR.
(MAIL_ATTACHMENTS_DIR, "mail-attachments", True),
(PERSONAL_DIR, "personal_docs", False),
(PERSONAL_UPLOADS_DIR, "personal_uploads", False),
)
configured_data_dir = os.path.abspath(os.path.expanduser(str(DATA_DIR)))
data_dir = os.path.realpath(configured_data_dir)
safe: list[str] = []
for raw, internal_name, external_ok in configured:
value = str(raw or "").strip()
# These paths are security-policy roots, not ordinary allowlist
# entries. Internal roles may inherit a relative DATA_DIR, but must
# still resolve to their exact canonical child below. External mail
# overrides require an absolute, disjoint directory.
if not value:
continue
expanded = os.path.abspath(os.path.expanduser(value))
# A policy root must not acquire an exemption by redirecting its final
# path component to protected state or to an unrelated external tree.
if os.path.islink(expanded):
continue
resolved = os.path.realpath(expanded)
if os.path.exists(resolved) and not os.path.isdir(resolved):
continue
expected_internal = os.path.join(data_dir, internal_name)
expected_configured = os.path.join(configured_data_dir, internal_name)
inside_data = (
os.path.normcase(expanded)
in {
os.path.normcase(expected_configured),
os.path.normcase(expected_internal),
}
and resolved == expected_internal
)
external_safe = (
external_ok
and os.path.isabs(os.path.expanduser(value))
and resolved != data_dir
and os.path.dirname(resolved) != resolved
and not _path_within(data_dir, resolved)
and not _path_within(resolved, data_dir)
)
if not (inside_data or external_safe) or _is_sensitive_path(resolved):
continue
safe.append(resolved)
return tuple(safe)
def _is_app_state_path(resolved: str) -> bool:
"""True for anything under DATA_DIR that is not agent-readable.
DATA_DIR holds the session store, the auth database, the app encryption key
and the settings file. A model-supplied path must not reach those through
any root, so this is checked in both resolvers rather than expressed as an
absence from the allowlist: a workspace bound at or above the data
directory, or an opt-in tool_path_extra_roots entry covering it, would
otherwise put them back in reach.
A containment rule rather than a filename deny list, so state files added
later are covered without anyone remembering to list them. The composed
sensitive-name policy also denies settings.json and app.db in workspaces.
"""
from src.constants import DATA_DIR
if not _path_within_conservative(resolved, os.path.realpath(DATA_DIR)):
return False
return not any(
_path_within(resolved, d)
for d in _agent_readable_data_subdirs()
)
def _is_hardlinked_regular_file(resolved: str) -> bool:
"""Reject inode aliases that can smuggle DATA_DIR state into an allow root."""
try:
target = os.stat(resolved, follow_symlinks=False)
except OSError:
return False
return stat.S_ISREG(target.st_mode) and getattr(target, "st_nlink", 1) > 1
def _is_denied_tool_path(resolved: str, *, snapshot=None) -> bool:
"""Apply every path deny to a canonical traversal result.
Directory scans pass one ``_control_plane_snapshot()`` for the whole walk;
rebuilding it per entry made grep/ls/glob linear in protected-state stats.
"""
from src.agent_runtime.resources import _control_plane_path
return (
_is_sensitive_path(resolved)
or _is_app_state_path(resolved)
or _is_hardlinked_regular_file(resolved)
or _control_plane_path(resolved, snapshot=snapshot)
)
def _can_traverse_tool_path(resolved: str, *, snapshot=None) -> bool:
"""Allow walking a denied state parent only to reach safe carve-outs."""
if _is_sensitive_path(resolved):
return False
if not _is_app_state_path(resolved):
from src.agent_runtime.resources import _control_plane_path
return not _control_plane_path(resolved, snapshot=snapshot)
return any(
_path_within(readable, resolved)
for readable in _agent_readable_data_subdirs()
)
def _tool_path_roots() -> list[str]:
"""Return the list of directory roots that read_file / write_file
may touch. Default: project data/ + system temp dirs. Extra roots
@@ -999,9 +794,9 @@ def _tool_path_roots() -> list[str]:
"""
roots: list[str] = []
# The agent's workspace plus the user-content directories inside data/.
# The rest of DATA_DIR is denied by _is_app_state_path.
roots.extend(_agent_readable_data_subdirs())
# Project data directory — the agent's primary workspace.
from src.constants import DATA_DIR
roots.append(DATA_DIR)
# /tmp (and its macOS realpath /private/tmp).
roots.append("/tmp")
@@ -1058,10 +853,7 @@ def _resolve_tool_path(raw_path: str) -> str:
"""
resource_operation = active_resource_operation()
if resource_operation is not None:
resolved = resource_operation.resolve_path(raw_path)
if _is_denied_tool_path(resolved):
raise ValueError("bound path is sensitive, application state or a hard-linked file")
return resolved
return resource_operation.resolve_path(raw_path)
ws = get_active_workspace()
if ws:
return _resolve_tool_path_in_workspace(ws, raw_path)
@@ -1070,20 +862,11 @@ def _resolve_tool_path(raw_path: str) -> str:
expanded = os.path.expanduser(str(raw_path).strip())
resolved = os.path.realpath(expanded)
if _is_app_state_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside the application state directory"
)
if _is_sensitive_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside a sensitive directory "
f"(e.g. .ssh, .gnupg) or matches a sensitive filename"
)
if _is_hardlinked_regular_file(resolved):
raise ValueError(f"path '{raw_path}' is a hard-linked file")
if _is_denied_tool_path(resolved):
raise ValueError("path addresses sensitive execution control-plane state")
for root in _tool_path_roots():
if is_inside(root, resolved):
@@ -1120,19 +903,11 @@ def _resolve_tool_path_in_workspace(workspace: str, raw_path: str) -> str:
# Confinement says "inside the root"; the deny list says "allowed". They
# are separate questions and this one stays here, with the policy that
# owns it.
if _is_app_state_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside the application state directory"
)
if _is_sensitive_path(resolved):
raise ValueError(
f"path '{raw_path}' is inside a sensitive directory "
f"(e.g. .ssh, .gnupg) or matches a sensitive filename"
)
if _is_hardlinked_regular_file(resolved):
raise ValueError(f"path '{raw_path}' is a hard-linked file")
if _is_denied_tool_path(resolved):
raise ValueError("path addresses sensitive execution control-plane state")
return resolved
@@ -1191,10 +966,6 @@ def vet_workspace(raw: str) -> Optional[str]:
resolved = os.path.realpath(os.path.expanduser(raw))
if not os.path.isdir(resolved) or _is_sensitive_path(resolved):
return None
# Refuse the bind rather than binding a workspace where every subsequent
# tool call would fail on the same deny list.
if _is_denied_tool_path(resolved):
return None
# Reject filesystem roots: binding / (or a Windows drive/UNC root) as the
# workspace would make every absolute path "inside" it, collapsing the
# confinement into host-wide file access. A root is its own dirname, which
@@ -1206,25 +977,11 @@ def vet_workspace(raw: str) -> Optional[str]:
def agent_cwd() -> str:
"""Working directory for agent subprocesses (bash/python/background jobs):
a validated sealed launch root, active workspace, or agent workspace."""
the active workspace when set, else the persistent data dir."""
from src.agent_runtime.process_resources import active_process_operation
bound = active_process_operation()
if bound is not None and bound.launch is not None:
bound.launch.scope.validate()
root = bound.launch.scope.root.path
if vet_workspace(root) != root:
raise RuntimeError("sealed process root is not a safe real directory")
return root
workspace = get_active_workspace()
if workspace:
vetted = vet_workspace(workspace)
if vetted is None:
raise RuntimeError("active workspace is not a safe real directory")
return vetted
resolved = os.path.realpath(_AGENT_WORKDIR)
if resolved not in _agent_readable_data_subdirs() or vet_workspace(resolved) != resolved:
raise RuntimeError("agent workspace is not a safe real directory")
return resolved
return (bound.launch.scope.root.path if bound is not None and bound.launch is not None
else get_active_workspace() or _AGENT_WORKDIR)
def get_mcp_manager():
@@ -1239,28 +996,19 @@ def _resolve_search_root(raw_path: str) -> str:
With a workspace active, the workspace folder is the root and a supplied
path is confined inside it. Otherwise an empty path defaults to the agent's
primary root (its workspace under the project data dir) and a supplied path
is confined by the global allowlist + sensitive-file policy.
primary root (project data dir) and a supplied path is confined by the
global allowlist + sensitive-file policy.
"""
resource_operation = active_resource_operation()
if resource_operation is not None:
resolved = resource_operation.resolve_path(raw_path, search=True)
if _is_denied_tool_path(resolved):
raise ValueError("bound search root is sensitive, application state or a hard-linked file")
return resolved
return resource_operation.resolve_path(raw_path, search=True)
raw = (raw_path or "").strip()
ws = get_active_workspace()
if ws:
# Resolve the empty case as the workspace path rather than returning
# it directly: returned unchecked it skipped both deny lists, so a
# bare ls listed whatever the workspace was bound to.
return _resolve_tool_path_in_workspace(ws, raw or ws)
return os.path.realpath(ws) if not raw else _resolve_tool_path_in_workspace(ws, raw)
if not raw:
roots = _tool_path_roots()
default_root = os.path.realpath(AGENT_WORKSPACE_DIR)
if default_root in roots and not _is_denied_tool_path(default_root):
return default_root
raise ValueError("default agent workspace is not a safe readable data subdirectory")
return roots[0] if roots else os.path.realpath(".")
return _resolve_tool_path(raw)
logger = logging.getLogger(__name__)
@@ -1651,14 +1399,6 @@ async def execute_tool_block(
"NO_TOOL_SECURITY_CONTEXT"
)
# Credential denial precedes authority, exact approval claim and bridges.
if (isinstance(security_context, ToolRunSecurityContext)
and security_context.delegated_credential):
from src.tool_capabilities import delegated_tool_is_blocked
if delegated_tool_is_blocked(getattr(block, "tool_type", None), getattr(block, "content", None)):
decision = security_context.decision_for(block.tool_type, block.content)
return blocked_tool_result(block.tool_type, decision.reason)
authority = active_request_authority() if request_authority is MISSING_AUTHORITY else request_authority
parent = active_request_authority()
if isinstance(authority, RequestAuthority) and parent is not None and authority is not parent:
@@ -2082,17 +1822,6 @@ async def _execute_tool_block_impl(
mark_authorized()
if bridge_owns_tool:
if tool == "write_file":
from src.agent_tools.filesystem_tools import _parse_write_intent, _unwrap_fenced_source_body
try:
path, body, section, explicit_clear = _parse_write_intent(content)
if not path or path.endswith(("/", "\\")) or "\n" in path or "\r" in path:
raise ValueError("write_file: valid file path required")
body = _unwrap_fenced_source_body(body, path)
if not section or (not body.strip() and not explicit_clear):
raise ValueError("write_file: bridge cannot safely enforce implicit empty content; use explicit JSON string content")
except ValueError as error:
return "write_file: invalid arguments", {"error": str(error), "exit_code": 1}
try:
return await dispatched(execution_bridge.route_tool(
tool,
@@ -2164,7 +1893,8 @@ async def _execute_tool_block_impl(
if tool == "generate_image":
from src.ai_interaction import do_generate_image
desc = "generate_image"
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_generate_image(content, session_id=session_id, owner=owner, **({'incognito': True} if image_privacy else {})))
elif (tool in NATIVE_FILESYSTEM_TOOLS
and (active_resource_operation() is not None or tool != "apply_patch"
or not _tui_host_bridge_patch_url(client_runtime_context))):
@@ -2324,7 +2054,8 @@ async def _execute_tool_block_impl(
result = await dispatched(do_list_cookbook_servers(content, owner=owner))
elif tool == "edit_image":
desc = "edit_image"
result = await dispatched(do_edit_image(content, owner=owner))
image_privacy = (client_runtime_context or {}).get('web_ui_state', {}).get('incognito') is True
result = await dispatched(do_edit_image(content, owner=owner, **({'incognito': True} if image_privacy else {})))
elif tool == "edit_file":
result = await dispatched(_direct_fallback(tool, content)) or {"error": "edit failed", "exit_code": 1}
desc = result.get("output") or result.get("error") or "edit_file"
src/tool_parsing.py
+25 -163
View File
@@ -194,33 +194,10 @@ _TOOL_CODE_OPEN_RE = re.compile(r"<tool_code>\s*\{", re.IGNORECASE)
_TOOL_CODE_CLOSE_RE = re.compile(r"\}\s*</tool_code>", re.IGNORECASE)
# Pattern 4b: Gemma-style <|tool_call|> call:tool_name{args} <tool_call|>
_GEMMA_TOOL_CALL_OPEN_RE = re.compile(
r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*\{",
_GEMMA_TOOL_CALL_RE = re.compile(
r"<\|?tool_call\|?>\s*call:([\w\d_-]+)\s*(\{[\s\S]*?\})\s*<\|?tool_call\|?>",
re.IGNORECASE,
)
_GEMMA_TOOL_CALL_CLOSE_RE = re.compile(
r"\}\s*<\|?tool_call\|?>",
re.IGNORECASE,
)
# Native Qwen markup shares the same non-nesting delimiter grammar as the
# XML helpers. Literal closers keep whitespace and opener floods linear;
# values are stripped by the caller, as in the original regex path.
_QWEN_FUNCTION_OPEN_RE = re.compile(r"<function=([A-Za-z_][\w:.-]*)>\s*")
_QWEN_FUNCTION_CLOSE_RE = re.compile(r"</function>")
_QWEN_PARAMETER_OPEN_RE = re.compile(r"<parameter=([A-Za-z_]\w*)>\s*")
_QWEN_PARAMETER_CLOSE_RE = re.compile(r"</parameter>")
_QWEN_PYTHON_ARG_KEY_RE = re.compile(r"[A-Za-z_]\w*")
_QWEN_PYTHON_ARG_VALUE_RE = re.compile(r"\s*=\s*(['\"].*?['\"]|[^,]+)")
_ANGLE_TAG_OPEN_RE = re.compile(r"<")
_NONEMPTY_ANGLE_TAG_OPEN_RE = re.compile(r"<(?=[^>])")
_ANGLE_TAG_CLOSE_RE = re.compile(r">")
_EMAIL_LOCAL_RE = re.compile(r"[\w.+-]+")
_EMAIL_ADDRESS_RE = re.compile(r"[\w.+-]+@[\w.-]+\.\w+")
_ASCII_EMAIL_LOCAL_RE = re.compile(r"[A-Za-z0-9.!#$%&\x27*+/=?^_`{|}~-]+")
_ASCII_EMAIL_ADDRESS_RE = re.compile(
r"[A-Za-z0-9.!#$%&\x27*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
)
# Pattern 4c: Open-function wrapper emitted by some local MLX/Exo models.
# Example:
@@ -565,34 +542,6 @@ _PLAIN_UI_OPEN_PANEL_RE = re.compile(
r"((?:\s+(?:day|week|month|year|agenda)(?:\s+view)?(?:\s+\d{4}-\d{2}(?:-\d{2})?)?)?)"
r"\s*(?:`{1,3})?\s*$"
)
_PLAIN_UI_CANDIDATE_RE = re.compile(r"ui_control", re.IGNORECASE)
def _iter_plain_ui_open_panel(text: str):
"""Yield line-anchored UI commands without retrying every blank line."""
pos = 0
while candidate := _PLAIN_UI_CANDIDATE_RE.search(text, pos):
line_start = text.rfind("\n", 0, candidate.start()) + 1
match = _PLAIN_UI_OPEN_PANEL_RE.match(text, line_start)
if match is not None:
yield match
pos = match.end()
continue
newline = text.find("\n", candidate.end())
pos = len(text) if newline < 0 else newline + 1
def _strip_plain_ui_open_panel(text: str) -> str:
matches = list(_iter_plain_ui_open_panel(text))
if not matches:
return text
out = []
pos = 0
for match in matches:
out.append(text[pos:match.start()])
pos = match.end()
out.append(text[pos:])
return "".join(out)
# ---------------------------------------------------------------------------
@@ -909,10 +858,7 @@ def _raw_openai_tool_call_to_block(value) -> Optional[ToolBlock]:
elif tool_type in ("grep", "glob", "ls", "edit_file"):
content = json.dumps(args) if args else "{}"
elif tool_type == "write_file":
# Keep raw OpenAI JSON on the canonical path so explicit empty content
# remains distinguishable from a missing body.
from src.tool_schemas import function_call_to_tool_block
return function_call_to_tool_block(name, json.dumps(args))
content = args.get("path", "") + "\n" + args.get("content", "")
elif tool_type == "create_document":
parts = [args.get("title", "Untitled")]
if args.get("language"):
@@ -1043,43 +989,6 @@ def _strip_raw_openai_tool_call_json(text: str) -> str:
return "".join(pieces)
def iter_email_addresses(text: str, *, ascii_only: bool = False):
"""Find legacy email-shaped strings without retrying every word suffix.
Match the address only at the start of each maximal local-part token.
When that attempt fails, every suffix has the same '@'/domain boundary
and must fail too. Local/domain character runs are each scanned a bounded
number of times, including malformed text with no '@' or domain dot.
"""
local_re = _ASCII_EMAIL_LOCAL_RE if ascii_only else _EMAIL_LOCAL_RE
address_re = _ASCII_EMAIL_ADDRESS_RE if ascii_only else _EMAIL_ADDRESS_RE
pos = 0
while token := local_re.search(text, pos):
address = address_re.match(text, token.start())
if address is not None:
yield address.group(0)
pos = address.end()
else:
pos = token.end()
def _iter_qwen_python_args(raw_args: str):
"""Match each maximal key once, then its value at that fixed position.
If a word has no following equals sign, none of its suffixes can have one
either. Advancing past it avoids the old unanchored regex's O(n^2) retries
on a long malformed key, while preserving permissive quoted/bare values.
"""
pos = 0
while key := _QWEN_PYTHON_ARG_KEY_RE.search(raw_args, pos):
value = _QWEN_PYTHON_ARG_VALUE_RE.match(raw_args, key.end())
if value is not None:
yield key.group(0), value.group(1)
pos = value.end()
else:
pos = key.end()
def _parse_qwen3_native_text_call(
text: str,
additional_tool_names: Optional[Iterable[str]] = None,
@@ -1106,14 +1015,13 @@ def _parse_qwen3_native_text_call(
# Qwen native text rendering:
# <tool_call><function=manage_notes><parameter=action>list</parameter>...
fn_match = next(_iter_named_blocks(
text, _QWEN_FUNCTION_OPEN_RE, _QWEN_FUNCTION_CLOSE_RE
), None)
fn_match = re.search(r"<function=([A-Za-z_][\w:.-]*)>\s*([\s\S]*?)\s*</function>", text)
if fn_match:
name, raw_body = fn_match
name, raw_body = fn_match.groups()
args = {}
for key, raw_value in _iter_named_blocks(
raw_body.rstrip(), _QWEN_PARAMETER_OPEN_RE, _QWEN_PARAMETER_CLOSE_RE,
for key, raw_value in re.findall(
r"<parameter=([A-Za-z_]\w*)>\s*([\s\S]*?)\s*</parameter>",
raw_body,
):
value = raw_value.strip()
if value and value[0] in "[{\"":
@@ -1174,7 +1082,7 @@ def _parse_qwen3_native_text_call(
or normalized_name in declared_names
):
args = {}
for key, raw_value in _iter_qwen_python_args(raw_args):
for key, raw_value in re.findall(r"([A-Za-z_]\w*)\s*=\s*(['\"].*?['\"]|[^,]+)", raw_args):
try:
args[key] = ast.literal_eval(raw_value.strip())
except (ValueError, SyntaxError):
@@ -1515,45 +1423,6 @@ def _parse_tool_code_block(raw: str) -> Optional[ToolBlock]:
return ToolBlock(tool_name, content.strip())
return None
_GEMMA_FALLBACK_KEY_RE = re.compile(r"\w++")
_GEMMA_FALLBACK_COLON_RE = re.compile(r"\s*+:")
_GEMMA_FALLBACK_BOUNDARY_RE = re.compile(r"(?<!\s)(\s*+)(?:,\s*+\w++\s*+:|})")
def _iter_gemma_fallback_items(body):
"""Preserve permissive k:v recovery with one pass over keys/boundaries."""
boundaries = list(_GEMMA_FALLBACK_BOUNDARY_RE.finditer(body))
boundary_index = 0
pos = 0
newline = body.find("\n")
while key := _GEMMA_FALLBACK_KEY_RE.search(body, pos):
colon = _GEMMA_FALLBACK_COLON_RE.match(body, key.end())
if colon is None:
pos = key.end()
continue
start = colon.end()
while start < len(body) and body[start].isspace():
start += 1
if start < len(body) and body[start] in "\"'":
start += 1
while boundary_index < len(boundaries) and boundaries[boundary_index].end(1) < start:
boundary_index += 1
if boundary_index == len(boundaries):
return
boundary = boundaries[boundary_index]
end = max(start, boundary.start())
if 0 <= newline < start:
newline = body.find("\n", start)
if newline >= 0 and newline < end:
pos = key.end()
continue
capture_end = end
if capture_end > start and body[capture_end - 1] in "\"'":
capture_end -= 1
yield key.group(0), body[start:capture_end].strip()
pos = end
def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
"""Parse a Gemma-style call:tool_name{...} block into a ToolBlock."""
tool_name = tool_name.strip().lower().replace("-", "_")
@@ -1578,7 +1447,12 @@ def _parse_gemma_tool_call(tool_name: str, body: str) -> Optional[ToolBlock]:
if not isinstance(params, dict):
params = {}
except Exception:
params = dict(_iter_gemma_fallback_items(body))
# Simple regex key-value extraction fallback
params = {}
for m in re.finditer(r'(\w+)\s*:\s*["\']?(.*?)["\']?(?=\s*,\s*\w+\s*:|\s*\})', body):
k = m.group(1)
v = m.group(2).strip()
params[k] = v
from src.tool_schemas import function_call_to_tool_block
return function_call_to_tool_block(tool_name, json.dumps(params))
@@ -1632,9 +1506,9 @@ def _iter_delimited(text, open_re, close_re):
pos = cm.end()
def _strip_delimited(text: str, open_re, close_re, replacement: str = "") -> str:
"""Replace every ``open_re ... close_re`` span (forward-only; see
_iter_delimited). Equivalent to ``open_re([\\s\\S]*?)close_re`` substitution
def _strip_delimited(text: str, open_re, close_re) -> str:
"""Remove every ``open_re ... close_re`` span (forward-only; see
_iter_delimited). Equivalent to ``open_re([\\s\\S]*?)close_re`` ``re.sub('')``
for these delimiters, without the O(n^2) rescan on unclosed openers."""
spans = list(_iter_delimited(text, open_re, close_re))
if not spans:
@@ -1643,23 +1517,11 @@ def _strip_delimited(text: str, open_re, close_re, replacement: str = "") -> str
last = 0
for match_start, _inner_start, _inner_end, match_end in spans:
out.append(text[last:match_start])
out.append(replacement)
last = match_end
out.append(text[last:])
return "".join(out)
def strip_angle_tags(text: str, replacement: str = "", *, allow_empty: bool = False) -> str:
"""Linear equivalent of replacing ``<[^>]+>`` (or ``<[^>]*>``).
This preserves the existing flat text cleanup, including nested '<' and
malformed tails; it does not interpret HTML. Stop once no '>' is reachable
instead of retrying a suffix scan at every '<' in untrusted text.
"""
opener = _ANGLE_TAG_OPEN_RE if allow_empty else _NONEMPTY_ANGLE_TAG_OPEN_RE
return _strip_delimited(text, opener, _ANGLE_TAG_CLOSE_RE, replacement)
def _iter_named_blocks(text, open_re, close_re):
"""Forward-only equivalent of ``open_re([\\s\\S]*?)close_re`` finditer where
open_re captures a name in group 1: yield ``(name, body)``, pairing each
@@ -2093,10 +1955,10 @@ def parse_tool_blocks(
# Pattern 4b: Gemma-style <|tool_call|> blocks
if not blocks:
for tool_name, body in _iter_named_blocks(
text, _GEMMA_TOOL_CALL_OPEN_RE, _GEMMA_TOOL_CALL_CLOSE_RE
):
block = _parse_gemma_tool_call(tool_name, "{" + body + "}")
for m in _GEMMA_TOOL_CALL_RE.finditer(text):
tool_name = m.group(1)
body = m.group(2)
block = _parse_gemma_tool_call(tool_name, body)
if block:
blocks.append(block)
@@ -2133,7 +1995,7 @@ def parse_tool_blocks(
# from weaker native-tool models after reading the tool docs but failing to
# emit the actual structured call.
if not blocks:
m = next(_iter_plain_ui_open_panel(text), None)
m = _PLAIN_UI_OPEN_PANEL_RE.search(text)
if m:
blocks.append(ToolBlock("ui_control", f"open_panel {m.group(1).lower()}{m.group(2).lower()}".strip()))
@@ -2176,7 +2038,7 @@ def strip_tool_blocks(
cleaned = _strip_delimited(cleaned, _XML_TOOL_CALL_OPEN_RE, _XML_TOOL_CALL_CLOSE_RE)
cleaned = _XML_OPEN_TOOL_CALL_RE.sub('', cleaned)
cleaned = _strip_delimited(cleaned, _TOOL_CODE_OPEN_RE, _TOOL_CODE_CLOSE_RE)
cleaned = _strip_delimited(cleaned, _GEMMA_TOOL_CALL_OPEN_RE, _GEMMA_TOOL_CALL_CLOSE_RE)
cleaned = _GEMMA_TOOL_CALL_RE.sub('', cleaned)
cleaned = _strip_delimited(cleaned, _FUNCTION_MODEL_OPEN_RE, _FUNCTION_MODEL_CLOSE_RE)
cleaned = _strip_raw_openai_tool_call_json(cleaned)
declared_xml_calls = _parse_declared_direct_xml_calls(
@@ -2195,7 +2057,7 @@ def strip_tool_blocks(
if raw_web_json:
_, (start, end) = raw_web_json
cleaned = cleaned[:start] + cleaned[end:]
cleaned = _strip_plain_ui_open_panel(cleaned)
cleaned = _PLAIN_UI_OPEN_PANEL_RE.sub("", cleaned)
# Strip bare <invoke> blocks not wrapped in <tool_call>
cleaned = _strip_bare_invoke_markup(cleaned)
cleaned = re.sub(r'\n{3,}', '\n\n', cleaned)
src/tool_security.py
+2 -13
View File
@@ -127,6 +127,7 @@ PLAN_MODE_READONLY_TOOLS = {
"resolve_contact",
"chat_with_model",
"ask_teacher",
"get_editor_screenshot",
}
@@ -163,6 +164,7 @@ _PLAN_MODE_KNOWN_MUTATORS = {
"download_model", "serve_model",
"stop_served_model", "cancel_download", "adopt_served_model", "serve_preset",
"generate_image", "edit_image", "trigger_research", "manage_research",
"plan_edit", "run_editor_command", "ai_edit_image",
# Shell is never read-only-safe; block it explicitly so it stays out of plan
# mode even if the schema list fails to load.
"bash", "python",
@@ -274,16 +276,3 @@ def blocked_tools_for_owner(owner: Optional[str]) -> Set[str]:
if owner_is_admin_or_single_user(owner):
return set()
return set(NON_ADMIN_BLOCKED_TOOLS)
def delegated_credential_blocked_tools() -> Set[str]:
"""Tools an agent run driven by a bearer API token must not reach.
Deliberately not owner-dependent. ``blocked_tools_for_owner`` asks whether
the OWNER is an admin, and for a token that question is always answered
yes: minting a token is an admin-only action, so the empty set comes back
for every token in existence. A token is a long-lived credential the owner
hands to a third party, so it is capped at the non-admin policy no matter
who minted it.
"""
return set(NON_ADMIN_BLOCKED_TOOLS)
src/turn_contract.py
+146 -269
View File
@@ -17,7 +17,6 @@ from typing import Iterable, Mapping
from src.action_intents import classify_tool_intent
from src.tool_policy import ToolPolicy
from src.text_scanning import has_prefixed_token_match
FAMILY_TOOLS = {
@@ -37,6 +36,7 @@ FAMILY_TOOLS = {
"sessions": frozenset({"list_sessions", "manage_session", "create_session", "send_to_session", "chat_with_model", "pipeline"}),
"image_generation": frozenset({"generate_image"}),
"image_editing": frozenset({"edit_image"}),
"photo_editor": frozenset({"plan_edit", "run_editor_command", "get_editor_screenshot", "ai_edit_image", "generate_image"}),
"transcription": frozenset({"transcribe_media"}),
"media_inspection": frozenset({"inspect_media"}),
"ocr": frozenset({"extract_text"}),
@@ -48,62 +48,6 @@ FAMILY_TOOLS = {
CONTRACT_CORE_TOOLS = frozenset({
"bash", "python", "read_file", "web_search", "web_fetch", "ask_user",
})
_WORKSPACE_PREFIX_RE = re.compile(r"/workspace/", re.I)
_WORKSPACE_ARTIFACT_RE = re.compile(
r"/workspace/[^\s`\"']+\.(?:csv|html?|json|md|svg|txt)\b", re.I
)
_WORKSPACE_OUTPUT_PREFIX_RE = re.compile(r"/workspace/(?!input/)", re.I)
_WORKSPACE_OUTPUT_RE = re.compile(
r"/workspace/(?!input/)[^\s`\"']+\."
r"(?:csv|html?|json|md|svg|txt|avif|bmp|gif|jpe?g|png|webp|pdf|mp4|webm)\b",
re.I,
)
_WORKSPACE_TOKEN_TAIL_RE = re.compile(r"[^\s`\"']*")
def _mentions_workspace_artifact(text: str) -> bool:
return has_prefixed_token_match(
str(text or ""),
_WORKSPACE_PREFIX_RE,
_WORKSPACE_ARTIFACT_RE,
_WORKSPACE_TOKEN_TAIL_RE,
)
def _mentions_workspace_output(text: str) -> bool:
return has_prefixed_token_match(
str(text or ""),
_WORKSPACE_OUTPUT_PREFIX_RE,
_WORKSPACE_OUTPUT_RE,
_WORKSPACE_TOKEN_TAIL_RE,
)
def _mentions_under_budget(text: str) -> bool:
value = str(text or "")
for under in re.finditer(r"\bunder", value, re.I):
cursor = under.end()
if cursor >= len(value) or not value[cursor].isspace():
continue
while cursor < len(value) and value[cursor].isspace():
cursor += 1
if cursor < len(value) and value[cursor] in "¥$€£":
cursor += 1
while cursor < len(value) and value[cursor].isspace():
cursor += 1
digit_start = cursor
while cursor < len(value) and value[cursor].isdecimal():
cursor += 1
if cursor == digit_start:
continue
if cursor == len(value) or not (value[cursor].isalnum() or value[cursor] == "_"):
return True
if value[cursor:cursor + 3].casefold() == "yen":
cursor += 3
if cursor == len(value) or not (value[cursor].isalnum() or value[cursor] == "_"):
return True
return False
_FAMILY_WORDS = {
"calendar": r"\b(?:calendar|calender|events?|appointments?|meetings?|agenda)\b",
"notes": r"\b(?:notes?|checklists?|groceries|remind\s+me)\b",
@@ -179,7 +123,13 @@ def _normalize_request_lead(value: str) -> str:
text,
flags=re.I,
)
text = _strip_cancelled_request_lead(text)
text = re.sub(
r"^(?:never\s*mind|scratch\s+that)\s*[,;:—–-]?\s*"
r"(?=(?:open|show|list|read|search|find|check|switch|go)\b)",
"",
text,
flags=re.I,
)
text = re.sub(r"^k(?:ay)?\s*[,!]?\s+(?=\S)", "", text, flags=re.I)
text = re.sub(
r"^(?:(?:great|nice|cool)\s*[,!.]|thanks?\s*[.!])\s+"
@@ -214,21 +164,6 @@ def _normalize_request_lead(value: str) -> str:
text = re.sub(r"^((?:can|could|would|will)\s+)u\b", r"\1you", text, flags=re.I)
text = re.sub(r"\boffical\b", "official", text, flags=re.I)
return text
def _strip_cancelled_request_lead(text: str) -> str:
lead = re.match(r"^(?:never\s*mind|scratch\s+that)", text, re.I)
if lead is None:
return text
cursor = lead.end()
while cursor < len(text) and text[cursor].isspace():
cursor += 1
if cursor < len(text) and text[cursor] in ",;:—–-":
cursor += 1
while cursor < len(text) and text[cursor].isspace():
cursor += 1
action = re.match(r"(?:open|show|list|read|search|find|check|switch|go)\b", text[cursor:], re.I)
return text[cursor:] if action is not None else text
_MISSPELLED_RESEARCH_ACTION = re.compile(
r"^\s*" + _REQUEST_PREFIX + r"(?:reserch|reasearch|reseach)\b",
re.I,
@@ -261,21 +196,6 @@ _PURE_ACTION_PROHIBITION = re.compile(
r"[^.;\n]*[.!?]*\s*$",
re.I,
)
def _is_pure_action_prohibition(text: str) -> bool:
value = str(text or "").strip()
prefix = re.match(
r"(?:read[- ]only(?:\s+and)?\s+)?(?:do\s+not|don['’]?t|never)\s+"
r"(?:add|create|make|write|draft|edit|change|update|delete|remove|send|reply|"
r"run|execute|download|serve|open|save|schedule|transcribe|inspect)\b",
value,
re.I,
)
if prefix is None:
return False
tail = value[prefix.end():].rstrip(".!?")
return not any(char in ".;\n" for char in tail)
_RETURN_TO_ACTION = re.compile(r"^\s*" + _REQUEST_PREFIX + r"return\s+to\b", re.I)
_PANEL_NAVIGATION = re.compile(
r"^\s*" + _REQUEST_PREFIX
@@ -487,7 +407,8 @@ _BOUND_EDITOR_WRITE = re.compile(
re.I,
)
_BOUND_EDITOR_IMPLICIT_REVISION = re.compile(
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|"
r"^\s*" + _REQUEST_PREFIX + r"(?:broaden|expand|deepen|lighten|go\s+deeper|go\s+more\s+into|"
r"g[aå]\s+mer\s+(?:in|ut)\s+p[aå]|"
r"clean\s+(?:this|it|the\s+(?:text|draft|document|doc))\s+up|"
r"give\s+(?:me\s+)?(?:feedback|a\s+critique|suggestions?|comments?)|"
r"remove\b[^.;\n]{0,100}\b(?:mistakes?|errors?|inaccurac(?:y|ies)|misinformation)|"
@@ -528,111 +449,11 @@ _WARM_RECALL_WITH_FOLLOWUP = re.compile(
r"(?P<followup>(?:what(?:['’]?s|\s+is)?|which|who|where|when|how|show|open|read|list|find|search)\b[\s\S]{0,180})$",
re.I,
)
_WARM_TARGET_RE = re.compile(
r"calendar|emails?|inbox|notes?|tasks?|skills?|memories|memory|"
r"documents?|docs?|web|browser|cookbook|files?|shell",
re.I,
)
_WARM_FOLLOWUP_RE = re.compile(
r"(?:what(?:['’]?s|\s+is)?|which|who|where|when|how|show|open|read|list|find|search)"
r"\b[\s\S]{0,180}\Z",
re.I,
)
def _consume_space(value: str, cursor: int, *, required: bool = False) -> int | None:
start = cursor
while cursor < len(value) and value[cursor].isspace():
cursor += 1
return None if required and cursor == start else cursor
def _phrase_end(value: str, cursor: int, phrase: str) -> int | None:
for index, word in enumerate(phrase.split(" ")):
if value[cursor:cursor + len(word)].casefold() != word:
return None
cursor += len(word)
if index + 1 < len(phrase.split(" ")):
cursor = _consume_space(value, cursor, required=True)
if cursor is None:
return None
return cursor
def _warm_recall_parts(value: str, *, with_followup: bool = False) -> tuple[str, str] | None:
value = str(value or "")
cursor = _consume_space(value, 0) or 0
states = [cursor]
discourse = re.match(r"(?:ok(?:ay)?|and|then)", value[cursor:], re.I)
if discourse is not None:
after = _consume_space(value, cursor + discourse.end(), required=True)
if after is not None:
states.insert(0, after)
action_phrases = ("back to", "return to", "what about", "check", "show", "open")
action_states: list[int] = []
for state in states:
if not with_followup:
action_states.append(state)
for phrase in action_phrases:
end = _phrase_end(value, state, phrase)
if end is None:
continue
if with_followup:
end = _consume_space(value, end, required=True)
if end is None:
continue
action_states.append(end)
for state in dict.fromkeys(action_states):
state = _consume_space(value, state) or 0
possessive_states = [state]
for possessive in ("my", "the"):
if value[state:state + len(possessive)].casefold() == possessive:
possessive_states.insert(0, state + len(possessive))
for target_state in possessive_states:
target_state = _consume_space(value, target_state) or 0
target = _WARM_TARGET_RE.match(value, target_state)
if target is None:
continue
target_text = target.group(0)
suffix = target.end()
if with_followup:
if suffix < len(value) and (value[suffix].isalnum() or value[suffix] == "_"):
continue
separator_states = []
spaced = _consume_space(value, suffix) or 0
if spaced > suffix:
separator_states.append(spaced)
if spaced < len(value) and value[spaced] in "-—,:;":
separator_states.append(_consume_space(value, spaced + 1) or 0)
for conjunction in ("and", "then"):
end = spaced + len(conjunction)
if (value[spaced:end].casefold() == conjunction
and (end == len(value) or not (value[end].isalnum() or value[end] == "_"))):
separator_states.append(_consume_space(value, end) or 0)
for followup_start in dict.fromkeys(separator_states):
followup = _WARM_FOLLOWUP_RE.match(value, followup_start)
if followup is not None:
return target_text, followup.group(0)
continue
suffix = _consume_space(value, suffix) or 0
suffix_states = [suffix]
for word in ("again", "now"):
if value[suffix:suffix + len(word)].casefold() == word:
suffix_states.insert(0, suffix + len(word))
for end in suffix_states:
while end < len(value) and value[end] in ".!?":
end += 1
end = _consume_space(value, end) or 0
if end == len(value):
return target_text, ""
return None
_REQUIRED_TOOLS = {
"calendar": "manage_calendar", "notes": "manage_notes",
"tasks": "manage_tasks", "skills": "manage_skills",
"image_generation": "generate_image", "image_editing": "edit_image",
"photo_editor": "plan_edit",
"transcription": "transcribe_media", "media_inspection": "inspect_media", "ocr": "extract_text",
}
@@ -895,6 +716,60 @@ def scheduled_automation_request(message: str) -> bool:
))
_TASK_OBJECT = (
r'(?:tasks?|to-?dos?|to-?do\s+lists?|task\s+lists?|reminders?|automations?|'
r'(?:schedul\w*|recurring|scheduled)\s+jobs?)'
)
_TASK_MANAGEMENT_VERB = (
r'(?:add|create|make|set\s+up|schedule|new|start|track|turn|convert|save|store|'
r'delete|remove|complete|finish|close|pause|resume|restart|re-?run|run|edit|'
r'update|rename|archive|cancel|list|show|view|see|check|manage|assign|mark|'
r'look|peek|pull\s+up|open|find|keep)'
)
def explicit_task_management(message: str) -> bool:
"""True only when the user explicitly asks to manage background tasks.
A bare project noun ("30-day task + protocol doc") describes work to do, not
a request to create or read a scheduled task, so it must not select the
tasks family or offer manage_tasks.
"""
text = _normalize_request_lead(message)
if not text:
return False
if scheduled_automation_request(text):
return True
# Possessive / near-noun references: "my tasks", "our reminders", "my jobs".
if re.search(
r'\b(?:my|our|the|any|all)\b[^.!?\n]{0,24}?\b' + _TASK_OBJECT + r'\b',
text, re.I,
):
return True
# An explicit management verb near a task noun: "add a task", "delete my todo".
if re.search(
r'\b' + _TASK_MANAGEMENT_VERB + r'\b[^.!?\n]{0,40}?\b' + _TASK_OBJECT + r'\b',
text, re.I,
):
return True
# A task noun with an interrogative/state tail: "tasks do I have", "tasks titled".
if re.search(
r'\b' + _TASK_OBJECT + r'\b[^.!?\n]{0,32}?'
r'\b(?:do\s+i\s+have|have\s+i|are\s+(?:set\s+up|scheduled|active)|'
r'titled|named|called|set\s+up|scheduled|active|due|overdue)\b',
text, re.I,
):
return True
# A task noun tied to a list/tracker surface.
if re.search(
r'\b' + _TASK_OBJECT + r'\b[^.!?\n]{0,24}?\b'
r'(?:list|tracker|manager|dashboard|schedule|inbox|library|board)\b',
text, re.I,
):
return True
return False
def creation_container_tool(message: str) -> str | None:
"""The explicitly created container owns its content, not vice versa."""
text = _normalize_request_lead(message)
@@ -1033,7 +908,11 @@ def selected_tools_for_request(message: str) -> frozenset[str] | None:
tools = {"inspect_media"}
if (
re.search(r"\b(?:create|write|save|build|produce)\b", raw_text, re.I)
and _mentions_workspace_artifact(raw_text)
and re.search(
r"(?:file://)?/workspace/[^\s`\"']+\.(?:csv|html?|json|md|svg|txt)\b",
raw_text,
re.I,
)
):
tools.update({"write_file", "read_file"})
if re.search(r"\b(?:preview|render|open)\b[^.\n]{0,100}\b(?:page|html|browser)\b", raw_text, re.I):
@@ -2296,34 +2175,6 @@ _READ_PRESENTATION_SUFFIX = re.compile(
)
def _terminal_clause_match(text: str, core_pattern: str) -> re.Match[str] | None:
"""Match a terminal clause after removing its ambiguous punctuation tail."""
end = len(text)
while end and text[end - 1].isspace():
end -= 1
while end and text[end - 1] in ".!?":
end -= 1
possessive = core_pattern.replace(r"\s+", r"\s++").replace(r"\s*", r"\s*+")
# An optional leading " and" branch must start at a whitespace boundary;
# otherwise search retries the whole possessive run at every suffix.
possessive = possessive.replace(r"|\s++and", r"|(?<!\s)\s++and")
return re.search(possessive + r"$", text[:end], re.I)
def _strip_terminal_but(text: str) -> str:
end = len(text)
while end and text[end - 1].isspace():
end -= 1
if end < 3 or text[end - 3:end].casefold() != "but":
return text
start = end - 3
if start == 0 or not text[start - 1].isspace():
return text
while start and text[start - 1].isspace():
start -= 1
return text[:start]
def _read_request_and_limit(message: str) -> tuple[str, int | None]:
"""Strip only whole, known presentation/safety suffixes, never actions."""
text = _normalize_request_lead(message)
@@ -2373,11 +2224,11 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
if keep_few_suffix:
maximum = 3
text = text[:keep_few_suffix.start()].strip()
few_suffix = _terminal_clause_match(
text,
few_suffix = re.search(
r"[,.;?]\s*(?:(?:only|just)\s+)?(?:(?:list|show)\s+(?:me\s+)?)?a\s+few"
r"(?:\s+(?:task\s+)?(?:names?|items?|results?|entries?))?"
r"(?:\s+and\s+(?:whether|if)\s+[^.;\n]+)?",
r"(?:\s+and\s+(?:whether|if)\s+[^.;\n]+)?[.!?]*\s*$",
text, re.I,
)
if few_suffix:
maximum = 3
@@ -2389,43 +2240,42 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
text,
flags=re.I,
).strip()
terminal_read_only = _terminal_clause_match(
text,
text = re.sub(
r"[.;]\s*read[- ]only(?:\s+(?:please|pls|plz))?\s*,?\s*"
r"(?:(?:and\s+)?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:change|edit|modify)(?:\s+or\s+send)?\s+(?:anything|data))?",
)
if terminal_read_only:
text = text[:terminal_read_only.start()].strip()
# Explanatory/safety tails do not alter a preceding exact read request.
safety_tail = _terminal_clause_match(
r"(?:change|edit|modify)(?:\s+or\s+send)?\s+(?:anything|data))?"
r"[.!?]*\s*$",
"",
text,
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:touch|change|edit|modify)(?:\s+(?:anything|data|them))?(?:\s+yet)?",
)
if safety_tail:
text = text[:safety_tail.start()].strip()
flags=re.I,
).strip()
# Explanatory/safety tails do not alter a preceding exact read request.
text = re.sub(
r"[.!?]\s*+read[- ]only(?:\s++(?:please|pls|plz))?\s*+(?:,\s*+)?"
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?(?:do\s+not|don['’]?t|dont)\s+"
r"(?:touch|change|edit|modify)(?:\s+(?:anything|data|them))?(?:\s+yet)?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[.!?]\s*read[- ]only(?:\s+(?:please|pls|plz))?\s*,?\s*"
r"(?:do\s+not|don['’]?t|dont)\s+(?:change|edit|modify)\s+"
r"(?:or\s+send\s+)?anything[.!?]*\s*$",
"", text, flags=re.I,
).strip()
for terminal_core in (
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+changes?",
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+edits?",
):
terminal_match = _terminal_clause_match(text, terminal_core)
if terminal_match:
text = text[:terminal_match.start()].strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+changes?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+edits?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[,;]\s*no\s+edits?[.!?]*\s*$", "", text, flags=re.I,
).strip()
terminal_no_write = _terminal_clause_match(
text, r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+writes?"
)
if terminal_no_write:
text = text[:terminal_no_write.start()].strip()
text = re.sub(
r"(?:(?:[,;]\s*(?:and\s+)?|\s+and\s+))?no\s+writes?[.!?]*\s*$",
"", text, flags=re.I,
).strip()
text = re.sub(
r"[.!?]\s*(?:i['’]?m|i\s+am)\s+(?:just\s+)?checking\b[^\n]*$",
"", text, flags=re.I,
@@ -2443,11 +2293,12 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
text,
flags=re.I,
).strip()
short_lines = _terminal_clause_match(
text, r"[,;]\s*(?:keep\s+(?:them|it)\s+)?short\s+lines?\s*,?"
)
if short_lines:
text = text[:short_lines.start()].strip()
text = re.sub(
r"[,;]\s*(?:keep\s+(?:them|it)\s+)?short\s+lines?\s*,?[.!?]*\s*$",
"",
text,
flags=re.I,
).strip()
text = re.sub(
r"[,.;]\s*keep\s+(?:the\s+answer|it|them)\s+short[.!?]*\s*$",
"",
@@ -2472,11 +2323,11 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
maximum = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
text = text[:approximate_limit.start()].strip()
natural_limit = re.search(
r"(?:[,.;?]|[—–-]|(?<!\s)\s++but\s++|(?<!\s)\s++)\s*+(?:"
r"(?:i\s++)?only\s++(?:need|want|show(?:\s++me)?)?\s*+(?:(?:the\s++)?first\s++)?"
r"|just\s++(?:(?:the\s++)?first\s++)?|(?:show\s++me\s++)?like\s++|no\s++more\s++than\s++"
r"|cap(?:\s++(?:it|them|the\s++(?:answer|list)))?\s++at\s++"
r"|(?:maybe\s++)?(?:first|same)\s++)"
r"(?:[,.;?]|[—–-]|\s+but\s+|\s+)\s*(?:"
r"(?:i\s+)?only\s+(?:need|want|show(?:\s+me)?)?\s*(?:(?:the\s+)?first\s+)?"
r"|just\s+(?:(?:the\s+)?first\s+)?|(?:show\s+me\s+)?like\s+|no\s+more\s+than\s+"
r"|cap(?:\s+(?:it|them|the\s+(?:answer|list)))?\s+at\s+"
r"|(?:maybe\s+)?(?:first|same)\s+)"
r"(" + _READ_COUNT + r")"
r"(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?|bits?|things?))?"
r"(?:\s*(?:and|\+)\s+(?:their\s+)?(?:status(?:es)?|states?))?"
@@ -2514,7 +2365,7 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
maximum = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
text = text[:compact_limit.start()].strip()
conversational_limit = re.search(
r"(?:[,.;?]\s*|(?<!\s)\s++)(?:maybe\s+)?(?:keep\s+it\s+to\s+|stick\s+to\s+|(?:first|top)\s+)"
r"(?:[,.;?]\s*|\s+)(?:maybe\s+)?(?:keep\s+it\s+to\s+|stick\s+to\s+|(?:first|top)\s+)"
r"(" + _READ_COUNT + r")(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?))?"
r"[.!?]*\s*$",
text, re.I,
@@ -2524,7 +2375,7 @@ def _read_request_and_limit(message: str) -> tuple[str, int | None]:
value = int(raw) if raw.isdecimal() else _READ_COUNT_WORDS[raw.lower()]
maximum = value if maximum is None else min(maximum, value)
text = text[:conversational_limit.start()].rstrip(' ,.;?')
text = _strip_terminal_but(text)
text = re.sub(r"\s+but\s*$", "", text, flags=re.I)
need_limit = re.search(
r"[.!?]\s*(?:i\s+)?only\s+need\s+(" + _READ_COUNT + r")"
r"(?:\s+(?:short\s+)?(?:titles?|items?|results?|entries?|names?|ones?))?"
@@ -4149,13 +4000,21 @@ def _families_for_tool(tool: str) -> frozenset[str]:
if bare in {"list_sessions", "manage_session", "create_session", "send_to_session",
"chat_with_model", "pipeline"}:
return frozenset({"sessions"})
return frozenset(family for family, tools in FAMILY_TOOLS.items() if bare in tools)
# The photo_editor family is context-only: it is only ever selected by an
# explicit editor request at the route, never inferred from message text.
# Overlapping tools such as generate_image must still resolve to their real
# product family, so exclude photo_editor rather than short-circuiting.
return frozenset(
family
for family, tools in FAMILY_TOOLS.items()
if bare in tools and family != "photo_editor"
)
def _clause_capabilities(text: str) -> set[str]:
# A prohibition constrains authority; it must never grant the family named
# only as the forbidden side effect (for example, "do not create a file").
if _is_pure_action_prohibition(text):
if _PURE_ACTION_PROHIBITION.fullmatch(text):
return set()
container_tool = creation_container_tool(text)
if container_tool:
@@ -4299,6 +4158,11 @@ def _clause_capabilities(text: str) -> set[str]:
if intent.reason == "terse calendar follow-up action":
return set()
words = {f for f, pattern in _FAMILY_WORDS.items() if re.search(pattern, text, re.I)}
if "tasks" in words and not explicit_task_management(text):
# A bare mention of "task"/"todo"/"automation" in a project description
# is content, not a request to create or read a scheduled task. Only an
# explicit task-management request may select the tasks family.
words.discard("tasks")
personal_stores = words & {
"calendar", "notes", "tasks", "skills", "memory", "documents", "email",
}
@@ -4406,6 +4270,14 @@ def _clause_capabilities(text: str) -> set[str]:
mapped = "tasks"
if mapped in FAMILY_TOOLS:
return {mapped}
if "shell_files" in words and re.search(
r"\b(?:bash|shell|terminal|zsh|command\s+line|cmd)\b", text, re.I,
):
# An explicitly named shell is a request to use the shell even when the
# rest of the message is a bare or typo'd command ("Bash commabd Ls la").
# Grant the shell family so the always-offered bash tool is not denied
# by request authority (offer/admission alignment).
return {"shell_files"}
# Families absent from action_intents still need a generic action gate;
# merely discussing a domain must not offer its mutation tools.
return words if _has_action_signal(text) or personal_lookup else set()
@@ -4816,7 +4688,12 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
raw_text,
re.I,
)
and _mentions_workspace_output(raw_text)
and re.search(
r"(?:file://)?/workspace/(?!input/)[^\s`\"']+\."
r"(?:csv|html?|json|md|svg|txt|avif|bmp|gif|jpe?g|png|webp|pdf|mp4|webm)\b",
raw_text,
re.I,
)
):
families.add("shell_files")
if re.search(
@@ -5275,7 +5152,7 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
and re.search(r"\b(?:ones?|top|apps?|services?|providers?)\b", text, re.I)
and re.search(r"\b(?:price|cheap|under|dimensions?|quote|trustworthy|app)\b", text, re.I)
)
or _mentions_under_budget(text)
or re.search(r"\bunder\s+[¥$€£]?\s*\d+(?:[.,]\d+)?(?:\s*yen)?\b", text, re.I)
):
return frozenset({"search_browser"})
if recent_family == ("search_browser",) and re.fullmatch(
@@ -6212,8 +6089,8 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
elif (recent == ("search_browser",) and families == {"cookbook_admin"}
and re.match(r"^\s*" + _REQUEST_PREFIX + r"(?:search|find)\s+(?:those|them|these)\b", text, re.I)):
families = {"search_browser"}
if not families and (recall := _warm_recall_parts(text)):
target = recall[0].lower()
if not families and (recall := _WARM_RECALL.fullmatch(text)):
target = recall["target"].lower()
family = {
"email": "email", "emails": "email", "inbox": "email",
"note": "notes", "notes": "notes", "task": "tasks", "tasks": "tasks",
@@ -6225,8 +6102,8 @@ def requested_capabilities(message: str, history: Iterable = (), *, active_docum
}[target]
if family in recently_executed_families(history):
families.add(family)
if not families and (recall := _warm_recall_parts(text, with_followup=True)):
target = recall[0].lower()
if not families and (recall := _WARM_RECALL_WITH_FOLLOWUP.fullmatch(text)):
target = recall["target"].lower()
family = {
"email": "email", "emails": "email", "inbox": "email",
"note": "notes", "notes": "notes", "task": "tasks", "tasks": "tasks",
@@ -6325,7 +6202,7 @@ def resolve_full_inventory_contract(*, schemas: Iterable[dict], policy: ToolPoli
"""Experimental trained inventory: permissions filter offers; model chooses actions."""
families = frozenset({"calendar", "notes", "tasks", "skills", "memory", "documents",
"email", "search_browser", "shell_files", "cookbook_admin",
"image_editing", "image_generation"})
"image_editing", "image_generation", "photo_editor"})
# ``ui_control`` is the executable bridge for explicit client-interface
# requests (for example, opening the gallery). It is not one of the ten
# persisted-data families, but omitting it here makes the full-inventory
src/url_safety.py
+2 -55
View File
@@ -12,7 +12,7 @@ break the primary use case. What it *always* rejects:
- a non-HTTP(S) scheme (``file://``, ``gopher://``, ``ftp://`` …), and
- the link-local range (``169.254.0.0/16`` / ``fe80::/10``), i.e. the cloud
instance-metadata SSRF credential-exfil vector — nobody serves embeddings
there, plus multicast / non-loopback reserved / unspecified addresses.
there — plus multicast / reserved / unspecified addresses.
For exposed multi-tenant deployments, set ``EMBEDDING_BLOCK_PRIVATE_IPS=true`` to
additionally reject all private and loopback targets (full SSRF lockdown).
@@ -94,15 +94,11 @@ def _classify(ip: ipaddress._BaseAddress, *, block_private: bool) -> Optional[st
return None
if ip.is_link_local:
return f"link-local address blocked (SSRF metadata risk): {ip}"
# IPv6 loopback is also reserved; its policy must match IPv4 loopback.
if ip.is_loopback:
if block_private:
return f"private/shared/loopback address blocked: {ip}"
return None
if ip.is_multicast or ip.is_reserved or ip.is_unspecified:
return f"disallowed address: {ip}"
if block_private and (
ip.is_private
or ip.is_loopback
or (isinstance(ip, ipaddress.IPv4Address) and ip in _SHARED_ADDRESS_SPACE_V4)
):
return f"private/shared/loopback address blocked: {ip}"
@@ -158,52 +154,3 @@ def check_outbound_url(
if not saw_ip:
return False, "host does not resolve to an IP"
return True, "ok"
class OutboundAddressBlocked(PermissionError):
"""A non-HTTP outbound host resolves into a disallowed address range."""
def connect_outbound_tcp(
host: str,
port: int,
*,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
block_private: bool = False,
source_address=None,
resolver: Optional[Callable[..., list]] = None,
) -> socket.socket:
"""Open a TCP connection to *host* under the outbound address policy.
For raw TCP clients (IMAP/SMTP). The host is resolved exactly once, every
resolved address is judged with the same policy as check_outbound_url, and
the socket connects only to those already-judged addresses. A second
lookup at connect time (what socket.create_connection(host) does) would let
a rebinding name pass the check with a public answer and then connect to
metadata/private space. TLS callers keep wrapping the returned socket with
``server_hostname=host``, so SNI and certificate checks are unchanged.
"""
resolve = resolver or socket.getaddrinfo
infos = resolve(host, port, 0, socket.SOCK_STREAM)
for _family, _type, _proto, _canon, sockaddr in infos:
ip = ipaddress.ip_address(str(sockaddr[0]).split("%")[0])
reason = _classify(ip, block_private=block_private)
if reason:
raise OutboundAddressBlocked(reason)
last_error: Optional[OSError] = None
for family, socktype, proto, _canon, sockaddr in infos:
sock = socket.socket(family, socktype, proto)
try:
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: # same contract as create_connection
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
sock.connect(sockaddr)
return sock
except OSError as exc:
last_error = exc
sock.close()
if last_error is not None:
raise last_error
raise socket.gaierror(socket.EAI_NONAME, f"{host} did not resolve to an address")
tests/test_agent_rounds_exhausted.py
+1 -2
View File
@@ -1873,8 +1873,7 @@ def test_tui_local_skill_request_reconciles_normal_tool_policy(monkeypatch):
},
)))
# Caller hard denials are absolute and cannot be re-enabled by later TUI turn contracts.
assert calls == []
assert calls == ["manage_skills"]
assert not any(e.get("type") == "loop_breaker_triggered" for e in events)
tests/test_aux_llm_owner_scope.py
+2 -3
View File
@@ -49,9 +49,8 @@ def test_scheduler_fallbacks_and_research_headers_are_owner_scoped():
assert "owner=task.owner or None" in src
assert "headers_from_resolver = False" in src
assert "headers_from_resolver = True" in src
assert "resolve_owner_registered_endpoint(db2, endpoint_url, task.owner or None)" in src
assert "resolve_owner_registered_endpoint(db, endpoint_url, task.owner or None)" in src
assert src.count("resolve_endpoint_runtime(ep, owner=task.owner or None)") == 2
assert "from src.auth_helpers import owner_filter" in src
assert "owner_filter(ep_q, ModelEndpoint, task.owner or None)" in src
def test_research_routes_fallbacks_are_owner_scoped():
tests/test_calendar_reminder_minutes_parsing.py
+12 -34
View File
@@ -49,6 +49,9 @@ async def _create_with_reminder(reminder, owner):
@pytest.mark.parametrize("reminder,expected", [
(0, 0), # At-event reminder must not compare equal to False.
(0.0, 0),
("0", 0),
("5 mins", 5),
("10 mins", 10),
("2 hrs", 120),
@@ -91,6 +94,15 @@ async def test_no_reminder_when_offset_absent():
assert "reminder set" not in res.get("response", ""), res
@pytest.mark.parametrize("reminder", [False, "off", "false", None])
async def test_disabled_reminder_is_distinct_from_zero(reminder):
owner = "tester-" + uuid.uuid4().hex[:6]
res = await _create_with_reminder(reminder, owner)
assert res.get("exit_code") == 0, res
assert not res.get("has_reminder"), res
assert res.get("reminder_note_id") is None, res
async def test_update_event_can_add_reminder_after_creation():
owner = "tester-" + uuid.uuid4().hex[:6]
from src.tool_implementations import do_manage_calendar
@@ -123,37 +135,3 @@ async def test_update_event_can_add_reminder_after_creation():
assert note is not None, "update_event should create the reminder note"
finally:
db.close()
async def test_reminder_and_duration_parsers_stay_linear_on_digit_and_space_floods():
"""CodeQL py/polynomial-redos: `(\\d+)\\s*unit` rescanned a digit run from
every offset and `alarm\\s*:?\\s*\\d+` split one whitespace run two ways.
Tool arguments come from model output, so keep them O(n)."""
import time
from src.tool_implementations import do_manage_calendar
owner = "tester-" + uuid.uuid4().hex[:6]
started = time.perf_counter()
res = await do_manage_calendar(json.dumps({
"action": "create_event",
"summary": "Flood",
"dtstart": "2030-04-01T10:00:00",
"reminder_minutes": "0" * 40_000 + "x",
"duration": "0" * 40_000 + "x",
}), owner=owner)
assert time.perf_counter() - started < 2.0
assert res.get("exit_code") == 0, res
assert "reminder set" not in res.get("response", ""), res
# A reminder is set, so the "is the description only a reminder?" check runs.
started = time.perf_counter()
res = await do_manage_calendar(json.dumps({
"action": "create_event",
"summary": "Flood 2",
"dtstart": "2030-04-02T10:00:00",
"description": "alarm" + "\t" * 40_000,
"reminder_minutes": 5,
}), owner=owner)
assert time.perf_counter() - started < 2.0
assert res.get("exit_code") == 0, res
assert "reminder 5 min before" in res.get("response", ""), res
tests/test_client_tool_routing.py
+11 -12
View File
@@ -624,8 +624,7 @@ def test_bridge_post_targets_endpoint_paths():
posted = {}
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
posted["timeout"] = timeout
async def __aenter__(self):
@@ -696,8 +695,8 @@ def test_bridge_post_treats_empty_or_error_payload_as_failure():
payloads = iter(({}, {"error": "token rejected"}))
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
pass
async def __aenter__(self):
return self
@@ -735,8 +734,8 @@ def test_bridge_post_http_or_payload_error_overrides_false_success_code():
))
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
pass
async def __aenter__(self):
return self
@@ -770,8 +769,8 @@ def test_bridge_post_rejects_malformed_exit_code():
return {"output": "looks successful", "exit_code": "zero"}
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
pass
async def __aenter__(self):
return self
@@ -805,8 +804,8 @@ def test_bridge_post_reports_empty_invalid_json_response():
raise ValueError("not JSON")
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
pass
async def __aenter__(self):
return self
@@ -834,8 +833,8 @@ def test_bridge_post_reports_empty_invalid_json_response():
def test_bridge_post_cancellation_schedules_host_operation_cancel():
class FakeClient:
def __init__(self, timeout=None, *, trust_env=True):
assert trust_env is False
def __init__(self, timeout=None):
pass
async def __aenter__(self):
return self
tests/test_code_nav_tools.py
+1 -21
View File
@@ -61,20 +61,11 @@ def test_grep_ignore_case(repo):
assert "b.txt:2:" in r["output"]
def test_grep_glob_filter(repo, monkeypatch):
def test_grep_glob_filter(repo):
r = _run("grep", f'{{"pattern": "needle", "ignore_case": true, "glob": "*.py", "path": "{repo}"}}')
assert r["exit_code"] == 0
assert "a.py" in r["output"]
assert "b.txt" not in r["output"]
# Positive control: verify that fallback path honors the exact same glob contract
real_which = shutil.which
monkeypatch.setattr(shutil, "which", lambda name, *a, **kw: None if name == "rg" else real_which(name, *a, **kw))
r_fallback = _run("grep", f'{{"pattern": "needle", "ignore_case": true, "glob": "*.py", "path": "{repo}"}}')
assert r_fallback["exit_code"] == 0
assert "a.py" in r_fallback["output"]
assert "b.txt" not in r_fallback["output"]
def test_grep_no_match(repo):
r = _run("grep", f'{{"pattern": "zzzznotfound", "path": "{repo}"}}')
@@ -103,17 +94,6 @@ def test_grep_python_fallback_when_no_rg(repo, monkeypatch):
assert ".git/config" not in r["output"]
def test_grep_python_fallback_uses_relative_glob_paths(repo, monkeypatch):
monkeypatch.setattr(shutil, "which", lambda name: None)
r = _run(
"grep",
f'{{"pattern": "needle|python", "glob": "**/*.py", "path": "{repo}"}}',
)
assert r["exit_code"] == 0
assert "a.py" in r["output"]
assert "sub/deep/c.py" in r["output"]
@pytest.mark.skipif(shutil.which("rg") is None, reason="targets the ripgrep fast-path")
def test_grep_skips_case_variant_sensitive_files_rg(repo):
"""The rg fast-path must exclude deny-listed key files case-insensitively.
tests/test_effect_resource_bindings.py
+1 -9
View File
@@ -179,16 +179,8 @@ def test_cancelled_write_unsettles_an_earlier_success(run, ws, monkeypatch):
def test_mid_write_failure_unsettles_but_refusal_preserves(run, ws, monkeypatch):
run("write_file", {"path": "a.txt", "content": "hello\n"})
# Explicit empty JSON clear/write is authoritative and succeeds.
run("write_file", {"path": "a.txt", "content": ""})
assert run.journal.effects.assessments()[-1].execution is fx.ExecutionOutcome.REPORTED_SUCCESS
# Re-seed the artifact content
run("write_file", {"path": "a.txt", "content": "hello\n"})
assert ledger(run.journal, ws).evaluate().can_complete
# A deterministic refusal before the mutation stage keeps the artifact.
run("write_file", "a.txt\n")
run("write_file", {"path": "a.txt", "content": ""})
assert run.journal.effects.assessments()[-1].execution is fx.ExecutionOutcome.FAILED
assert ledger(run.journal, ws).evaluate().can_complete
tests/test_env_reference.py
+1 -47
View File
@@ -167,56 +167,10 @@ def test_renders_one_table_row_per_variable(generator, fake_tree, monkeypatch):
page = generator.render(found, generator.naive_line_scan(fake_tree))
assert page.startswith("---\nlayout: default\n---\n")
assert "| `ODYSSEUS_VIA_HELPER` | `5 * 1024` | `src/indirect.py` | A synthetic limit. |" in page
assert "| `ODYSSEUS_VIA_HELPER` | `5 * 1024` | `src/indirect.py:16` | A synthetic limit. |" in page
assert "### Search" in page
def _render_one(generator, root, name, monkeypatch):
monkeypatch.setitem(
generator.VARIABLE_NOTES, name, ("Search", generator.USER, "A synthetic flag.")
)
return generator.render(
{name: generator.collect(root)[name]}, generator.naive_line_scan(root)
)
def test_page_does_not_change_when_an_unrelated_edit_shifts_lines(
generator, fake_tree, monkeypatch
):
"""An edit that moves a read down a file must not make the page stale.
The page used to cite `path:line`, so any PR adding a line above a read had
to regenerate it and then conflicted with every other PR that had.
"""
before = _render_one(generator, fake_tree, "ODYSSEUS_VIA_HELPER", monkeypatch)
source = fake_tree / "src" / "indirect.py"
source.write_text("import sys\n\n\n" + source.read_text(encoding="utf-8"), encoding="utf-8")
after = _render_one(generator, fake_tree, "ODYSSEUS_VIA_HELPER", monkeypatch)
assert generator.collect(fake_tree)["ODYSSEUS_VIA_HELPER"].primary.lineno == 19
assert after == before
def test_extra_reads_are_counted_per_file(generator, tmp_path, monkeypatch):
monkeypatch.setattr(generator, "SOURCE_ROOTS", ("app.py", "src"))
_write(tmp_path, "app.py", """
import os
FIRST = os.environ.get("ODYSSEUS_READ_OFTEN", "1")
SECOND = os.environ.get("ODYSSEUS_READ_OFTEN", "1")
""")
_write(tmp_path, "src/elsewhere.py", """
import os
AGAIN = os.getenv("ODYSSEUS_READ_OFTEN")
""")
page = _render_one(generator, tmp_path, "ODYSSEUS_READ_OFTEN", monkeypatch)
assert "| `app.py` (+1 more) |" in page
def test_every_variable_read_in_the_repository_is_documented(built):
_, variables, problems = built
tests/test_execution_bridge.py
+7 -68
View File
@@ -1,6 +1,4 @@
import asyncio
import json
import pytest
import logging
import src.tool_execution as tool_execution
from tests.runtime_evidence_helpers import server_authorized_executor
@@ -17,60 +15,6 @@ from src.tool_execution import (
execute_tool_block = server_authorized_executor(execute_tool_block)
@pytest.mark.parametrize("content", ['{"path":"/workspace/x"}', '{"path":"/workspace/x","content":null}',
'/workspace/x', '/workspace/x\n', '/workspace/x\n \t',
json.dumps({'path': '/workspace/x.py', 'content': '```python\n\n```'})])
def test_bridge_refuses_missing_or_ambiguous_empty_content(monkeypatch, tmp_path, content):
async def forbidden(*args, **kwargs):
raise AssertionError("ambiguous write reached remote producer")
monkeypatch.setattr(tool_execution, "_client_bridge", lambda context: {"url": "http://bridge", "token": "x"})
monkeypatch.setattr(tool_execution, "_bridge_post", forbidden)
async def invoke():
token = tool_execution._active_workspace.set(str(tmp_path))
try:
return await tool_execution._route_tool_via_bridge("write_file", content, "s", {})
finally:
tool_execution._active_workspace.reset(token)
_, result = asyncio.run(invoke())
assert result["exit_code"] == 1
assert "content" in result["error"] or "JSON" in result["error"]
@pytest.mark.parametrize("body", ["", " \t", "normal source"])
def test_bridge_honors_explicit_json_string_content(monkeypatch, tmp_path, body):
import base64
seen = []
async def post(bridge, endpoint, payload, **kwargs):
seen.append((endpoint, base64.b64decode(payload["content_b64"])))
return {"output": "written", "exit_code": 0}
monkeypatch.setattr(tool_execution, "_client_bridge", lambda context: {"url": "http://bridge", "token": "x"})
monkeypatch.setattr(tool_execution, "_bridge_post", post)
async def invoke():
token = tool_execution._active_workspace.set(str(tmp_path))
try:
return await tool_execution._route_tool_via_bridge("write_file", json.dumps({"path": "/workspace/x", "content": body}), "s", {})
finally:
tool_execution._active_workspace.reset(token)
_, result = asyncio.run(invoke())
assert result["exit_code"] == 0
assert seen == [("/write", body.encode())]
@pytest.mark.parametrize("content", ['/workspace/x\n', '{"path":"/workspace/x"}', '/workspace/x\n \t'])
def test_scoped_bridge_cannot_blindly_write_implicit_empty(content):
async def forbidden(*args):
raise AssertionError("scoped bridge received an ambiguous destructive write")
bridge = AgentExecutionBridge(forbidden, frozenset({'write_file'}), 'empty-intent-test')
async def invoke():
with bind_execution_bridge(bridge):
block = Block(content)
block.tool_type = 'write_file'
return await execute_tool_block(block, security_context=NO_TOOL_SECURITY_CONTEXT)
_, result = asyncio.run(invoke())
assert result['exit_code'] == 1
assert 'content' in result['error']
class Block:
tool_type = "host_shell"
@@ -215,7 +159,7 @@ def test_scoped_execution_bridge_failure_logs_compact_warning(caplog) -> None:
assert "Scoped execution bridge test-environment failed for tool=host_shell" in caplog.text
def test_tui_bridge_write_file_rejects_text_for_binary_artifact(monkeypatch, tmp_path) -> None:
def test_tui_bridge_write_file_rejects_text_for_binary_artifact(monkeypatch) -> None:
"""The bridge path must preserve rendered media just like local writes."""
called = False
@@ -228,17 +172,12 @@ def test_tui_bridge_write_file_rejects_text_for_binary_artifact(monkeypatch, tmp
monkeypatch.setattr(tool_execution, "_bridge_post", fake_post)
async def invoke():
token = tool_execution._active_workspace.set(str(tmp_path))
try:
return await tool_execution._route_tool_via_bridge(
"write_file",
'{"path":"/tmp_workspace/results/screenshot.png","content":"The screenshot is complete."}',
"run-1",
{"surface": "odysseus-tui"},
)
finally:
tool_execution._active_workspace.reset(token)
return await tool_execution._route_tool_via_bridge(
"write_file",
'{"path":"/tmp_workspace/results/screenshot.png","content":"The screenshot is complete."}',
"run-1",
{"surface": "odysseus-tui"},
)
description, result = asyncio.run(invoke())
tests/test_security_regressions.py
+3 -5
View File
@@ -984,7 +984,7 @@ def test_attachment_extract_dir_stays_contained(folder, uid):
"""User-controlled folder/uid must never escape ATTACHMENTS_DIR — pins the
fix for the attachment-extraction path traversal."""
aed, base = _import_attachment_extract_dir()
target = aed(folder, uid, owner="../../owner", account_id="../acct")
target = aed(folder, uid)
base_r = base.resolve()
assert target == base_r or base_r in target.parents
# exactly one extra path segment, and no `..` component survived
@@ -992,11 +992,9 @@ def test_attachment_extract_dir_stays_contained(folder, uid):
assert ".." not in rel.parts
def test_attachment_extract_dir_is_stable_single_segment_per_scope():
def test_attachment_extract_dir_normal_inputs_unchanged():
aed, base = _import_attachment_extract_dir()
target = aed("INBOX", "123", owner="alice", account_id="acct-1")
assert target == aed("INBOX", "123", owner="alice", account_id="acct-1")
assert target.parent == base.resolve()
assert aed("INBOX", "123") == base.resolve() / "INBOX_123"
def test_diagnostics_routes_are_admin_gated():
tests/test_session_endpoint_owner_scope.py
+3 -52
View File
@@ -6,21 +6,13 @@ from fastapi import HTTPException
# Import the route helper during collection so sibling session tests that use
# partial import stubs do not become the first loader of core.session_manager.
from routes.session_routes import (
_reject_delegated_session_options,
_reject_raw_endpoint_url_for_non_admin,
)
from routes.session_routes import _reject_raw_endpoint_url_for_non_admin
def _request(user, *, admin=False, api_token=False, scopes=None):
def _request(user, *, admin=False):
auth_manager = SimpleNamespace(is_admin=lambda username: bool(admin))
return SimpleNamespace(
state=SimpleNamespace(
current_user="api" if api_token else user,
api_token=api_token,
api_token_owner=user if api_token else None,
api_token_scopes=scopes or [],
),
state=SimpleNamespace(current_user=user),
app=SimpleNamespace(state=SimpleNamespace(auth_manager=auth_manager)),
)
@@ -52,47 +44,6 @@ def test_admin_and_registered_endpoint_can_use_endpoint_url():
)
def test_bearer_token_does_not_inherit_owner_admin_raw_endpoint_authority():
request = _request("admin", admin=True, api_token=True, scopes=["chat"])
with pytest.raises(HTTPException) as exc:
_reject_raw_endpoint_url_for_non_admin(
request,
"admin",
"",
"http://127.0.0.1:8000/v1/chat/completions",
)
assert exc.value.status_code == 403
def test_chat_scoped_bearer_can_still_choose_an_owner_registered_endpoint():
_reject_raw_endpoint_url_for_non_admin(
_request("admin", admin=True, api_token=True, scopes=["chat"]),
"admin",
"owner-endpoint-id",
"http://127.0.0.1:8000/v1/chat/completions",
)
@pytest.mark.parametrize(
("skip_validation", "api_key"),
[(True, ""), (False, "caller-secret")],
)
def test_bearer_token_cannot_use_interactive_session_options(
skip_validation,
api_key,
):
with pytest.raises(HTTPException) as exc:
_reject_delegated_session_options(
_request("admin", admin=True, api_token=True, scopes=["chat"]),
skip_validation=skip_validation,
api_key=api_key,
)
assert exc.value.status_code == 403
def test_chat_endpoint_recovery_paths_are_owner_scoped():
root = Path(__file__).resolve().parents[1]
chat_routes = (root / "routes" / "chat_routes.py").read_text(encoding="utf-8")
tests/test_tool_path_confinement.py
+7 -9
View File
@@ -169,14 +169,12 @@ def test_blocks_netrc():
_resolve_tool_path("~/.netrc")
def test_allows_agent_workspace(tmp_path):
"""Paths under the agent's workspace in project data/ must resolve
cleanly. The rest of data/ is application state and is rejected;
tests/test_agent_state_dir_confinement.py covers that side."""
def test_allows_project_data(tmp_path):
"""Paths under project data/ must resolve cleanly."""
from src.tool_execution import _resolve_tool_path
from src.constants import AGENT_WORKSPACE_DIR
target = os.path.join(AGENT_WORKSPACE_DIR, "test-confinement-ok.txt")
os.makedirs(AGENT_WORKSPACE_DIR, exist_ok=True)
from src.constants import DATA_DIR
target = os.path.join(DATA_DIR, "test-confinement-ok.txt")
os.makedirs(DATA_DIR, exist_ok=True)
with open(target, "w") as f:
f.write("ok")
try:
@@ -315,8 +313,8 @@ async def test_write_file_dispatch_rejects_empty_directory_like_workspace_path(m
workspace=str(tmp_path),
security_context=NO_TOOL_SECURITY_CONTEXT,
)
assert "BLOCKED" in desc
assert "content required; missing content section" in (
assert desc == "write_file: /workspace/papers"
assert "content required; refusing to create an empty file" in (
result.get("error") or ""
)
assert result.get("exit_code") == 1
tests/test_url_safety.py
+11 -35
View File
@@ -3,8 +3,6 @@
A stub resolver is injected so the tests never touch real DNS.
"""
import pytest
from src.url_safety import check_outbound_url
@@ -66,39 +64,6 @@ def test_strict_mode_blocks_private_and_loopback():
assert ok is False and "private" in reason
@pytest.mark.parametrize("block_private", [False, True])
@pytest.mark.parametrize("host,ips", [
("127.0.0.1", ["127.0.0.1"]),
("[::1]", ["::1"]),
("[::ffff:127.0.0.1]", ["::ffff:127.0.0.1"]),
("localhost", ["127.0.0.1", "::1"]),
("localhost", ["::1", "127.0.0.1"]),
])
def test_loopback_urls_follow_private_policy(host, ips, block_private):
ok, reason = check_outbound_url(
f"http://{host}:8080", block_private=block_private,
resolver=_resolver({host.strip("[]"): ips}),
)
assert ok is (not block_private), reason
if block_private:
assert "loopback address blocked" in reason
@pytest.mark.parametrize("block_private", [False, True])
@pytest.mark.parametrize("ip", [
"169.254.169.254", "fe80::1", "0.0.0.0", "::", "224.0.0.1", "ff02::1",
"240.0.0.1", "::2", "100::1", "::ffff:169.254.169.254",
"::ffff:0.0.0.0", "::ffff:224.0.0.1", "::ffff:240.0.0.1",
])
def test_special_ranges_stay_blocked_alongside_loopback(ip, block_private):
ok, reason = check_outbound_url(
"http://localhost:8080", block_private=block_private,
resolver=_resolver({"localhost": [ip, "127.0.0.1", "::1"]}),
)
assert ok is False
assert "link-local" in reason or "disallowed address" in reason
def test_strict_mode_blocks_cgnat_shared_space():
# RFC 6598 shared/CGNAT space (100.64.0.0/10) is not globally routable.
# A public redirect into it must be rejected under full SSRF lockdown,
@@ -282,3 +247,14 @@ def test_ordinary_ipv6_behaviour_is_unchanged():
LL6 = _resolver({"ll.example": ["fe80::1"]})
ok, reason = check_outbound_url("http://ll.example/", resolver=LL6)
assert ok is False and "link-local" in reason
# Pre-existing behaviour, unchanged here: CPython reports ::1 as
# is_reserved, so IPv6 loopback is rejected in both modes (unlike 127.0.0.1,
# which the local-first default allows).
LOOP6 = _resolver({"loop6.example": ["::1"]})
for strict in (False, True):
ok, reason = check_outbound_url(
"http://loop6.example/", block_private=strict, resolver=LOOP6
)
assert ok is False, strict
assert "NAT64" not in reason