mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-31 10:12:21 +01:00
fix(harness): ship coherent vendor harness cluster
Rebuild PR C to ship the vendor/photocraft harness as a coherent unit (vendor implementation + vendor tests) instead of a per-file ahead/behind mix that produced broken imports and 6+ test failures. - Flip 36 harness impl/route/test files back to vendor (the per-file line-set policy misclassified user-rewritten files as 'behind'). - Flip core/models.py + src/tool_approval_scopes.py to vendor: vendor's harness (authority.py, turn_contract.py, clean_agent_preview.py) depends on vendor's older approval-scope API; upstream added CHAT_SESSION_APPROVAL_SIGNATURE_FIELD that vendor's core/models.py lacks. - Add src/shutdown.py (new vendor infra) required by routes/chat_routes.py. - Keep src/agent_tools/__init__.py at upstream (editor_tools registration lives in PR B). Upstream-only security divergence preserved in C_upstream_delta.patch and flagged for integration-time reconciliation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
pewdiepie-archdaemon
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
364229f2dc
commit
9bdea753ae
@@ -29,7 +29,6 @@ from .interaction_tools import AskUserTool, UpdatePlanTool
|
||||
from .model_interaction_tools import ChatWithModelTool, AskTeacherTool, ListModelsTool
|
||||
from .bg_job_tools import ManageBgJobsTool
|
||||
from .session_tools import CreateSessionTool, ListSessionsTool, SendToSessionTool, ManageSessionTool
|
||||
from .editor_tools import PlanEditTool, RunEditorCommandTool, GetEditorScreenshotTool, AiEditImageTool
|
||||
from .admin_tools import (
|
||||
ADMIN_TOOL_HANDLERS,
|
||||
do_manage_endpoints, do_manage_mcp, do_manage_webhooks,
|
||||
@@ -73,10 +72,6 @@ TOOL_HANDLERS = {
|
||||
"list_sessions": ListSessionsTool().execute,
|
||||
"send_to_session": SendToSessionTool().execute,
|
||||
"manage_session": ManageSessionTool().execute,
|
||||
"plan_edit": PlanEditTool().execute,
|
||||
"run_editor_command": RunEditorCommandTool().execute,
|
||||
"get_editor_screenshot": GetEditorScreenshotTool().execute,
|
||||
"ai_edit_image": AiEditImageTool().execute,
|
||||
}
|
||||
# Config/integration admin tools (manage_endpoints/mcp/webhooks/tokens/settings).
|
||||
TOOL_HANDLERS.update(ADMIN_TOOL_HANDLERS)
|
||||
|
||||
+114
-572
@@ -3,9 +3,8 @@ import json
|
||||
import os
|
||||
import re
|
||||
import difflib
|
||||
import secrets
|
||||
import fnmatch
|
||||
import shutil
|
||||
import time
|
||||
import tempfile
|
||||
from typing import Optional, Dict, Any, Tuple, List
|
||||
|
||||
@@ -19,8 +18,6 @@ _CODENAV_SKIP_DIRS = frozenset({
|
||||
})
|
||||
_CODENAV_MAX_HITS = 200
|
||||
_CODENAV_MAX_LINE = 400
|
||||
_GREP_TIMEOUT_SECONDS = 20
|
||||
_GREP_STDERR_PREFIX = 20_000
|
||||
_STRUCTURED_DOCUMENT_SUFFIXES = frozenset({
|
||||
".doc", ".docx", ".epub", ".pdf", ".pptx", ".xls", ".xlsx",
|
||||
})
|
||||
@@ -91,79 +88,6 @@ def _glob_to_regex(pat: str) -> "re.Pattern":
|
||||
i += 1
|
||||
return re.compile("".join(out))
|
||||
|
||||
|
||||
def _validate_grep_descriptor(descriptor):
|
||||
"""Revalidate an inert parent snapshot without inherited ContextVars."""
|
||||
path, identity, ancestors = descriptor
|
||||
if os.path.islink(path) or os.path.realpath(path) != path:
|
||||
raise ValueError("grep: resource path changed")
|
||||
for parent, observed in ancestors:
|
||||
info = os.stat(parent, follow_symlinks=False)
|
||||
if not os.path.isdir(parent) or (info.st_dev, info.st_ino) != tuple(observed):
|
||||
raise ValueError("grep: resource ancestor changed")
|
||||
info = os.stat(path, follow_symlinks=False)
|
||||
import stat
|
||||
if (not stat.S_ISREG(info.st_mode) or info.st_nlink > 1
|
||||
or (info.st_dev, info.st_ino) != tuple(identity)):
|
||||
raise ValueError("grep: resource identity changed")
|
||||
|
||||
|
||||
def _python_grep_worker(payload: dict, output_queue) -> None:
|
||||
"""Spawn-safe fallback grep worker used when ripgrep is unavailable.
|
||||
|
||||
Keep this at module scope: a frozen Windows executable cannot safely be
|
||||
relaunched as ``sys.executable -c ...``, while multiprocessing can invoke a
|
||||
top-level target through its frozen-process bootstrap.
|
||||
"""
|
||||
try:
|
||||
flags = re.IGNORECASE if payload["ignore_case"] else 0
|
||||
try:
|
||||
regex = re.compile(payload["pattern"], flags)
|
||||
glob_regex = (
|
||||
_glob_to_regex(payload["glob"].replace("\\", "/"))
|
||||
if payload["glob"]
|
||||
else None
|
||||
)
|
||||
except re.error as exc:
|
||||
output_queue.put(("error", f"grep: bad pattern: {exc}"))
|
||||
return
|
||||
|
||||
max_hits = payload["max_hits"]
|
||||
hits = 0
|
||||
|
||||
for descriptor in payload["files"]:
|
||||
if hits >= max_hits:
|
||||
break
|
||||
path, identity, ancestors = descriptor
|
||||
_validate_grep_descriptor(descriptor)
|
||||
relative = os.path.relpath(path, payload["base"]).replace(os.sep, "/")
|
||||
if glob_regex and not (
|
||||
glob_regex.fullmatch(relative) or glob_regex.fullmatch(os.path.basename(path))
|
||||
):
|
||||
continue
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", errors="strict") as handle:
|
||||
info = os.fstat(handle.fileno())
|
||||
if (info.st_dev, info.st_ino) != tuple(identity) or info.st_nlink > 1:
|
||||
raise ValueError("grep: resource identity changed before read")
|
||||
for number, line in enumerate(handle, 1):
|
||||
if regex.search(line):
|
||||
output_queue.put(("match", path, number, line.rstrip()[:_CODENAV_MAX_LINE]))
|
||||
hits += 1
|
||||
if hits >= max_hits:
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
except OSError as error:
|
||||
output_queue.put(("error", f"grep: {error}"))
|
||||
return
|
||||
output_queue.put(("done",))
|
||||
except BaseException as exc:
|
||||
try:
|
||||
output_queue.put(("error", f"grep: fallback worker failed: {exc}"))
|
||||
except BaseException:
|
||||
pass
|
||||
|
||||
def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
|
||||
if old == new:
|
||||
return None
|
||||
@@ -344,82 +268,32 @@ class ReadFileTool:
|
||||
data = data[:MAX_READ_CHARS] + f"\n... [truncated at {MAX_READ_CHARS} chars]"
|
||||
return {"output": data, "exit_code": 0}
|
||||
|
||||
|
||||
def _write_new_file_without_overwrite(path: str, body: str) -> None:
|
||||
"""Publish a new file without exposing a writable placeholder at its path.
|
||||
|
||||
Stage beside the destination, then hard-link it into place. The link is
|
||||
atomic and fails if another writer created the destination first.
|
||||
"""
|
||||
directory = os.path.dirname(path) or "."
|
||||
temporary_path = os.path.join(
|
||||
directory, f".odysseus-write-{secrets.token_hex(16)}.tmp"
|
||||
)
|
||||
fd = os.open(
|
||||
temporary_path,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
||||
0o666,
|
||||
)
|
||||
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as temporary_file:
|
||||
fd = None
|
||||
temporary_file.write(body)
|
||||
os.link(temporary_path, path)
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
try:
|
||||
os.unlink(temporary_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
class _EmptyBodyWouldTruncate(Exception):
|
||||
"""Raised inside the write thread when an undeclared empty body is about to
|
||||
replace a file that holds bytes. Carries the size at risk so the caller can be
|
||||
told what it would have lost (#6414)."""
|
||||
|
||||
def __init__(self, path: str, existing_bytes: int):
|
||||
super().__init__(path)
|
||||
self.path = path
|
||||
self.existing_bytes = existing_bytes
|
||||
|
||||
def _parse_write_intent(content: str) -> tuple[str, str, bool, bool]:
|
||||
"""Classify the original transport, before binding or source normalization.
|
||||
|
||||
The returned clear flag is derived here, never from a caller-supplied key.
|
||||
"""
|
||||
if not isinstance(content, str):
|
||||
raise ValueError("write_file: expected string arguments")
|
||||
if content.lstrip().startswith("{"):
|
||||
try:
|
||||
args = json.loads(content)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError("write_file: expected valid JSON arguments") from error
|
||||
if not isinstance(args, dict):
|
||||
raise ValueError("write_file: expected a JSON object")
|
||||
path, body = args.get("path"), args.get("content")
|
||||
if not isinstance(body, str):
|
||||
raise ValueError("write_file: content required and must be a string")
|
||||
if not isinstance(path, str):
|
||||
raise ValueError("write_file: path required and must be a string")
|
||||
return path.strip(), body, True, not body.strip()
|
||||
path, delimiter, body = content.partition("\n")
|
||||
return path.strip(), body, bool(delimiter), False
|
||||
|
||||
|
||||
class WriteFileTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import _display_tool_path, _resolve_tool_path
|
||||
from src.agent_runtime.resource_binding import active_resource_operation
|
||||
bound = active_resource_operation()
|
||||
original = bound.operation.input if bound is not None else content
|
||||
try:
|
||||
_, _, has_section, declared_clear = _parse_write_intent(original)
|
||||
raw_path, body, _, _ = _parse_write_intent(content)
|
||||
except ValueError as error:
|
||||
return {"error": str(error), "exit_code": 1}
|
||||
from src.tool_execution import _display_tool_path, _resolve_tool_path, _resolve_search_root, _truncate
|
||||
lines = content.split("\n", 1)
|
||||
raw_path = lines[0].strip()
|
||||
body = lines[1] if len(lines) > 1 else ""
|
||||
# Decode JSON-object args (the fenced inline-args shape
|
||||
# ```write_file {"path": "...", "content": "..."}```), matching
|
||||
# ReadFileTool above. Without this the whole JSON string becomes the
|
||||
# path and the file is written under a garbage name. This is the live
|
||||
# path: there is no filesystem MCP server, so write_file always runs
|
||||
# here via _direct_fallback, not through _build_mcp_args.
|
||||
_stripped = content.strip()
|
||||
if _stripped.startswith("{"):
|
||||
try:
|
||||
_a = json.loads(_stripped)
|
||||
if not isinstance(_a, dict):
|
||||
return {"error": "write_file: expected a JSON object", "exit_code": 1}
|
||||
raw_path_value = _a.get("path")
|
||||
body_value = _a.get("content")
|
||||
raw_path = raw_path_value.strip() if isinstance(raw_path_value, str) else ""
|
||||
if not isinstance(body_value, str):
|
||||
return {"error": "write_file: content required", "exit_code": 1}
|
||||
body = body_value
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return {"error": "write_file: expected valid JSON arguments", "exit_code": 1}
|
||||
if not raw_path:
|
||||
return {"error": "write_file: path required", "exit_code": 1}
|
||||
try:
|
||||
@@ -449,6 +323,16 @@ class WriteFileTool:
|
||||
"exit_code": 1,
|
||||
"artifact_format_error": True,
|
||||
}
|
||||
if not body:
|
||||
return {
|
||||
"error": (
|
||||
f"write_file: {path}: content required; refusing to create an "
|
||||
"empty file. Call write_file again with the exact filename and "
|
||||
"non-empty content. If you need a directory, create it from "
|
||||
"bash/python instead."
|
||||
),
|
||||
"exit_code": 1,
|
||||
}
|
||||
# write_file is a UTF-8 text writer. Refuse to silently destroy an
|
||||
# existing PDF, image, archive, or media artifact produced by a
|
||||
# format-aware tool, especially after the agent has verified it.
|
||||
@@ -463,10 +347,6 @@ class WriteFileTool:
|
||||
"exit_code": 1,
|
||||
"binary_artifact_preserved": target_existed,
|
||||
}
|
||||
if not has_section:
|
||||
return {"error": "write_file: content required; missing content section", "exit_code": 1}
|
||||
if raw_path.endswith(("/", "\\")) or os.path.isdir(path):
|
||||
return {"error": "write_file: target is a directory", "exit_code": 1}
|
||||
# This writer truncates in place. Once that stage is reached, a failure
|
||||
# may leave a partial file; report it so effect evidence stays honest.
|
||||
attempted = []
|
||||
@@ -481,67 +361,21 @@ class WriteFileTool:
|
||||
d = os.path.dirname(path)
|
||||
if d:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
if not body.strip() and not declared_clear:
|
||||
# Empty/whitespace-only writes to an already-empty file are no-ops.
|
||||
# Avoid reopening in truncating mode: another writer may have added
|
||||
# data since the read above.
|
||||
if os.path.isfile(path):
|
||||
existing_bytes = os.path.getsize(path)
|
||||
if existing_bytes > 0:
|
||||
raise _EmptyBodyWouldTruncate(path, existing_bytes)
|
||||
return old, 0
|
||||
|
||||
try:
|
||||
if body:
|
||||
# Publish whitespace content atomically. Writing it
|
||||
# after exclusive creation could overwrite bytes from
|
||||
# a writer that filled the new placeholder meanwhile.
|
||||
attempted.append(True)
|
||||
_write_new_file_without_overwrite(path, body)
|
||||
else:
|
||||
# An exact empty body needs no staged data, so create
|
||||
# the file exclusively and never write through it.
|
||||
attempted.append(True)
|
||||
with open(path, "x", encoding="utf-8"):
|
||||
pass
|
||||
except FileExistsError:
|
||||
if os.path.isfile(path):
|
||||
existing_bytes = os.path.getsize(path)
|
||||
if existing_bytes > 0:
|
||||
raise _EmptyBodyWouldTruncate(path, existing_bytes)
|
||||
return old, 0
|
||||
raise
|
||||
return old, len(body)
|
||||
|
||||
attempted.append(True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(body)
|
||||
return old, len(body)
|
||||
old_content, size = await asyncio.to_thread(_write)
|
||||
except _EmptyBodyWouldTruncate as e:
|
||||
clear_call = json.dumps({"path": raw_path, "content": ""})
|
||||
return {
|
||||
"error": (
|
||||
f"write_file: refused to write an empty body over {e.path} — it holds "
|
||||
f"{e.existing_bytes} bytes, which the write would have destroyed, so "
|
||||
f"the file is unchanged. To clear it on purpose, resend with an "
|
||||
f"explicit empty content: {clear_call}"
|
||||
),
|
||||
"exit_code": 1,
|
||||
}
|
||||
except PermissionError:
|
||||
return {"error": f"write_file: {path}: permission denied", "exit_code": 1,
|
||||
**({"mutation_attempted": True} if attempted else {})}
|
||||
except OSError as e:
|
||||
return {"error": f"write_file: {path}: {e}", "exit_code": 1,
|
||||
**({"mutation_attempted": True} if attempted else {})}
|
||||
committed_body = old_content if size == 0 and not declared_clear and not body.strip() else body
|
||||
diff = _unified_diff(old_content, committed_body, path)
|
||||
diff = _unified_diff(old_content, body, path)
|
||||
result = {
|
||||
"output": (f"Wrote {size} bytes to {_display_tool_path(path)}" if attempted else
|
||||
f"No write performed for {_display_tool_path(path)} (implicit empty body)"),
|
||||
"output": f"Wrote {size} bytes to {_display_tool_path(path)}",
|
||||
"exit_code": 0,
|
||||
**({"write_noop": True} if not attempted else {}),
|
||||
}
|
||||
if diff:
|
||||
result["diff"] = diff
|
||||
@@ -785,7 +619,7 @@ def _apply_patch_hunks(original: str, hunks: List[List[str]], label: str) -> str
|
||||
|
||||
class LsTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import _display_tool_path, _is_denied_tool_path, _resolve_search_root, _truncate
|
||||
from src.tool_execution import _display_tool_path, _resolve_search_root, _truncate
|
||||
raw_path = ""
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
@@ -801,18 +635,15 @@ class LsTool:
|
||||
return {"error": f"ls: {e}", "exit_code": 1}
|
||||
|
||||
def _ls():
|
||||
from src.agent_runtime.resources import _control_plane_snapshot
|
||||
if not os.path.isdir(root):
|
||||
return None, f"ls: {root}: not a directory"
|
||||
rows = []
|
||||
snapshot = _control_plane_snapshot()
|
||||
try:
|
||||
with os.scandir(root) as it:
|
||||
for entry in it:
|
||||
if entry.name.startswith("."):
|
||||
continue
|
||||
if (_is_denied_tool_path(os.path.realpath(entry.path), snapshot=snapshot)
|
||||
or not _visible_bound_resource(entry.path)):
|
||||
if not _visible_bound_resource(entry.path):
|
||||
continue
|
||||
try:
|
||||
is_dir = entry.is_dir(follow_symlinks=False)
|
||||
@@ -841,8 +672,6 @@ class GlobTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_SENSITIVE_BASENAMES,
|
||||
_can_traverse_tool_path,
|
||||
_is_denied_tool_path,
|
||||
_display_tool_path,
|
||||
_is_sensitive_path,
|
||||
_resolve_tool_path,
|
||||
@@ -867,12 +696,10 @@ class GlobTool:
|
||||
return {"error": f"glob: {e}", "exit_code": 1}
|
||||
|
||||
def _glob():
|
||||
from src.agent_runtime.resources import _control_plane_snapshot
|
||||
base = os.path.abspath(root)
|
||||
if not os.path.isdir(base):
|
||||
return None, f"glob: {root}: not a directory"
|
||||
rbase = os.path.realpath(base)
|
||||
snapshot = _control_plane_snapshot()
|
||||
norm_pat = pattern.replace("\\", "/")
|
||||
# Fast path: literal pattern (no wildcards) → direct path lookup.
|
||||
if not any(c in norm_pat for c in "*?["):
|
||||
@@ -889,8 +716,7 @@ class GlobTool:
|
||||
# .ssh/id_rsa, …) falls through to the walk, which skips it —
|
||||
# otherwise glob would surface secret paths that read_file /
|
||||
# grep already refuse to touch.
|
||||
if (inside and os.path.exists(cand) and not _is_denied_tool_path(cand, snapshot=snapshot)
|
||||
and _visible_bound_resource(cand)):
|
||||
if inside and os.path.exists(cand) and not _is_sensitive_path(cand) and _visible_bound_resource(cand):
|
||||
return [cand], None
|
||||
# Literal not at exact path — fall through to walk so
|
||||
# e.g. "foo.py" still matches at any depth (like rglob).
|
||||
@@ -900,18 +726,13 @@ class GlobTool:
|
||||
cap = _CODENAV_MAX_HITS * 5
|
||||
try:
|
||||
for dp, dns, fns in os.walk(base):
|
||||
if not _can_traverse_tool_path(os.path.realpath(dp), snapshot=snapshot):
|
||||
dns[:] = []
|
||||
continue
|
||||
# Prune skipped dirs before descending (unlike rglob which
|
||||
# descends first then filters — fatal on large node_modules).
|
||||
# Sensitive dirs (.ssh, .gnupg, …) are pruned too so glob
|
||||
# never enumerates the keys/tokens inside them.
|
||||
dns[:] = [
|
||||
d for d in dns
|
||||
if d not in _CODENAV_SKIP_DIRS
|
||||
and d not in _SENSITIVE_BASENAMES
|
||||
and _can_traverse_tool_path(os.path.realpath(os.path.join(dp, d)), snapshot=snapshot)
|
||||
if d not in _CODENAV_SKIP_DIRS and d not in _SENSITIVE_BASENAMES
|
||||
]
|
||||
for name in fns + dns:
|
||||
full = os.path.join(dp, name)
|
||||
@@ -919,8 +740,7 @@ class GlobTool:
|
||||
if regex.fullmatch(rel) or regex.fullmatch(name):
|
||||
# Skip deny-listed sensitive files (.env, id_rsa,
|
||||
# known_hosts, …) the same way grep does.
|
||||
if (_is_denied_tool_path(os.path.realpath(full), snapshot=snapshot)
|
||||
or not _visible_bound_resource(full)):
|
||||
if _is_sensitive_path(os.path.realpath(full)) or not _visible_bound_resource(full):
|
||||
continue
|
||||
try:
|
||||
mtime = os.stat(full).st_mtime
|
||||
@@ -947,14 +767,10 @@ class GlobTool:
|
||||
class GrepTool:
|
||||
async def execute(self, content: str, ctx: dict) -> dict:
|
||||
from src.tool_execution import (
|
||||
_SENSITIVE_BASENAMES,
|
||||
_SENSITIVE_FILE_PATTERNS,
|
||||
_agent_readable_data_subdirs,
|
||||
_is_denied_tool_path,
|
||||
_display_tool_path,
|
||||
_can_traverse_tool_path,
|
||||
_is_sensitive_path,
|
||||
_path_within,
|
||||
_resolve_tool_path,
|
||||
_resolve_search_root,
|
||||
_truncate,
|
||||
)
|
||||
@@ -983,357 +799,83 @@ class GrepTool:
|
||||
return {"error": f"grep: {e}", "exit_code": 1}
|
||||
|
||||
def _grep():
|
||||
import multiprocessing
|
||||
import queue
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
from src.agent_runtime.resources import _control_plane_snapshot
|
||||
from src.constants import DATA_DIR
|
||||
|
||||
rg = shutil.which("rg")
|
||||
real_root = os.path.realpath(root)
|
||||
deadline = time.monotonic() + _GREP_TIMEOUT_SECONDS
|
||||
base = real_root if os.path.isdir(real_root) else os.path.dirname(real_root)
|
||||
import re as _re
|
||||
import shutil
|
||||
from src.agent_runtime.resource_binding import active_resource_operation
|
||||
bound = active_resource_operation()
|
||||
if bound is not None:
|
||||
bound.validate()
|
||||
files = []
|
||||
snapshot = _control_plane_snapshot()
|
||||
|
||||
def check_deadline():
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("grep: timed out")
|
||||
|
||||
def observe_file(path):
|
||||
check_deadline()
|
||||
if os.path.islink(path):
|
||||
return
|
||||
canonical = os.path.realpath(path)
|
||||
if not _path_within(canonical, base) or _is_denied_tool_path(canonical, snapshot=snapshot):
|
||||
return
|
||||
if bound is not None:
|
||||
try:
|
||||
bound.resolve_path(canonical)
|
||||
except ValueError:
|
||||
# Intentionally denied publication control-plane paths
|
||||
# are omitted before any producer is allowed to read.
|
||||
return
|
||||
info = os.stat(canonical, follow_symlinks=False)
|
||||
if not os.path.isfile(canonical):
|
||||
return
|
||||
ancestors = []
|
||||
parent = os.path.dirname(canonical)
|
||||
while _path_within(parent, base):
|
||||
observed = os.stat(parent, follow_symlinks=False)
|
||||
ancestors.append((parent, (observed.st_dev, observed.st_ino)))
|
||||
if parent == base:
|
||||
break
|
||||
parent = os.path.dirname(parent)
|
||||
files.append((canonical, (info.st_dev, info.st_ino), tuple(ancestors)))
|
||||
if len(files) > 100_000:
|
||||
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
|
||||
|
||||
try:
|
||||
if os.path.islink(root):
|
||||
raise ValueError("grep: symlink search root is not allowed")
|
||||
if os.path.isfile(real_root):
|
||||
observe_file(real_root)
|
||||
elif os.path.isdir(real_root):
|
||||
pending_directories = [real_root]
|
||||
enumerated = 0
|
||||
while pending_directories:
|
||||
check_deadline()
|
||||
directory = pending_directories.pop()
|
||||
if not _can_traverse_tool_path(directory, snapshot=snapshot):
|
||||
continue
|
||||
with os.scandir(directory) as entries:
|
||||
for entry in entries:
|
||||
check_deadline()
|
||||
enumerated += 1
|
||||
if enumerated > 100_000:
|
||||
raise ValueError("grep: enumeration limit exceeded; scan incomplete")
|
||||
if entry.is_symlink():
|
||||
continue
|
||||
canonical = os.path.realpath(entry.path)
|
||||
if not _path_within(canonical, base):
|
||||
raise ValueError("grep: directory identity changed during enumeration")
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if (entry.name not in _CODENAV_SKIP_DIRS
|
||||
and _can_traverse_tool_path(canonical, snapshot=snapshot)
|
||||
and (bound is None or _is_denied_tool_path(canonical, snapshot=snapshot)
|
||||
or _visible_bound_resource(canonical))):
|
||||
pending_directories.append(canonical)
|
||||
else:
|
||||
observe_file(entry.path)
|
||||
else:
|
||||
raise FileNotFoundError(f"grep: {root}: not found")
|
||||
check_deadline()
|
||||
except (OSError, ValueError) as error:
|
||||
return None, str(error) if str(error).startswith("grep:") else f"grep: {error}"
|
||||
descriptors = {record[0]: record for record in files}
|
||||
targets = list(descriptors)
|
||||
lines: list[str] = []
|
||||
|
||||
def parse_rg_result(raw: str) -> Optional[str]:
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
if record.get("type") != "match":
|
||||
return None
|
||||
data = record.get("data") or {}
|
||||
path = (data.get("path") or {}).get("text")
|
||||
text_value = (data.get("lines") or {}).get("text")
|
||||
number = data.get("line_number")
|
||||
if not isinstance(path, str) or not isinstance(text_value, str):
|
||||
return None
|
||||
absolute = path if os.path.isabs(path) else os.path.join(base, path)
|
||||
canonical = os.path.realpath(absolute)
|
||||
if canonical not in descriptors:
|
||||
raise ValueError("grep: producer returned an undeclared resource")
|
||||
_validate_grep_descriptor(descriptors[canonical])
|
||||
if bound is not None:
|
||||
bound.resolve_path(canonical)
|
||||
return f"{_display_tool_path(canonical)}:{number}:{text_value.rstrip()[:_CODENAV_MAX_LINE]}"
|
||||
|
||||
def run_rg(cmd: list[str]) -> Optional[str]:
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=base,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
except Exception as exc:
|
||||
return f"grep: {exc}"
|
||||
output: queue.Queue[Optional[str]] = queue.Queue(maxsize=max_hits + 2)
|
||||
stderr_prefix: list[str] = []
|
||||
stderr_size = 0
|
||||
stop_reader = threading.Event()
|
||||
|
||||
def enqueue_stdout(value: Optional[str]) -> bool:
|
||||
# The consumer stops at the result cap or deadline. Never
|
||||
# leave a producer blocked on its bounded queue afterward.
|
||||
while not stop_reader.is_set():
|
||||
try:
|
||||
output.put(value, timeout=0.05)
|
||||
return True
|
||||
except queue.Full:
|
||||
continue
|
||||
return False
|
||||
|
||||
def read_stdout() -> None:
|
||||
assert process.stdout is not None
|
||||
try:
|
||||
for line in process.stdout:
|
||||
if not enqueue_stdout(line.rstrip("\n")):
|
||||
break
|
||||
finally:
|
||||
enqueue_stdout(None)
|
||||
|
||||
def read_stderr() -> None:
|
||||
nonlocal stderr_size
|
||||
assert process.stderr is not None
|
||||
while True:
|
||||
chunk = process.stderr.read(4096)
|
||||
if not chunk:
|
||||
break
|
||||
if stderr_size < _GREP_STDERR_PREFIX:
|
||||
kept = chunk[:_GREP_STDERR_PREFIX - stderr_size]
|
||||
stderr_prefix.append(kept)
|
||||
stderr_size += len(kept)
|
||||
|
||||
stdout_thread = threading.Thread(target=read_stdout, daemon=True)
|
||||
stderr_thread = threading.Thread(target=read_stderr, daemon=True)
|
||||
stdout_thread.start()
|
||||
stderr_thread.start()
|
||||
timed_out = False
|
||||
capped = False
|
||||
try:
|
||||
while len(lines) < max_hits:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
timed_out = True
|
||||
break
|
||||
try:
|
||||
raw = output.get(timeout=remaining)
|
||||
except queue.Empty:
|
||||
timed_out = True
|
||||
break
|
||||
if raw is None:
|
||||
break
|
||||
parsed = parse_rg_result(raw)
|
||||
if parsed and parsed not in lines:
|
||||
lines.append(parsed)
|
||||
capped = len(lines) >= max_hits
|
||||
finally:
|
||||
stop_reader.set()
|
||||
if (timed_out or capped) and process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
remaining = max(0.01, deadline - time.monotonic())
|
||||
return_code = process.wait(timeout=min(1, remaining))
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
return_code = process.wait()
|
||||
stdout_thread.join()
|
||||
stderr_thread.join()
|
||||
if timed_out:
|
||||
return "grep: timed out"
|
||||
if not capped and return_code not in (0, 1):
|
||||
detail = "".join(stderr_prefix).strip()
|
||||
return f"grep: {detail or f'process exited {return_code}'}"
|
||||
return None
|
||||
|
||||
if not os.path.exists(root):
|
||||
return None, f"grep: search target not found: {_display_tool_path(root)}"
|
||||
# The pathname-only fast path scans before individual resources can
|
||||
# be checked. Bound searches must validate every file before read.
|
||||
rg = None if active_resource_operation() is not None else shutil.which("rg")
|
||||
if rg:
|
||||
cmd = [rg, "--line-number", "--with-filename", "--no-heading", "--color=never",
|
||||
"--max-count", str(max_hits)]
|
||||
if ignore_case:
|
||||
cmd.append("--ignore-case")
|
||||
if glob_pat:
|
||||
try:
|
||||
glob_regex = _glob_to_regex(glob_pat.replace("\\", "/"))
|
||||
targets = [
|
||||
target for target in targets
|
||||
if glob_regex.fullmatch(os.path.relpath(target, base).replace(os.sep, "/"))
|
||||
or glob_regex.fullmatch(os.path.basename(target))
|
||||
]
|
||||
except re.error:
|
||||
pass
|
||||
# Validate even when policy filtering leaves no search targets.
|
||||
if not targets:
|
||||
cmd_probe = [rg, "--json", "--no-config"]
|
||||
if glob_pat:
|
||||
cmd_probe += ["--glob", glob_pat]
|
||||
cmd_probe += ["--regexp", pattern, "--", "-"]
|
||||
error = run_rg(cmd_probe)
|
||||
return (None, error) if error else ([], None)
|
||||
relative_targets = [os.path.relpath(target, base) for target in targets]
|
||||
for offset in range(0, len(relative_targets), 128):
|
||||
if len(lines) >= max_hits:
|
||||
break
|
||||
cmd = [
|
||||
rg, "--json", "--no-config", "--no-follow",
|
||||
"--max-count", str(max_hits - len(lines)),
|
||||
"--max-columns", str(_CODENAV_MAX_LINE),
|
||||
"--max-columns-preview",
|
||||
]
|
||||
if ignore_case:
|
||||
cmd.append("--ignore-case")
|
||||
if glob_pat:
|
||||
cmd += ["--glob", glob_pat]
|
||||
for sensitive_pattern in _SENSITIVE_FILE_PATTERNS:
|
||||
cmd += ["--iglob", f"!{sensitive_pattern}"]
|
||||
for skipped_dir in _CODENAV_SKIP_DIRS:
|
||||
cmd += ["--glob", f"!**/{skipped_dir}/**"]
|
||||
cmd += ["--regexp", pattern, "--", *relative_targets[offset:offset + 128]]
|
||||
try:
|
||||
for path in targets[offset:offset + 128]:
|
||||
check_deadline()
|
||||
_validate_grep_descriptor(descriptors[path])
|
||||
if bound is not None:
|
||||
bound.resolve_path(path)
|
||||
error = run_rg(cmd)
|
||||
except (OSError, ValueError) as exc:
|
||||
return None, f"grep: {exc}"
|
||||
if error:
|
||||
return None, error
|
||||
return lines, None
|
||||
|
||||
# This runs inside asyncio.to_thread(), so forking would clone a
|
||||
# multithreaded process and can deadlock. Spawn is platform-safe and
|
||||
# PyInstaller-compatible via launcher's early freeze_support().
|
||||
payload = {
|
||||
"root": base,
|
||||
"base": base,
|
||||
"files": tuple(files),
|
||||
"pattern": pattern,
|
||||
"ignore_case": ignore_case,
|
||||
"glob": glob_pat,
|
||||
"max_hits": max_hits,
|
||||
"skip_dirs": tuple(_CODENAV_SKIP_DIRS),
|
||||
"sensitive_names": tuple(
|
||||
set(_SENSITIVE_BASENAMES) | set(_SENSITIVE_FILE_PATTERNS)
|
||||
),
|
||||
}
|
||||
try:
|
||||
context = multiprocessing.get_context("spawn")
|
||||
output_queue = context.Queue(maxsize=max_hits + 2)
|
||||
worker = context.Process(
|
||||
target=_python_grep_worker, args=(payload, output_queue)
|
||||
)
|
||||
worker.start()
|
||||
except Exception as exc:
|
||||
cmd += ["--glob", glob_pat]
|
||||
# --iglob (not --glob) so the exclusion is case-insensitive:
|
||||
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
|
||||
# resolve to the same secret as their lowercase forms, and the
|
||||
# Python fallback below already folds case via _is_sensitive_path.
|
||||
for _pat in _SENSITIVE_FILE_PATTERNS:
|
||||
cmd += ["--iglob", f"!*{_pat}*"]
|
||||
for _d in _CODENAV_SKIP_DIRS:
|
||||
cmd += ["--glob", f"!**/{_d}/**"]
|
||||
cmd += ["--regexp", pattern, root]
|
||||
try:
|
||||
output_queue.close()
|
||||
except (NameError, OSError, ValueError):
|
||||
pass
|
||||
return None, f"grep: could not start fallback worker: {exc}"
|
||||
error = None
|
||||
completed = False
|
||||
import subprocess
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
|
||||
# ripgrep: 0 = matches, 1 = no matches, 2 = failed scan.
|
||||
# Do not present invalid patterns or IO failures as absence.
|
||||
if p.returncode not in (0, 1):
|
||||
detail = (p.stderr or '').strip()[:1200]
|
||||
return None, f"grep: search failed (exit {p.returncode}): {detail or 'no diagnostic available'}"
|
||||
lines = [ln for ln in (p.stdout or "").splitlines() if ln][:max_hits]
|
||||
return lines, None
|
||||
except subprocess.TimeoutExpired:
|
||||
return None, "grep: timed out"
|
||||
except Exception as _e:
|
||||
return None, f"grep: {_e}"
|
||||
try:
|
||||
while len(lines) < max_hits:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
error = "grep: timed out"
|
||||
break
|
||||
try:
|
||||
# Keep queue waits short enough to observe a spawn
|
||||
# worker that dies during bootstrap/import before it
|
||||
# can enqueue either an error or the done sentinel.
|
||||
record = output_queue.get(timeout=min(0.05, remaining))
|
||||
except queue.Empty:
|
||||
if worker.is_alive():
|
||||
rx = _re.compile(pattern, _re.IGNORECASE if ignore_case else 0)
|
||||
except _re.error as _e:
|
||||
return None, f"grep: bad pattern: {_e}"
|
||||
hits = []
|
||||
scan_errors = []
|
||||
if os.path.isfile(root):
|
||||
file_iter = [root]
|
||||
else:
|
||||
file_iter = []
|
||||
for dp, dns, fns in os.walk(root, onerror=scan_errors.append):
|
||||
dns[:] = [d for d in dns if d not in _CODENAV_SKIP_DIRS]
|
||||
for fn in fns:
|
||||
if glob_pat and not fnmatch.fnmatch(fn, glob_pat):
|
||||
continue
|
||||
worker.join(timeout=0)
|
||||
try:
|
||||
# A multiprocessing queue's feeder can make the
|
||||
# final record visible at process-exit time. Give
|
||||
# that record precedence over the exit status.
|
||||
remaining = deadline - time.monotonic()
|
||||
record = output_queue.get(
|
||||
timeout=min(0.05, max(0, remaining))
|
||||
)
|
||||
except queue.Empty:
|
||||
error = f"grep: fallback worker exited {worker.exitcode}"
|
||||
break
|
||||
if record[0] == "done":
|
||||
completed = True
|
||||
break
|
||||
if record[0] == "error":
|
||||
error = record[1]
|
||||
break
|
||||
_, path, number, text_value = record
|
||||
canonical = os.path.realpath(path)
|
||||
if canonical not in descriptors:
|
||||
error = "grep: fallback returned an undeclared resource"
|
||||
break
|
||||
try:
|
||||
_validate_grep_descriptor(descriptors[canonical])
|
||||
if bound is not None:
|
||||
bound.resolve_path(canonical)
|
||||
except (OSError, ValueError) as exc:
|
||||
error = f"grep: {exc}"
|
||||
break
|
||||
rendered = f"{_display_tool_path(canonical)}:{number}:{text_value}"
|
||||
if rendered not in lines:
|
||||
lines.append(rendered)
|
||||
finally:
|
||||
if completed:
|
||||
worker.join(timeout=min(1, max(0.01, deadline - time.monotonic())))
|
||||
if worker.is_alive():
|
||||
worker.terminate()
|
||||
worker.join(timeout=1)
|
||||
if worker.is_alive():
|
||||
worker.kill()
|
||||
worker.join()
|
||||
output_queue.close()
|
||||
if error:
|
||||
return None, error
|
||||
if (not completed or worker.exitcode not in (0, None)) and len(lines) < max_hits:
|
||||
return None, f"grep: fallback worker exited {worker.exitcode}"
|
||||
return lines, None
|
||||
file_iter.append(os.path.join(dp, fn))
|
||||
for fp in file_iter:
|
||||
if len(hits) >= max_hits:
|
||||
break
|
||||
try:
|
||||
resolved_file = _resolve_tool_path(fp)
|
||||
except ValueError:
|
||||
# Apply the same workspace/sensitive-path checks to each
|
||||
# discovered file, not just the initial search directory.
|
||||
continue
|
||||
try:
|
||||
with open(resolved_file, "r", encoding="utf-8", errors="strict") as f:
|
||||
for i, line in enumerate(f, 1):
|
||||
if rx.search(line):
|
||||
hits.append(f"{fp}:{i}:{line.rstrip()[:_CODENAV_MAX_LINE]}")
|
||||
if len(hits) >= max_hits:
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
except OSError as error:
|
||||
scan_errors.append(error)
|
||||
if scan_errors:
|
||||
return None, "grep: search incomplete; one or more files or directories could not be read"
|
||||
return hits, None
|
||||
|
||||
lines, err = await asyncio.to_thread(_grep)
|
||||
if err:
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
import time
|
||||
import json
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse, urlsplit, urlunsplit
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -76,14 +76,12 @@ def _resolve_fontfile_for_text(text: str) -> str:
|
||||
async def _cancel_host_shell_bridge_request(
|
||||
url: str, token: str, request_id: str,
|
||||
) -> None:
|
||||
if not token or not is_host_shell_bridge_url_allowed(url):
|
||||
return
|
||||
target = host_shell_bridge_endpoint_url(url, "/cancel")
|
||||
base = url.rsplit("/", 1)[0]
|
||||
try:
|
||||
timeout = httpx.Timeout(5.0, connect=2.0, write=2.0, pool=2.0)
|
||||
async with httpx.AsyncClient(timeout=timeout, trust_env=False) as client:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
await client.post(
|
||||
target,
|
||||
f"{base}/cancel",
|
||||
json={"request_id": request_id},
|
||||
headers={"X-Odysseus-TUI-Bridge-Token": token},
|
||||
)
|
||||
@@ -218,12 +216,6 @@ def is_host_shell_bridge_url_allowed(url: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def host_shell_bridge_endpoint_url(url: str, path: str) -> str:
|
||||
"""Replace a validated bridge URL's path without changing its authority."""
|
||||
parsed = urlsplit(url)
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, path, "", ""))
|
||||
|
||||
|
||||
def _replace_workspace_alias(content: str, cwd: str) -> str:
|
||||
"""Map virtual /workspace paths without corrupting absolute host paths."""
|
||||
return re.sub(
|
||||
@@ -690,7 +682,6 @@ class HostShellTool:
|
||||
return {"error": "host_shell: invalid bridge URL", "exit_code": 1}
|
||||
if not token:
|
||||
return {"error": "host_shell: bridge token missing", "exit_code": 1}
|
||||
run_url = host_shell_bridge_endpoint_url(url, "/run")
|
||||
|
||||
job_id = str(args.get("job_id") or "").strip()
|
||||
if not command and not job_id:
|
||||
@@ -725,9 +716,9 @@ class HostShellTool:
|
||||
request_body["request_id"] = request_id
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout + 5, trust_env=False) as client:
|
||||
async with httpx.AsyncClient(timeout=timeout + 5) as client:
|
||||
resp = await client.post(
|
||||
run_url,
|
||||
url,
|
||||
json=request_body,
|
||||
headers={"X-Odysseus-TUI-Bridge-Token": token},
|
||||
)
|
||||
@@ -755,7 +746,7 @@ class HostShellTool:
|
||||
while time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.25)
|
||||
poll = await client.post(
|
||||
run_url,
|
||||
url,
|
||||
json={"job_id": auto_job_id},
|
||||
headers={"X-Odysseus-TUI-Bridge-Token": token},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user