jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-10-12 18:02:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: bound protected filesystem searches

Browse Source
This commit is contained in:
RaresKeY
2026-08-31 14:19:57 +00:00
parent 51a518a061
commit f7063b5364
4 changed files with 362 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent_tools/filesystem_tools.py
+121 -35
View File
@@ -3,8 +3,8 @@ import json
import os
import re
import difflib
import fnmatch
import shutil
import time
from typing import Optional, Dict, Any, Tuple, List
from src.constants import MAX_READ_CHARS, MAX_DIFF_LINES, MAX_OUTPUT_CHARS
@@ -407,7 +407,11 @@ def _apply_patch_hunks(original: str, hunks: List[List[str]], label: str) -> str
class LsTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import _resolve_tool_path, _resolve_search_root, _truncate
from src.tool_execution import (
_is_denied_tool_path,
_resolve_search_root,
_truncate,
)
raw_path = ""
_s = (content or "").strip()
if _s.startswith("{"):
@@ -431,6 +435,8 @@ class LsTool:
for entry in it:
if entry.name.startswith("."):
continue
if _is_denied_tool_path(os.path.realpath(entry.path)):
continue
try:
is_dir = entry.is_dir(follow_symlinks=False)
size = entry.stat(follow_symlinks=False).st_size if not is_dir else 0
@@ -565,6 +571,7 @@ class GrepTool:
async def execute(self, content: str, ctx: dict) -> dict:
from src.tool_execution import (
_SENSITIVE_FILE_PATTERNS,
_agent_readable_data_subdirs,
_can_traverse_tool_path,
_is_denied_tool_path,
_path_within,
@@ -600,44 +607,118 @@ class GrepTool:
import re as _re
import shutil
rg = shutil.which("rg")
# ripgrep does not offer a policy callback for each traversed
# result. When the search root contains DATA_DIR, use the Python
# walker so protected state can be pruned while legitimate data
# carve-outs remain searchable.
from src.constants import DATA_DIR
if _path_within(os.path.realpath(DATA_DIR), os.path.realpath(root)):
rg = None
real_root = os.path.realpath(root)
data_dir = os.path.realpath(DATA_DIR)
spans_state = _path_within(data_dir, real_root)
if spans_state and not rg:
return None, "grep: ripgrep is required when the search root contains application state"
if rg:
cmd = [
rg, "--no-config", "--no-follow", "--line-number",
"--no-heading", "--color=never", "--max-count", str(max_hits),
]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
cmd += ["--glob", glob_pat]
# --iglob (not --glob) so the exclusion is case-insensitive:
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
# resolve to the same secret as their lowercase forms, and the
# Python fallback below already folds case via _is_sensitive_path.
for _pat in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!*{_pat}*"]
for _d in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{_d}/**"]
cmd += ["--regexp", pattern, root]
try:
import subprocess
p = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
lines = [ln for ln in (p.stdout or "").splitlines() if ln][:max_hits]
return lines, None
except subprocess.TimeoutExpired:
return None, "grep: timed out"
except Exception as _e:
return None, f"grep: {_e}"
searches: list[tuple[str, list[str]]] = [(real_root, [])]
if spans_state:
searches = []
# Search everything outside DATA_DIR with a native rg glob
# exclusion. Search validated carve-outs separately so
# their contents remain available without exposing state
# siblings. --no-follow prevents a symlink from bypassing
# the excluded canonical subtree.
if real_root != data_dir:
rel_data = os.path.relpath(data_dir, real_root).replace(
os.sep, "/"
)
searches.append(
(real_root, [f"!{rel_data}", f"!{rel_data}/**"])
)
seen_roots: set[str] = set()
for readable in _agent_readable_data_subdirs():
if not _path_within(
readable, real_root
) or not os.path.exists(readable):
continue
canonical = os.path.realpath(readable)
if canonical not in seen_roots:
seen_roots.add(canonical)
searches.append((canonical, []))
lines: list[str] = []
deadline = time.monotonic() + 20
for search_root, state_excludes in searches:
remaining_hits = max_hits - len(lines)
if remaining_hits <= 0:
break
cmd = [
rg, "--no-config", "--no-follow", "--line-number",
"--no-heading", "--color=never", "--max-count",
str(remaining_hits),
]
if ignore_case:
cmd.append("--ignore-case")
if glob_pat:
cmd += ["--glob", glob_pat]
# --iglob (not --glob) so the exclusion is case-insensitive:
# on a case-insensitive filesystem "ID_RSA"/"Known_Hosts"
# resolve to the same secret as their lowercase forms.
for _pat in _SENSITIVE_FILE_PATTERNS:
cmd += ["--iglob", f"!*{_pat}*"]
for _d in _CODENAV_SKIP_DIRS:
cmd += ["--glob", f"!**/{_d}/**"]
for exclusion in state_excludes:
cmd += ["--glob", exclusion]
cmd += ["--regexp", pattern, search_root]
timeout = deadline - time.monotonic()
if timeout <= 0:
return None, "grep: timed out"
try:
import queue
import subprocess
import threading
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
bufsize=1,
)
except Exception as _e:
return None, f"grep: {_e}"
output: queue.Queue[Optional[str]] = queue.Queue()
def _read_stdout() -> None:
assert process.stdout is not None
try:
for line in process.stdout:
output.put(line.rstrip("\n"))
finally:
output.put(None)
threading.Thread(target=_read_stdout, daemon=True).start()
try:
while len(lines) < max_hits:
remaining = deadline - time.monotonic()
if remaining <= 0:
return None, "grep: timed out"
try:
line = output.get(timeout=remaining)
except queue.Empty:
return None, "grep: timed out"
if line is None:
break
if line and line not in lines:
lines.append(line)
finally:
if process.poll() is None:
process.terminate()
try:
process.wait(timeout=1)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
return lines, None
try:
rx = _re.compile(pattern, _re.IGNORECASE if ignore_case else 0)
except _re.error as _e:
return None, f"grep: bad pattern: {_e}"
glob_rx = _glob_to_regex(glob_pat.replace("\\", "/")) if glob_pat else None
hits = []
if os.path.isfile(root):
file_iter = [root]
@@ -653,7 +734,12 @@ class GrepTool:
and _can_traverse_tool_path(os.path.realpath(os.path.join(dp, d)))
]
for fn in fns:
if glob_pat and not fnmatch.fnmatch(fn, glob_pat):
rel = os.path.relpath(os.path.join(dp, fn), root).replace(
os.sep, "/"
)
if glob_rx and not (
glob_rx.fullmatch(rel) or glob_rx.fullmatch(fn)
):
continue
file_iter.append(os.path.join(dp, fn))
for fp in file_iter:
src/tool_execution.py
+59 -25
View File
@@ -129,14 +129,39 @@ def _is_sensitive_path(resolved: str) -> bool:
def _path_within(resolved: str, root: str) -> bool:
"""True when *resolved* is *root* itself or sits underneath it.
Folds case before comparing, because the caller is a deny rule: a missed
match here ALLOWS the path, where a missed match in the allowlist below
only rejects it. On default macOS a case-variant name opens the same file
and realpath does not canonicalise case, so an unfolded compare would let
the state directory through. casefold rather than os.path.normcase for the
reason given above _is_sensitive_path: normcase is a no-op on POSIX, which
is exactly where that gap lives.
Use the platform's path-case rules. This helper participates in allow
decisions, so unconditional case-folding would let a distinct ``/DATA``
tree masquerade as a descendant of ``/data`` on case-sensitive systems.
"""
resolved, root = os.path.normcase(resolved), os.path.normcase(root)
if resolved == root:
return True
try:
if os.path.commonpath([resolved, root]) == root:
return True
except ValueError:
return False
# normcase is intentionally conservative about assumptions (notably on
# POSIX), so consult the filesystem when paths exist. This recognizes a
# case alias on a case-insensitive volume without treating distinct
# case-sensitive paths as the same allow root.
if os.path.exists(root):
candidate = resolved
while True:
try:
if os.path.exists(candidate) and os.path.samefile(candidate, root):
return True
except OSError:
pass
parent = os.path.dirname(candidate)
if parent == candidate:
break
candidate = parent
return False
def _path_within_conservative(resolved: str, root: str) -> bool:
"""Containment for deny decisions, folding case to fail closed."""
resolved, root = resolved.casefold(), root.casefold()
if resolved == root:
return True
@@ -151,7 +176,8 @@ def _agent_readable_data_subdirs() -> tuple[str, ...]:
The agent's own scratch folder, plus the directories of user content whose
paths the application itself gives to the model, which it would then be
unable to open:
unable to open. These normally live under DATA_DIR; the documented mail
attachment override may instead name a disjoint external directory:
UPLOAD_DIR the chat upload manifest renders "path=<p>" and
says to read it with read_file (agent_loop.py)
@@ -174,29 +200,37 @@ def _agent_readable_data_subdirs() -> tuple[str, ...]:
UPLOAD_DIR,
)
configured = (
AGENT_WORKSPACE_DIR,
UPLOAD_DIR,
MAIL_ATTACHMENTS_DIR,
PERSONAL_DIR,
PERSONAL_UPLOADS_DIR,
(AGENT_WORKSPACE_DIR, False),
(UPLOAD_DIR, False),
# This has a documented environment override and may legitimately
# live outside DATA_DIR, but it must never equal/contain DATA_DIR.
(MAIL_ATTACHMENTS_DIR, True),
(PERSONAL_DIR, False),
(PERSONAL_UPLOADS_DIR, False),
)
data_dir = os.path.realpath(DATA_DIR)
safe: list[str] = []
for raw in configured:
for raw, external_ok in configured:
value = str(raw or "").strip()
# These paths are security-policy carve-outs, not ordinary allowlist
# entries. Accept only explicit absolute paths whose canonical target
# is a strict descendant of DATA_DIR. In particular, an empty/dot,
# filesystem-root, ancestor, equality, or symlink-equivalent setting
# must not turn the whole state directory into readable content.
# These paths are security-policy roots, not ordinary allowlist
# entries. Accept only explicit absolute directory paths. State
# carve-outs must be strict DATA_DIR descendants; the documented mail
# override may also be disjoint. Empty/dot, filesystem-root, ancestor,
# equality, file, or symlink-equivalent settings fail closed.
if not value or not os.path.isabs(os.path.expanduser(value)):
continue
resolved = os.path.realpath(os.path.expanduser(value))
if (
resolved == data_dir
or not _path_within(resolved, data_dir)
or _is_sensitive_path(resolved)
):
if os.path.exists(resolved) and not os.path.isdir(resolved):
continue
inside_data = resolved != data_dir and _path_within(resolved, data_dir)
external_safe = (
external_ok
and resolved != data_dir
and os.path.dirname(resolved) != resolved
and not _path_within(data_dir, resolved)
and not _path_within(resolved, data_dir)
)
if not (inside_data or external_safe) or _is_sensitive_path(resolved):
continue
safe.append(resolved)
return tuple(safe)
@@ -217,7 +251,7 @@ def _is_app_state_path(resolved: str) -> bool:
own settings.json or app.db inside a real workspace is not caught.
"""
from src.constants import DATA_DIR
if not _path_within(resolved, os.path.realpath(DATA_DIR)):
if not _path_within_conservative(resolved, os.path.realpath(DATA_DIR)):
return False
return not any(
_path_within(resolved, d)