mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-11-08 14:12:22 +01:00
Compare commits
16
Commits
b448119919
...
463713c2c6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
463713c2c6 | ||
|
|
c2017fa089 | ||
|
|
53fd856ea8 | ||
|
|
66599b02a2 | ||
|
|
fb3e89b011 | ||
|
|
f72e1bd412 | ||
|
|
2bdf43b74d | ||
|
|
c8b4cd24e0 | ||
|
|
f4aa661502 | ||
|
|
5911b8c0dc | ||
|
|
08e543d1ff | ||
|
|
47a47bf71d | ||
|
|
71dda5b106 | ||
|
|
ad82ee1c83 | ||
|
|
545e692565 | ||
|
|
fa9f62b44c |
@@ -30,10 +30,12 @@ DEFAULT_PRIVILEGES = {
|
||||
"can_manage_memory": True,
|
||||
"max_messages_per_day": 0,
|
||||
"allowed_models": [],
|
||||
"allowed_models_restricted": False,
|
||||
}
|
||||
|
||||
# Admins get everything
|
||||
ADMIN_PRIVILEGES = {k: (True if isinstance(v, bool) else (0 if isinstance(v, int) else [])) for k, v in DEFAULT_PRIVILEGES.items()}
|
||||
ADMIN_PRIVILEGES["allowed_models_restricted"] = False
|
||||
|
||||
DEFAULT_AUTH_PATH = os.path.join(
|
||||
Path(__file__).parent.parent, "data", "auth.json"
|
||||
|
||||
+95
-1
@@ -1458,7 +1458,11 @@ class CalendarCal(TimestampMixin, Base):
|
||||
owner = Column(String, nullable=True, index=True)
|
||||
name = Column(String, nullable=False)
|
||||
color = Column(String, default="#5b8abf")
|
||||
source = Column(String, default="local") # "local" or "timetree"
|
||||
source = Column(String, default="local") # "local" or "caldav"
|
||||
# UUID of the CalDAV account in user prefs that owns this calendar.
|
||||
# NULL for local calendars and for CalDAV calendars created before
|
||||
# multi-account support was added (treated as "use any configured account").
|
||||
account_id = Column(String, nullable=True, index=True)
|
||||
|
||||
events = relationship("CalendarEvent", back_populates="calendar", cascade="all, delete-orphan")
|
||||
|
||||
@@ -1622,11 +1626,80 @@ def init_db():
|
||||
_migrate_add_calendar_metadata()
|
||||
_migrate_add_calendar_is_utc()
|
||||
_migrate_add_calendar_origin()
|
||||
_migrate_add_calendar_account_id()
|
||||
_migrate_chat_messages_fts()
|
||||
_migrate_encrypt_email_passwords()
|
||||
_migrate_encrypt_signatures()
|
||||
_migrate_encrypt_endpoint_keys()
|
||||
|
||||
|
||||
def _migrate_chat_messages_fts():
|
||||
"""Create and backfill the session transcript FTS index for SQLite."""
|
||||
if not DATABASE_URL.startswith("sqlite"):
|
||||
return
|
||||
|
||||
db_path = DATABASE_URL.replace("sqlite:///", "")
|
||||
if db_path == ":memory:":
|
||||
return
|
||||
conn = None
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)")
|
||||
conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe")
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f"chat_messages FTS migration skipped; FTS5 unavailable: {e}")
|
||||
return
|
||||
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5(
|
||||
content,
|
||||
message_id UNINDEXED,
|
||||
session_id UNINDEXED,
|
||||
role UNINDEXED
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai
|
||||
AFTER INSERT ON chat_messages BEGIN
|
||||
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
|
||||
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad
|
||||
AFTER DELETE ON chat_messages BEGIN
|
||||
DELETE FROM chat_messages_fts WHERE message_id = old.id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_au
|
||||
AFTER UPDATE ON chat_messages BEGIN
|
||||
DELETE FROM chat_messages_fts WHERE message_id = old.id;
|
||||
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
|
||||
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
|
||||
END;
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
|
||||
SELECT COALESCE(cm.content, ''), cm.id, cm.session_id, cm.role
|
||||
FROM chat_messages cm
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM chat_messages_fts fts
|
||||
WHERE fts.message_id = cm.id
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}")
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _migrate_add_email_smtp_security():
|
||||
"""Add explicit SMTP security mode for Proton Bridge/custom local SMTP."""
|
||||
import sqlite3
|
||||
@@ -1786,6 +1859,27 @@ def _migrate_add_calendar_origin():
|
||||
logging.getLogger(__name__).warning(f"calendar_events.origin migration failed: {e}")
|
||||
|
||||
|
||||
def _migrate_add_calendar_account_id():
|
||||
"""Add `account_id` to calendars so each CalDAV-backed calendar knows which
|
||||
credential set (from caldav_accounts in user prefs) owns it. Idempotent."""
|
||||
import sqlite3
|
||||
db_path = DATABASE_URL.replace("sqlite:///", "")
|
||||
if not os.path.exists(db_path):
|
||||
return
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.execute("PRAGMA table_info(calendars)")
|
||||
columns = [row[1] for row in cursor.fetchall()]
|
||||
if columns and "account_id" not in columns:
|
||||
conn.execute("ALTER TABLE calendars ADD COLUMN account_id TEXT")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS ix_calendars_account_id ON calendars(account_id)")
|
||||
conn.commit()
|
||||
logging.getLogger(__name__).info("Migrated: added 'account_id' column to calendars")
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f"calendars.account_id migration failed: {e}")
|
||||
|
||||
|
||||
def _migrate_add_calendar_metadata():
|
||||
"""Add importance/event_type/last_pinged columns to calendar_events table."""
|
||||
import sqlite3
|
||||
|
||||
@@ -585,6 +585,27 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
|
||||
hint = " If this is Docker Compose ntfy, set NTFY_BIND to that host/Tailscale IP and NTFY_BASE_URL to the same server URL in .env, then recreate ntfy."
|
||||
return {"ok": False, "message": f"ntfy publish to {full_url} failed: {e}.{hint}"[:500]}
|
||||
|
||||
if preset == "discord_webhook":
|
||||
import httpx
|
||||
webhook_url = (integ.get("base_url") or "").strip()
|
||||
if not webhook_url:
|
||||
return {"ok": False, "message": "No webhook URL set — paste the full Discord webhook URL into the Base URL field."}
|
||||
payload = {
|
||||
"embeds": [{
|
||||
"title": "Odysseus connectivity test",
|
||||
"description": "If you see this, your Discord Webhook integration is wired up correctly.",
|
||||
"color": 5793266,
|
||||
}]
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
r = await client.post(webhook_url, json=payload)
|
||||
if r.is_success:
|
||||
return {"ok": True, "message": "Test embed sent — check your Discord channel to confirm it arrived."}
|
||||
return {"ok": False, "message": f"Discord returned HTTP {r.status_code}: {r.text[:200]}"}
|
||||
except Exception as e:
|
||||
return {"ok": False, "message": f"Request failed: {e}"[:400]}
|
||||
|
||||
# All other presets: GET against a known health endpoint.
|
||||
# Fall back to detecting from name if preset is missing.
|
||||
health_paths = {
|
||||
|
||||
+161
-50
@@ -579,72 +579,178 @@ def _expand_rrule(
|
||||
def setup_calendar_routes() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/calendar", tags=["calendar"])
|
||||
|
||||
# CalDAV connect form (Integrations → Calendar). Storage is local
|
||||
# SQLite; sync (src/caldav_sync.py) pulls remote events into it on
|
||||
# calendar open and periodically via the scheduler.
|
||||
# ── CalDAV multi-account helpers ─────────────────────────────────────────
|
||||
|
||||
def _get_caldav_accounts(owner: str) -> list:
|
||||
from src.caldav_sync import _load_caldav_accounts
|
||||
return _load_caldav_accounts(owner)
|
||||
|
||||
def _save_caldav_accounts(owner: str, accounts: list) -> None:
|
||||
from routes.prefs_routes import _load_for_user, _save_for_user
|
||||
prefs = _load_for_user(owner) or {}
|
||||
prefs["caldav_accounts"] = accounts
|
||||
prefs.pop("caldav", None)
|
||||
_save_for_user(owner, prefs)
|
||||
|
||||
# ── CalDAV config routes (backward-compat single-account API) ────────────
|
||||
|
||||
@router.get("/config")
|
||||
async def get_config(request: Request):
|
||||
"""Legacy single-account endpoint — returns the first configured account."""
|
||||
owner = _require_user(request)
|
||||
from routes.prefs_routes import _load_for_user
|
||||
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
|
||||
caldav_password = cfg.get("password") or ""
|
||||
if caldav_password:
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
if not accounts:
|
||||
return {"url": "", "username": "", "password": "", "has_password": False, "local": True}
|
||||
first = accounts[0]
|
||||
pw = first.get("password") or ""
|
||||
has_pw = False
|
||||
if pw:
|
||||
try:
|
||||
from src.secret_storage import decrypt
|
||||
caldav_password = decrypt(caldav_password)
|
||||
has_pw = bool(decrypt(pw))
|
||||
except Exception:
|
||||
pass
|
||||
# Surface url+username but never hand the password back to the
|
||||
# client — saved-state UI shouldn't leak the credential.
|
||||
has_pw = bool(pw)
|
||||
return {
|
||||
"url": cfg.get("url", "") or "",
|
||||
"username": cfg.get("username", "") or "",
|
||||
"url": first.get("url", "") or "",
|
||||
"username": first.get("username", "") or "",
|
||||
"password": "",
|
||||
"has_password": bool(caldav_password),
|
||||
"local": not bool(cfg.get("url")),
|
||||
"has_password": has_pw,
|
||||
"local": not bool(first.get("url")),
|
||||
}
|
||||
|
||||
@router.post("/config")
|
||||
async def save_config(request: Request):
|
||||
"""Legacy single-account endpoint — upserts the first account."""
|
||||
owner = _require_user(request)
|
||||
from routes.prefs_routes import _load_for_user, _save_for_user
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
prefs = _load_for_user(owner) or {}
|
||||
cfg = dict(prefs.get("caldav") or {})
|
||||
# Empty url => clear the whole entry (treat as "remove integration").
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
if not (body.get("url") or "").strip():
|
||||
prefs.pop("caldav", None)
|
||||
_save_for_user(owner, prefs)
|
||||
_save_caldav_accounts(owner, [])
|
||||
return {"ok": True, "cleared": True}
|
||||
from src.caldav_sync import validate_caldav_url
|
||||
try:
|
||||
cfg["url"] = validate_caldav_url(body.get("url", ""))
|
||||
validated_url = validate_caldav_url(body.get("url", ""))
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
cfg["username"] = (body.get("username") or "").strip()
|
||||
# Preserve the stored password when the client sends an empty
|
||||
# one (edit form re-submitted without re-typing the password).
|
||||
# cfg already holds the existing (already-encrypted) password from
|
||||
# prefs, so we only touch it when a new password is supplied —
|
||||
# re-encrypting the stored value would double-encrypt it.
|
||||
if accounts:
|
||||
acc = dict(accounts[0])
|
||||
else:
|
||||
import uuid as _uuid
|
||||
acc = {"id": str(_uuid.uuid4()), "label": "CalDAV"}
|
||||
acc["url"] = validated_url
|
||||
acc["username"] = (body.get("username") or "").strip()
|
||||
if body.get("password"):
|
||||
from src.secret_storage import encrypt
|
||||
cfg["password"] = encrypt(body["password"])
|
||||
prefs["caldav"] = cfg
|
||||
_save_for_user(owner, prefs)
|
||||
acc["password"] = encrypt(body["password"])
|
||||
new_accounts = [acc] + (accounts[1:] if len(accounts) > 1 else [])
|
||||
_save_caldav_accounts(owner, new_accounts)
|
||||
return {"ok": True}
|
||||
|
||||
# ── CalDAV multi-account CRUD ─────────────────────────────────────────────
|
||||
|
||||
@router.get("/config/accounts")
|
||||
async def list_caldav_accounts(request: Request):
|
||||
"""Return all configured CalDAV accounts (passwords never returned)."""
|
||||
owner = _require_user(request)
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
safe = []
|
||||
for acc in accounts:
|
||||
pw = acc.get("password") or ""
|
||||
has_pw = False
|
||||
if pw:
|
||||
try:
|
||||
from src.secret_storage import decrypt
|
||||
has_pw = bool(decrypt(pw))
|
||||
except Exception:
|
||||
has_pw = bool(pw)
|
||||
safe.append({
|
||||
"id": acc.get("id", ""),
|
||||
"label": acc.get("label", "") or acc.get("url", ""),
|
||||
"url": acc.get("url", "") or "",
|
||||
"username": acc.get("username", "") or "",
|
||||
"has_password": has_pw,
|
||||
})
|
||||
return {"accounts": safe}
|
||||
|
||||
@router.post("/config/accounts")
|
||||
async def add_caldav_account(request: Request):
|
||||
"""Add a new CalDAV account."""
|
||||
import uuid as _uuid
|
||||
owner = _require_user(request)
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
from src.caldav_sync import validate_caldav_url
|
||||
try:
|
||||
url = validate_caldav_url(body.get("url", ""))
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
if not body.get("password"):
|
||||
raise HTTPException(400, "Password is required")
|
||||
from src.secret_storage import encrypt
|
||||
new_acc = {
|
||||
"id": str(_uuid.uuid4()),
|
||||
"label": (body.get("label") or "").strip() or "CalDAV",
|
||||
"url": url,
|
||||
"username": (body.get("username") or "").strip(),
|
||||
"password": encrypt(body["password"]),
|
||||
}
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
accounts.append(new_acc)
|
||||
_save_caldav_accounts(owner, accounts)
|
||||
return {"ok": True, "id": new_acc["id"]}
|
||||
|
||||
@router.put("/config/accounts/{account_id}")
|
||||
async def update_caldav_account(account_id: str, request: Request):
|
||||
"""Update an existing CalDAV account by id."""
|
||||
owner = _require_user(request)
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
idx = next((i for i, a in enumerate(accounts) if a.get("id") == account_id), None)
|
||||
if idx is None:
|
||||
raise HTTPException(404, "Account not found")
|
||||
acc = dict(accounts[idx])
|
||||
if body.get("url"):
|
||||
from src.caldav_sync import validate_caldav_url
|
||||
try:
|
||||
acc["url"] = validate_caldav_url(body["url"])
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
if body.get("label") is not None:
|
||||
acc["label"] = (body.get("label") or "").strip() or "CalDAV"
|
||||
if body.get("username") is not None:
|
||||
acc["username"] = (body.get("username") or "").strip()
|
||||
if body.get("password"):
|
||||
from src.secret_storage import encrypt
|
||||
acc["password"] = encrypt(body["password"])
|
||||
accounts[idx] = acc
|
||||
_save_caldav_accounts(owner, accounts)
|
||||
return {"ok": True}
|
||||
|
||||
@router.delete("/config/accounts/{account_id}")
|
||||
async def delete_caldav_account(account_id: str, request: Request):
|
||||
"""Remove a CalDAV account by id."""
|
||||
owner = _require_user(request)
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
new_accounts = [a for a in accounts if a.get("id") != account_id]
|
||||
if len(new_accounts) == len(accounts):
|
||||
raise HTTPException(404, "Account not found")
|
||||
_save_caldav_accounts(owner, new_accounts)
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/test")
|
||||
async def test_connection(request: Request):
|
||||
"""Actually probe the configured CalDAV server with a PROPFIND
|
||||
request (the same handshake every CalDAV client uses). Accepts
|
||||
an optional {url, username, password} body so the user can test
|
||||
a configuration BEFORE saving it; falls back to the stored
|
||||
creds otherwise. Returns {ok, error?} with a useful message on
|
||||
failure (status code, auth issue, network error)."""
|
||||
"""Probe a CalDAV server with a PROPFIND. Accepts an optional body:
|
||||
{url, username, password} to test before saving, or {account_id} to
|
||||
test an already-saved account. Falls back to the first saved account
|
||||
when nothing is provided."""
|
||||
owner = _require_user(request)
|
||||
try:
|
||||
body = await request.json()
|
||||
@@ -654,19 +760,24 @@ def setup_calendar_routes() -> APIRouter:
|
||||
user = (body.get("username") or "").strip()
|
||||
pw = body.get("password") or ""
|
||||
if not (url and user and pw):
|
||||
# Fall back to saved settings for this user.
|
||||
from routes.prefs_routes import _load_for_user
|
||||
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
|
||||
url = url or (cfg.get("url") or "")
|
||||
user = user or (cfg.get("username") or "")
|
||||
if not pw:
|
||||
pw = cfg.get("password") or ""
|
||||
if pw:
|
||||
try:
|
||||
from src.secret_storage import decrypt
|
||||
pw = decrypt(pw)
|
||||
except Exception:
|
||||
pass
|
||||
# Look up a saved account: by id if supplied, else first account.
|
||||
accounts = _get_caldav_accounts(owner)
|
||||
acc = None
|
||||
if body.get("account_id"):
|
||||
acc = next((a for a in accounts if a.get("id") == body["account_id"]), None)
|
||||
if acc is None and accounts:
|
||||
acc = accounts[0]
|
||||
if acc:
|
||||
url = url or (acc.get("url") or "")
|
||||
user = user or (acc.get("username") or "")
|
||||
if not pw:
|
||||
pw = acc.get("password") or ""
|
||||
if pw:
|
||||
try:
|
||||
from src.secret_storage import decrypt
|
||||
pw = decrypt(pw)
|
||||
except Exception:
|
||||
pass
|
||||
if not (url and user and pw):
|
||||
return {"ok": False, "error": "Missing URL, username, or password"}
|
||||
from src.caldav_sync import validate_caldav_url
|
||||
|
||||
@@ -75,7 +75,7 @@ def _enforce_chat_privileges(request, sess) -> None:
|
||||
allowlist, or HTTPException(429) if the user has hit their daily message
|
||||
cap. No-op for unauthenticated callers or when auth_manager is absent
|
||||
(single-user mode). Admins receive ADMIN_PRIVILEGES from get_privileges,
|
||||
which means empty allowed_models / zero cap → no-op for them.
|
||||
which means unrestricted allowed_models / zero cap -> no-op for them.
|
||||
"""
|
||||
try:
|
||||
user = get_current_user(request)
|
||||
@@ -88,8 +88,10 @@ def _enforce_chat_privileges(request, sess) -> None:
|
||||
return
|
||||
|
||||
privs = auth_manager.get_privileges(user) or {}
|
||||
allowed = privs.get("allowed_models") or []
|
||||
if allowed and sess.model and sess.model not in allowed:
|
||||
allowed_raw = privs.get("allowed_models")
|
||||
allowed = allowed_raw if isinstance(allowed_raw, list) else []
|
||||
restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
|
||||
if restricted and sess.model and sess.model not in allowed:
|
||||
raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
|
||||
|
||||
cap = int(privs.get("max_messages_per_day") or 0)
|
||||
|
||||
+10
-38
@@ -20,6 +20,7 @@ from src import agent_runs
|
||||
from src.model_context import estimate_tokens
|
||||
from src.chat_helpers import coerce_message_and_session
|
||||
from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_url
|
||||
from src.session_search import search_session_messages
|
||||
from src.prompt_security import untrusted_context_message
|
||||
from core.exceptions import SessionNotFoundError
|
||||
from src.auth_helpers import get_current_user
|
||||
@@ -1208,45 +1209,16 @@ def setup_chat_routes(
|
||||
return []
|
||||
|
||||
_user = get_current_user(request)
|
||||
query_term = q.strip()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
base_q = (
|
||||
db.query(DBChatMessage, DBSession.name)
|
||||
.join(DBSession, DBChatMessage.session_id == DBSession.id)
|
||||
.filter(
|
||||
DBSession.archived == False,
|
||||
DBChatMessage.content.ilike(f"%{query_term}%"),
|
||||
DBChatMessage.role.in_(["user", "assistant"]),
|
||||
)
|
||||
return [
|
||||
result.to_dict()
|
||||
for result in search_session_messages(
|
||||
q,
|
||||
limit=limit,
|
||||
owner=_user,
|
||||
restrict_owner=_user is not None,
|
||||
include_legacy_owner=False,
|
||||
)
|
||||
if _user:
|
||||
base_q = base_q.filter(DBSession.owner == _user)
|
||||
rows = base_q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
|
||||
|
||||
results = []
|
||||
for msg, session_name in rows:
|
||||
content = msg.content or ""
|
||||
lower_content = content.lower()
|
||||
idx = lower_content.find(query_term.lower())
|
||||
if idx == -1:
|
||||
snippet = content[:120]
|
||||
else:
|
||||
start = max(0, idx - 50)
|
||||
end = min(len(content), idx + len(query_term) + 50)
|
||||
snippet = ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
|
||||
|
||||
results.append({
|
||||
"session_id": msg.session_id,
|
||||
"session_name": session_name or "Untitled",
|
||||
"role": msg.role,
|
||||
"content_snippet": snippet,
|
||||
"timestamp": msg.timestamp.isoformat() if msg.timestamp else None,
|
||||
})
|
||||
|
||||
return results
|
||||
finally:
|
||||
db.close()
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# POST /api/rewrite — lightweight rewrite of last AI message (no tools)
|
||||
|
||||
@@ -714,6 +714,10 @@ def _open_imap_connection(host: str, port: int, *, starttls: bool, timeout: int
|
||||
conn.sock.settimeout(timeout)
|
||||
except Exception:
|
||||
pass
|
||||
# Raise the IMAP line-length limit from the default 1 MB to 50 MB so that
|
||||
# large mailboxes (tens of thousands of messages) don't crash with
|
||||
# "got more than 1000000 bytes" on UID SEARCH ALL. (#2883)
|
||||
imaplib._MAXLINE = 50_000_000
|
||||
return conn
|
||||
|
||||
def _imap_connect(account_id: str | None = None, owner: str = ""):
|
||||
|
||||
+35
-6
@@ -5,6 +5,7 @@ import re
|
||||
import uuid
|
||||
import json
|
||||
import socket
|
||||
import hashlib
|
||||
import time as _time
|
||||
import logging
|
||||
import httpx
|
||||
@@ -850,6 +851,14 @@ def _visible_models(cached_models, hidden_models, pinned_models=None):
|
||||
return [m for m in merged if m not in hidden]
|
||||
|
||||
|
||||
def _api_key_fingerprint(api_key: Optional[str]) -> str:
|
||||
"""Stable, non-secret label for distinguishing same-URL credentials."""
|
||||
key = (api_key or "").strip()
|
||||
if not key:
|
||||
return ""
|
||||
return hashlib.sha256(key.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
def setup_model_routes(model_discovery):
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
@@ -1397,6 +1406,7 @@ def setup_model_routes(model_discovery):
|
||||
"name": r.name,
|
||||
"base_url": r.base_url,
|
||||
"has_key": bool(r.api_key),
|
||||
"api_key_fingerprint": _api_key_fingerprint(r.api_key),
|
||||
"is_enabled": r.is_enabled,
|
||||
"models": visible,
|
||||
"pinned_models": pinned,
|
||||
@@ -1463,21 +1473,34 @@ def setup_model_routes(model_discovery):
|
||||
)
|
||||
explicit_timeout = _explicit_model_list_timeout(base_url, requested_kind, refresh_timeout)
|
||||
|
||||
# Dedupe: if an endpoint with the same base_url already exists and
|
||||
# is reachable by the caller (shared or owned by them), return it
|
||||
# instead of creating a duplicate row. Fixes "Scan for Servers"
|
||||
# re-adding manually-added endpoints under their host:port name.
|
||||
# Dedupe: if an endpoint with the same base_url and compatible
|
||||
# credentials already exists and is reachable by the caller (shared or
|
||||
# owned by them), return it instead of creating a duplicate row. Keep
|
||||
# same-url/different-key rows distinct so users can group the same
|
||||
# provider URL under multiple credentials.
|
||||
from src.auth_helpers import get_current_user as _gcu_dedup
|
||||
_caller = _gcu_dedup(request) or None
|
||||
_incoming_api_key = api_key.strip()
|
||||
_db_dedup = SessionLocal()
|
||||
try:
|
||||
existing = (
|
||||
_same_url_rows = (
|
||||
_db_dedup.query(ModelEndpoint)
|
||||
.filter(ModelEndpoint.base_url == base_url)
|
||||
.filter((ModelEndpoint.owner.is_(None)) | (ModelEndpoint.owner == _caller))
|
||||
.order_by(ModelEndpoint.owner.desc()) # prefer owned over shared
|
||||
.first()
|
||||
.all()
|
||||
)
|
||||
existing = None
|
||||
_empty_key_existing = None
|
||||
for _candidate in _same_url_rows:
|
||||
_candidate_key = (getattr(_candidate, "api_key", None) or "").strip()
|
||||
if _candidate_key == _incoming_api_key:
|
||||
existing = _candidate
|
||||
break
|
||||
if _incoming_api_key and not _candidate_key and _empty_key_existing is None:
|
||||
_empty_key_existing = _candidate
|
||||
if existing is None and _incoming_api_key and _empty_key_existing is not None:
|
||||
existing = _empty_key_existing
|
||||
if existing:
|
||||
changed = False
|
||||
# Persist any incoming pinned IDs onto the existing row. An
|
||||
@@ -1526,6 +1549,8 @@ def setup_model_routes(model_discovery):
|
||||
"id": existing.id,
|
||||
"name": existing.name,
|
||||
"base_url": existing.base_url,
|
||||
"has_key": bool(existing.api_key),
|
||||
"api_key_fingerprint": _api_key_fingerprint(existing.api_key),
|
||||
"models": _visible_models(
|
||||
existing_models,
|
||||
getattr(existing, "hidden_models", None),
|
||||
@@ -1599,6 +1624,8 @@ def setup_model_routes(model_discovery):
|
||||
"id": ep_id,
|
||||
"name": name.strip(),
|
||||
"base_url": base_url,
|
||||
"has_key": bool(api_key.strip()),
|
||||
"api_key_fingerprint": _api_key_fingerprint(api_key),
|
||||
"models": _merge_model_ids(model_ids, _pinned),
|
||||
"pinned_models": _pinned,
|
||||
"online": bool(model_ids) or bool(_pinned) or bool(ping.get("reachable")),
|
||||
@@ -1948,6 +1975,8 @@ def setup_model_routes(model_discovery):
|
||||
"name": ep.name,
|
||||
"model_type": ep.model_type,
|
||||
"base_url": ep.base_url,
|
||||
"has_key": bool(ep.api_key),
|
||||
"api_key_fingerprint": _api_key_fingerprint(ep.api_key),
|
||||
"pinned_models": _normalize_model_ids(getattr(ep, "pinned_models", None)),
|
||||
"endpoint_kind": getattr(ep, "endpoint_kind", None) or "auto",
|
||||
"model_refresh_mode": getattr(ep, "model_refresh_mode", None) or "auto",
|
||||
|
||||
+91
-7
@@ -114,8 +114,9 @@ async def dispatch_reminder(
|
||||
note_id: str,
|
||||
owner: str = "",
|
||||
queue_browser: bool = True,
|
||||
settings_override: dict | None = None,
|
||||
) -> dict:
|
||||
"""Fire a reminder via the configured channel (browser/email/ntfy).
|
||||
"""Fire a reminder via the configured channel (browser/email/ntfy/webhook).
|
||||
|
||||
Args:
|
||||
title: short headline shown to the user
|
||||
@@ -129,7 +130,7 @@ async def dispatch_reminder(
|
||||
nothing is "sent" synchronously for it — the channel just routes there.
|
||||
"""
|
||||
from src.settings import load_settings
|
||||
settings = load_settings()
|
||||
settings = {**load_settings(), **(settings_override or {})}
|
||||
channel = settings.get("reminder_channel", "browser")
|
||||
llm_on = bool(settings.get("reminder_llm_synthesis", False))
|
||||
title = (title or "").strip()
|
||||
@@ -160,13 +161,14 @@ async def dispatch_reminder(
|
||||
# Treat those as browser-only dedupe so email reminders can be
|
||||
# retried by the backend scanner after a failed frontend path.
|
||||
should_skip = last_dt >= _dt.now(_tz.utc) - _td(minutes=25)
|
||||
if should_skip and channel in ("email", "ntfy"):
|
||||
if should_skip and channel in ("email", "ntfy", "webhook"):
|
||||
should_skip = last_channel == channel
|
||||
if should_skip:
|
||||
return {
|
||||
"synthesis": None,
|
||||
"email_sent": False,
|
||||
"ntfy_sent": False,
|
||||
"webhook_sent": False,
|
||||
"browser_sent": True,
|
||||
"skipped": True,
|
||||
}
|
||||
@@ -360,6 +362,76 @@ async def dispatch_reminder(
|
||||
email_error = str(e) or e.__class__.__name__
|
||||
logger.warning(f"Reminder email send failed: {e}")
|
||||
|
||||
webhook_sent = False
|
||||
webhook_error = ""
|
||||
if channel == "webhook":
|
||||
try:
|
||||
import httpx
|
||||
import json as _wjson
|
||||
from src.integrations import load_integrations
|
||||
# Built-in payload defaults for known presets so users don't have
|
||||
# to configure a template just to use a standard service.
|
||||
_PRESET_TEMPLATE_DEFAULTS = {
|
||||
"discord_webhook": '{"embeds": [{"title": "{{title}}", "description": "{{message}}", "color": 5793266}]}',
|
||||
}
|
||||
intg_id = settings.get("reminder_webhook_integration_id", "").strip()
|
||||
template = settings.get("reminder_webhook_payload_template", "").strip()
|
||||
if not intg_id:
|
||||
webhook_error = "No webhook integration selected"
|
||||
else:
|
||||
intg = next(
|
||||
(i for i in load_integrations()
|
||||
if i.get("id") == intg_id and i.get("base_url")),
|
||||
None,
|
||||
)
|
||||
if not intg:
|
||||
webhook_error = f"Integration {intg_id!r} not found or missing base URL"
|
||||
else:
|
||||
# Fall back to a built-in default for known presets so
|
||||
# users don't have to configure a template for standard
|
||||
# services like Discord.
|
||||
if not template:
|
||||
template = _PRESET_TEMPLATE_DEFAULTS.get(intg.get("preset", ""), "")
|
||||
if not template:
|
||||
webhook_error = "No payload template configured"
|
||||
else:
|
||||
# Render template: JSON-escape the values so the result
|
||||
# is always valid JSON regardless of special characters.
|
||||
# dumps() returns `"value"` — strip outer quotes.
|
||||
msg = (synthesis or note_body or title or "Reminder")[:4000]
|
||||
_t = _wjson.dumps(title or "Reminder")[1:-1]
|
||||
_m = _wjson.dumps(msg)[1:-1]
|
||||
rendered = template.replace("{{title}}", _t).replace("{{message}}", _m)
|
||||
hdrs = {"Content-Type": "application/json"}
|
||||
api_key = intg.get("api_key", "")
|
||||
auth_type = (intg.get("auth_type") or "none").lower()
|
||||
if api_key:
|
||||
if auth_type == "bearer":
|
||||
hdrs["Authorization"] = f"Bearer {api_key}"
|
||||
elif auth_type == "header":
|
||||
hdrs[intg.get("auth_header") or "Authorization"] = api_key
|
||||
url = intg["base_url"].rstrip("/")
|
||||
# SSRF guard — matches the pattern used by webhook_routes,
|
||||
# CalDAV, search, and embeddings. Blocks link-local / metadata
|
||||
# addresses (169.254.x.x) by default; set
|
||||
# REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS=true to also block
|
||||
# RFC-1918 ranges for locked-down deployments.
|
||||
import os as _os
|
||||
from src.url_safety import check_outbound_url as _chk
|
||||
_block = _os.getenv("REMINDER_WEBHOOK_BLOCK_PRIVATE_IPS", "false").lower() == "true"
|
||||
_ok, _reason = _chk(url, block_private=_block)
|
||||
if not _ok:
|
||||
webhook_error = f"Webhook URL rejected: {_reason}"
|
||||
else:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.post(url, content=rendered.encode(), headers=hdrs)
|
||||
webhook_sent = resp.is_success
|
||||
if not webhook_sent:
|
||||
webhook_error = f"Webhook returned HTTP {resp.status_code}"
|
||||
except Exception as e:
|
||||
webhook_error = str(e) or e.__class__.__name__
|
||||
logger.warning(f"Reminder webhook send failed: {e}")
|
||||
|
||||
ntfy_sent = False
|
||||
ntfy_error = ""
|
||||
if channel == "ntfy":
|
||||
@@ -415,7 +487,7 @@ async def dispatch_reminder(
|
||||
# second send for the same note within 25 min. Without this, a note
|
||||
# whose due_date fires while the user has the app open got TWO emails
|
||||
# (frontend-fired here + background-fired by ping_notes 0–5 min later).
|
||||
if (email_sent or ntfy_sent or browser_sent or local_browser_sent) and note_id:
|
||||
if (email_sent or ntfy_sent or webhook_sent or browser_sent or local_browser_sent) and note_id:
|
||||
try:
|
||||
import json as _json
|
||||
from datetime import datetime as _dt, timezone as _tz
|
||||
@@ -431,7 +503,7 @@ async def dispatch_reminder(
|
||||
_cache = cache or (_json.loads(_STATE.read_text(encoding="utf-8")) if _STATE.exists() else {})
|
||||
except Exception:
|
||||
_cache = {}
|
||||
sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "browser"
|
||||
sent_channel = "email" if email_sent else "ntfy" if ntfy_sent else "webhook" if webhook_sent else "browser"
|
||||
_cache[cache_key or str(note_id)] = {
|
||||
"at": _dt.now(_tz.utc).isoformat(),
|
||||
"channel": sent_channel,
|
||||
@@ -441,11 +513,14 @@ async def dispatch_reminder(
|
||||
logger.debug(f"dispatch_reminder: cache write failed: {_e}")
|
||||
|
||||
return {
|
||||
"channel": channel,
|
||||
"synthesis": synthesis,
|
||||
"email_sent": email_sent,
|
||||
"email_error": email_error,
|
||||
"ntfy_sent": ntfy_sent,
|
||||
"ntfy_error": ntfy_error,
|
||||
"webhook_sent": webhook_sent,
|
||||
"webhook_error": webhook_error,
|
||||
"browser_sent": browser_sent or local_browser_sent,
|
||||
}
|
||||
|
||||
@@ -692,12 +767,21 @@ def setup_note_routes(task_scheduler=None):
|
||||
if not note_id:
|
||||
raise HTTPException(400, "note_id required")
|
||||
|
||||
# Delegate to the module-level helper so background tasks can reuse
|
||||
# the same dispatch without an HTTP roundtrip + auth cookie.
|
||||
# Optional overrides let the test button pass the current UI values
|
||||
# directly so the test never races against a pending settings save.
|
||||
_override: dict = {}
|
||||
if body.get("channel"):
|
||||
_override["reminder_channel"] = body["channel"]
|
||||
if body.get("webhook_integration_id"):
|
||||
_override["reminder_webhook_integration_id"] = body["webhook_integration_id"]
|
||||
if body.get("webhook_payload_template"):
|
||||
_override["reminder_webhook_payload_template"] = body["webhook_payload_template"]
|
||||
|
||||
return await dispatch_reminder(
|
||||
title=title, note_body=note_body, note_id=note_id,
|
||||
owner=_owner(request) or "",
|
||||
queue_browser=False,
|
||||
settings_override=_override or None,
|
||||
)
|
||||
|
||||
# --- REORDER NOTES ---
|
||||
|
||||
@@ -11,7 +11,7 @@ from core.session_manager import SessionManager
|
||||
from core.models import ChatMessage
|
||||
from src.request_models import SessionResponse
|
||||
from core.database import Session as DbSession, SessionLocal, Document, GalleryImage
|
||||
from src.auth_helpers import get_current_user, effective_user
|
||||
from src.auth_helpers import get_current_user, effective_user, _auth_disabled
|
||||
|
||||
|
||||
def _sanitize_export_filename(name: str) -> str:
|
||||
@@ -106,8 +106,8 @@ def _verify_session_owner(request: Request, session_id: str, session_manager=Non
|
||||
that only care about persisted sessions keep their exact prior behavior.
|
||||
"""
|
||||
user = effective_user(request)
|
||||
if not user:
|
||||
raise HTTPException(403, "Authentication required")
|
||||
if not user and not _auth_disabled():
|
||||
raise HTTPException(401, "Authentication required")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
row = db.query(DbSession.owner).filter(DbSession.id == session_id).first()
|
||||
|
||||
+50
-34
@@ -82,14 +82,14 @@ def _detect_nvidia():
|
||||
# Retry through a login shell with the common CUDA bin dirs on PATH.
|
||||
if not out and _remote_host:
|
||||
out = _run(
|
||||
"bash -lc 'export PATH=\"$PATH:/usr/bin:/usr/local/bin:/usr/local/cuda/bin\"; "
|
||||
"bash -lc 'export PATH=\"$PATH:/usr/bin:/usr/local/bin:/usr/local/cuda/bin:/usr/lib/wsl/lib\"; "
|
||||
"nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits'"
|
||||
)
|
||||
# Last resort: call nvidia-smi by absolute path. Some hosts have a login
|
||||
# shell that isn't bash (or a profile that errors), so the bash -lc retry
|
||||
# above still comes back empty even though the binary is right there.
|
||||
if not out and _remote_host:
|
||||
for _p in ("/usr/bin/nvidia-smi", "/usr/local/bin/nvidia-smi", "/usr/local/cuda/bin/nvidia-smi"):
|
||||
for _p in ("/usr/bin/nvidia-smi", "/usr/local/bin/nvidia-smi", "/usr/local/cuda/bin/nvidia-smi", "/usr/lib/wsl/lib/nvidia-smi"):
|
||||
out = _run(f"{_p} --query-gpu=memory.total,name --format=csv,noheader,nounits")
|
||||
if out:
|
||||
break
|
||||
@@ -468,39 +468,55 @@ def _detect_windows():
|
||||
"""
|
||||
# Single PowerShell command that gathers all hardware info at once
|
||||
ps_cmd = (
|
||||
"$r = @{}; "
|
||||
"$os = Get-CimInstance Win32_OperatingSystem; "
|
||||
"$r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1); "
|
||||
"$r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1); "
|
||||
"$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1; "
|
||||
"$r.cpu_name = $cpu.Name; "
|
||||
"$r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum; "
|
||||
"$r.arch = $cpu.AddressWidth; "
|
||||
"""
|
||||
$r = @{}
|
||||
$os = Get-CimInstance Win32_OperatingSystem
|
||||
$r.ram_gb = [math]::Round($os.TotalVisibleMemorySize / 1048576, 1)
|
||||
$r.avail_gb = [math]::Round($os.FreePhysicalMemory / 1048576, 1)
|
||||
$cpu = Get-CimInstance Win32_Processor | Select-Object -First 1
|
||||
$r.cpu_name = $cpu.Name
|
||||
$r.cpu_cores = (Get-CimInstance Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
|
||||
$r.arch = $cpu.AddressWidth
|
||||
# GPU detection via nvidia-smi (fastest) or WMI fallback
|
||||
"try { "
|
||||
" $nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null; "
|
||||
" if ($LASTEXITCODE -eq 0 -and $nv) { "
|
||||
" $gpus = @(); "
|
||||
" foreach ($line in $nv -split \"`n\") { "
|
||||
" $p = $line -split ','; "
|
||||
" if ($p.Count -ge 2) { $gpus += [pscustomobject]@{name=$p[1].Trim(); vram_mb=[double]$p[0].Trim()} } "
|
||||
" }; "
|
||||
" $r.gpu_name = $gpus[0].name; "
|
||||
" $r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1); "
|
||||
" $r.gpu_count = $gpus.Count; "
|
||||
" $r.gpu_backend = 'cuda'; "
|
||||
" } "
|
||||
"} catch {}; "
|
||||
"if (-not $r.gpu_name) { "
|
||||
" $wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1; "
|
||||
" if ($wmiGpu) { "
|
||||
" $r.gpu_name = $wmiGpu.Name; "
|
||||
" $r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1); "
|
||||
" $r.gpu_count = 1; "
|
||||
" $r.gpu_backend = 'cpu_x86'; " # WMI doesn't tell us CUDA/ROCm
|
||||
" } "
|
||||
"}; "
|
||||
"$r | ConvertTo-Json -Compress"
|
||||
try {
|
||||
$nv = nvidia-smi --query-gpu=memory.total,name --format=csv,noheader,nounits 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and $nv) {
|
||||
$gpus = @()
|
||||
foreach ($line in $nv -split "`n") {
|
||||
$p = $line -split ','
|
||||
if ($p.Count -ge 2) { $gpus += [pscustomobject]@{name = $p[1].Trim(); vram_mb = [double]$p[0].Trim() } }
|
||||
}
|
||||
$r.gpu_name = $gpus[0].name
|
||||
$r.gpu_vram_gb = [math]::Round(($gpus | Measure-Object -Property vram_mb -Sum).Sum / 1024, 1)
|
||||
$r.gpu_count = $gpus.Count
|
||||
$r.gpu_backend = 'cuda'
|
||||
}
|
||||
}
|
||||
catch {}
|
||||
if (-not $r.gpu_name) {
|
||||
$wmiGpu = Get-CimInstance Win32_VideoController | Where-Object { $_.AdapterRAM -gt 0 } | Select-Object -First 1
|
||||
$GPUDriverKey = "HKLM:\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0*"
|
||||
$GPUDeviceID = $wmiGpu.PNPDeviceID.Split('&')[0..1] -join '&'
|
||||
$VRAMfromRegistry = Get-ItemProperty -Path $GPUDriverKey |
|
||||
Where-Object { $_.MatchingDeviceId -like "${GPUDeviceID}*" } |
|
||||
# Sometimes there happen to be multiple driver classes for the same gpu.
|
||||
Select-Object -ExpandProperty HardwareInformation.qwMemorySize -ErrorAction SilentlyContinue -First 1
|
||||
if ($wmiGpu) {
|
||||
$r.gpu_name = $wmiGpu.Name
|
||||
# Edge case: driver is broken, otherwise $wmiGpu.AdapterRAM is redundant
|
||||
if ($VRAMfromRegistry -ge $wmiGpu.AdapterRAM) {
|
||||
$r.gpu_vram_gb = [math]::Round($VRAMfromRegistry / 1073741824, 1)
|
||||
}
|
||||
else {
|
||||
$r.gpu_vram_gb = [math]::Round($wmiGpu.AdapterRAM / 1073741824, 1)
|
||||
}
|
||||
$r.gpu_count = 1
|
||||
# WMI doesn't tell us CUDA/ROCm
|
||||
$r.gpu_backend = 'cpu_x86';
|
||||
}
|
||||
}
|
||||
$r | ConvertTo-Json -Compress
|
||||
"""
|
||||
)
|
||||
if _remote_host:
|
||||
# Remote: ship a single command string over SSH. The remote shell parses
|
||||
|
||||
+1
-1
@@ -332,7 +332,7 @@ If the user asks for a reminder/alarm before the event, pass `reminder_minutes`
|
||||
"create_session": "- ```create_session``` — Create a new chat. Line 1 = chat name, line 2 = model name. Use for background/parallel work.",
|
||||
"list_sessions": "- ```list_sessions``` — List chats sorted MOST-RECENT FIRST (the UI calls them 'chats') with clickable chat-title links. Output includes a relative \"last active\" timestamp per row, so the first row is the user's most recent chat. Content = optional filter keyword (matches chat name). When answering, preserve the `[title](#session-id)` links exactly; do not convert them into plain text.",
|
||||
"send_to_session": "- ```send_to_session``` — Send a message to another session. Line 1 = session_id, rest = message. Use for orchestrating work across sessions.",
|
||||
"search_chats": "- ```search_chats``` — Search across all chat history. Use when user asks 'did we discuss X?' or 'find the conversation about Y'.",
|
||||
"search_chats": "- ```search_chats``` — Search past session transcripts for direct conversation evidence. Use when user asks 'did we discuss X?', 'find the conversation about Y', or when prior chat context is more appropriate than persistent memory.",
|
||||
"pipeline": "- ```pipeline``` — Run a multi-step AI pipeline. Args (JSON) with ordered steps, each specifying a model and prompt. Use for complex workflows.",
|
||||
"ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle <name> on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel <name>` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply <uid> <folder> <reply|reply-all|ai-reply>` (opens an email compose document, does NOT send), `set_mode agent/chat`, `switch_model <name>`, `set_theme <preset>`, `create_theme <name> <bg> <fg> <panel> <border> <accent>` (optional key=val for advanced colors AND background effects: bgPattern=<none|dots|synapse|rain|constellations|perlin-flow|petals|sparkles|embers>, bgEffectColor=#RRGGBB, bgEffectIntensity=<num>, bgEffectSize=<num>, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel <name>`. Theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute.",
|
||||
"ask_user": "- ```ask_user``` — Ask the user a multiple-choice question when the task is genuinely ambiguous and the answer changes what you do next (pick an approach, confirm an assumption, choose a target). Args (JSON): {\"question\": \"...\", \"options\": [{\"label\": \"...\", \"description\": \"...\"?}, ...], \"multi\": false?}. 2-6 options. The user gets clickable buttons; calling this ENDS your turn and their choice comes back as your next message. Prefer sensible defaults — only ask when you truly can't proceed well without their input.",
|
||||
|
||||
+4
-3
@@ -14,16 +14,17 @@ Sub-modules:
|
||||
import logging
|
||||
from collections import namedtuple
|
||||
|
||||
from src.constants import MAX_OUTPUT_CHARS, MAX_READ_CHARS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants (kept here — sub-modules import from here)
|
||||
# Constants (re-exported for backward compatibility — single source of truth
|
||||
# is src.constants; always prefer importing from there for new code)
|
||||
# ---------------------------------------------------------------------------
|
||||
MAX_AGENT_ROUNDS = 50
|
||||
SHELL_TIMEOUT = 60
|
||||
PYTHON_TIMEOUT = 30
|
||||
MAX_OUTPUT_CHARS = 10_000
|
||||
MAX_READ_CHARS = 20_000
|
||||
|
||||
# Tool types that trigger execution
|
||||
TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_file", "edit_file",
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ def get_current_user(request: Request) -> Optional[str]:
|
||||
return getattr(request.state, 'current_user', None)
|
||||
|
||||
|
||||
def effective_user(request: Request):
|
||||
def effective_user(request: Request) -> Optional[str]:
|
||||
"""The real human behind the request, for ownership/attribution.
|
||||
|
||||
Cookie sessions resolve to the logged-in username. Bearer ``ody_`` callers
|
||||
|
||||
@@ -1902,6 +1902,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]:
|
||||
delivered = bool(dispatch_result.get("email_sent"))
|
||||
elif channel == "ntfy":
|
||||
delivered = bool(dispatch_result.get("ntfy_sent"))
|
||||
elif channel == "webhook":
|
||||
delivered = bool(dispatch_result.get("webhook_sent"))
|
||||
if delivered:
|
||||
newly_notified.update(new_urgent)
|
||||
else:
|
||||
|
||||
+87
-30
@@ -128,10 +128,14 @@ def validate_caldav_url(raw_url: str) -> str:
|
||||
return urlunparse(parsed._replace(fragment="")).rstrip("/")
|
||||
|
||||
|
||||
def _stable_cal_id(remote_url: str) -> str:
|
||||
"""Deterministic local id for a remote CalDAV calendar — same URL
|
||||
always maps to the same local row across restarts and re-syncs."""
|
||||
h = hashlib.sha256(remote_url.encode("utf-8")).hexdigest()[:24]
|
||||
def _stable_cal_id(remote_url: str, owner: str = "", account_id: str = "") -> str:
|
||||
"""Deterministic local id for a remote CalDAV calendar, scoped to owner
|
||||
and account so two users — or one user with two accounts — pointing at
|
||||
the same server URL get distinct local rows (avoids PK collision, #2765).
|
||||
The owner and account_id default to "" for the legacy/URL-only path so
|
||||
existing callers without those arguments keep working."""
|
||||
key = f"{owner}\n{account_id}\n{remote_url}"
|
||||
h = hashlib.sha256(key.encode("utf-8")).hexdigest()[:24]
|
||||
return f"caldav-{h}"
|
||||
|
||||
|
||||
@@ -212,7 +216,7 @@ def _open_url_as_calendar(client, url: str):
|
||||
return client.calendar(url=target)
|
||||
|
||||
|
||||
def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict:
|
||||
def _sync_blocking(owner: str, url: str, username: str, password: str, account_id: str = "") -> dict:
|
||||
"""The actual sync — synchronous, intended to run in a threadpool.
|
||||
Returns counts: {calendars, events, deleted, errors}."""
|
||||
# Lazy imports so a missing `caldav` dep doesn't break app startup —
|
||||
@@ -258,7 +262,7 @@ def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict:
|
||||
for remote_cal in calendars:
|
||||
try:
|
||||
remote_url = str(remote_cal.url)
|
||||
cal_id = _stable_cal_id(remote_url)
|
||||
cal_id = _stable_cal_id(remote_url, owner=owner, account_id=account_id)
|
||||
display_name = (remote_cal.name or "").strip() or "CalDAV"
|
||||
|
||||
local_cal = db.query(CalendarCal).filter(
|
||||
@@ -272,14 +276,20 @@ def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict:
|
||||
name=display_name,
|
||||
color="#5b8abf",
|
||||
source="caldav",
|
||||
account_id=account_id or None,
|
||||
)
|
||||
db.add(local_cal)
|
||||
db.commit()
|
||||
else:
|
||||
# Refresh the display name if the user renamed it
|
||||
# remotely; preserve any local color override.
|
||||
# Refresh display name and stamp account_id if missing.
|
||||
changed = False
|
||||
if local_cal.name != display_name:
|
||||
local_cal.name = display_name
|
||||
changed = True
|
||||
if account_id and not local_cal.account_id:
|
||||
local_cal.account_id = account_id
|
||||
changed = True
|
||||
if changed:
|
||||
db.commit()
|
||||
result["calendars"] += 1
|
||||
|
||||
@@ -401,31 +411,78 @@ def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
async def sync_caldav(owner: str) -> dict:
|
||||
"""Pull CalDAV state into local DB for `owner`. Returns counts +
|
||||
errors. Loads credentials from the user's prefs; no-ops with a
|
||||
clear error if CalDAV isn't configured."""
|
||||
def _load_caldav_accounts(owner: str) -> list:
|
||||
"""Return the list of CalDAV accounts for *owner*, auto-migrating the legacy
|
||||
single-account ``caldav`` key to the new ``caldav_accounts`` list on first call.
|
||||
|
||||
The save step is best-effort: if ``_save_for_user`` is unavailable (e.g. in a
|
||||
test with a minimal prefs mock) the migrated accounts are still returned; the
|
||||
next real call will just re-run the cheap migration again.
|
||||
"""
|
||||
import uuid as _uuid
|
||||
from routes.prefs_routes import _load_for_user
|
||||
|
||||
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
|
||||
url = (cfg.get("url") or "").strip()
|
||||
user = (cfg.get("username") or "").strip()
|
||||
pw = cfg.get("password") or ""
|
||||
try:
|
||||
from src.secret_storage import decrypt
|
||||
pw = decrypt(pw)
|
||||
except Exception:
|
||||
pass
|
||||
if not (url and user and pw):
|
||||
prefs = _load_for_user(owner) or {}
|
||||
if "caldav_accounts" in prefs:
|
||||
return list(prefs["caldav_accounts"] or [])
|
||||
# Migrate legacy single-account config to the list format.
|
||||
legacy = prefs.get("caldav", {}) or {}
|
||||
if legacy.get("url"):
|
||||
accounts = [{
|
||||
"id": str(_uuid.uuid4()),
|
||||
"label": "CalDAV",
|
||||
"url": legacy["url"],
|
||||
"username": legacy.get("username", ""),
|
||||
"password": legacy.get("password", ""),
|
||||
}]
|
||||
prefs["caldav_accounts"] = accounts
|
||||
prefs.pop("caldav", None)
|
||||
try:
|
||||
from routes.prefs_routes import _save_for_user
|
||||
_save_for_user(owner, prefs)
|
||||
except (ImportError, AttributeError):
|
||||
pass # best-effort; next call re-migrates from the still-present legacy key
|
||||
return accounts
|
||||
return []
|
||||
|
||||
|
||||
async def sync_caldav(owner: str) -> dict:
|
||||
"""Pull CalDAV state into local DB for `owner` across all configured accounts.
|
||||
Returns aggregated counts + per-account errors."""
|
||||
from src.secret_storage import decrypt
|
||||
|
||||
accounts = _load_caldav_accounts(owner)
|
||||
if not accounts:
|
||||
return {
|
||||
"calendars": 0, "events": 0, "deleted": 0,
|
||||
"errors": ["CalDAV is not configured"],
|
||||
}
|
||||
try:
|
||||
url = validate_caldav_url(url)
|
||||
return await asyncio.to_thread(_sync_blocking, owner, url, user, pw)
|
||||
except ValueError as e:
|
||||
return {"calendars": 0, "events": 0, "deleted": 0, "errors": [str(e)]}
|
||||
except Exception as e:
|
||||
logger.exception("CalDAV sync raised")
|
||||
return {"calendars": 0, "events": 0, "deleted": 0, "errors": [str(e)[:200]]}
|
||||
|
||||
totals: dict = {"calendars": 0, "events": 0, "deleted": 0, "errors": []}
|
||||
for acc in accounts:
|
||||
url = (acc.get("url") or "").strip()
|
||||
user = (acc.get("username") or "").strip()
|
||||
pw = acc.get("password") or ""
|
||||
account_id = acc.get("id") or ""
|
||||
label = acc.get("label") or url or account_id
|
||||
try:
|
||||
pw = decrypt(pw)
|
||||
except Exception:
|
||||
pass
|
||||
if not (url and user and pw):
|
||||
totals["errors"].append(f"{label}: missing URL, username, or password")
|
||||
continue
|
||||
try:
|
||||
url = validate_caldav_url(url)
|
||||
result = await asyncio.to_thread(_sync_blocking, owner, url, user, pw, account_id)
|
||||
except ValueError as e:
|
||||
result = {"calendars": 0, "events": 0, "deleted": 0, "errors": [str(e)]}
|
||||
except Exception as e:
|
||||
logger.exception("CalDAV sync raised for account %s", label)
|
||||
result = {"calendars": 0, "events": 0, "deleted": 0, "errors": [str(e)[:200]]}
|
||||
totals["calendars"] += result.get("calendars", 0)
|
||||
totals["events"] += result.get("events", 0)
|
||||
totals["deleted"] += result.get("deleted", 0)
|
||||
for err in result.get("errors", []):
|
||||
totals["errors"].append(f"{label}: {err}")
|
||||
return totals
|
||||
|
||||
+49
-21
@@ -23,11 +23,10 @@ from datetime import timezone
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _stable_cal_id(remote_url: str) -> str:
|
||||
# Reuse the sync module's hashing so a local CalDAV calendar id maps back to
|
||||
# the same remote URL it was pulled from.
|
||||
def _stable_cal_id(remote_url: str, owner: str = "", account_id: str = "") -> str:
|
||||
# Reuse the sync module's hashing so owner+account_id scoping stays consistent.
|
||||
from src.caldav_sync import _stable_cal_id as _sync_id
|
||||
return _sync_id(remote_url)
|
||||
return _sync_id(remote_url, owner=owner, account_id=account_id)
|
||||
|
||||
|
||||
def build_event_ical(ev: dict) -> str:
|
||||
@@ -76,28 +75,34 @@ def build_event_ical(ev: dict) -> str:
|
||||
return cal.to_ical().decode("utf-8")
|
||||
|
||||
|
||||
def find_remote_calendar(calendars, local_cal_id: str):
|
||||
"""Find the remote calendar whose URL hashes to ``local_cal_id``, or None."""
|
||||
def find_remote_calendar(calendars, local_cal_id: str, owner: str = "", account_id: str = ""):
|
||||
"""Find the remote calendar whose URL hashes to ``local_cal_id``, or None.
|
||||
|
||||
``owner`` and ``account_id`` must match what was used when the local calendar
|
||||
id was originally computed in ``_sync_blocking`` so the hash round-trips."""
|
||||
for cal in calendars:
|
||||
try:
|
||||
if _stable_cal_id(str(cal.url)) == local_cal_id:
|
||||
if _stable_cal_id(str(cal.url), owner=owner, account_id=account_id) == local_cal_id:
|
||||
return cal
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def push_event(calendars, local_cal_id: str, ev: dict, *, delete: bool = False) -> dict:
|
||||
def push_event(calendars, local_cal_id: str, ev: dict, *, delete: bool = False,
|
||||
owner: str = "", account_id: str = "") -> dict:
|
||||
"""Create/update (or delete) ``ev`` on the matching remote calendar.
|
||||
|
||||
Returns ``{"ok": bool, ...}``. ``calendars`` is the discovered caldav
|
||||
calendar list (injected so this is unit-testable with fakes).
|
||||
``owner`` and ``account_id`` are forwarded to ``find_remote_calendar``
|
||||
so the URL hash round-trips correctly (#2765).
|
||||
"""
|
||||
uid = (ev or {}).get("uid") if isinstance(ev, dict) else None
|
||||
if not uid:
|
||||
return {"ok": False, "error": "event uid is required"}
|
||||
|
||||
remote = find_remote_calendar(calendars, local_cal_id)
|
||||
remote = find_remote_calendar(calendars, local_cal_id, owner=owner, account_id=account_id)
|
||||
if remote is None:
|
||||
return {"ok": False, "error": "remote calendar not found"}
|
||||
|
||||
@@ -136,13 +141,15 @@ def _discover_calendars(client):
|
||||
return []
|
||||
|
||||
|
||||
def _writeback_blocking(local_cal_id, ev, delete, url, username, password) -> dict:
|
||||
def _writeback_blocking(local_cal_id, ev, delete, url, username, password,
|
||||
owner="", account_id="") -> dict:
|
||||
import caldav
|
||||
client = caldav.DAVClient(url=url, username=username, password=password)
|
||||
calendars = _discover_calendars(client)
|
||||
if not calendars:
|
||||
return {"ok": False, "error": "no remote calendars discovered"}
|
||||
return push_event(calendars, local_cal_id, ev, delete=delete)
|
||||
return push_event(calendars, local_cal_id, ev, delete=delete,
|
||||
owner=owner, account_id=account_id)
|
||||
|
||||
|
||||
async def writeback_event(owner: str, calendar_source: str, calendar_id: str,
|
||||
@@ -156,24 +163,45 @@ async def writeback_event(owner: str, calendar_source: str, calendar_id: str,
|
||||
if calendar_source != "caldav":
|
||||
return {"skipped": "not a caldav calendar"}
|
||||
try:
|
||||
from routes.prefs_routes import _load_for_user
|
||||
from src.caldav_sync import _load_caldav_accounts
|
||||
from src.secret_storage import decrypt
|
||||
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
|
||||
url = (cfg.get("url") or "").strip()
|
||||
user = (cfg.get("username") or "").strip()
|
||||
# Stored encrypted by routes/calendar_routes; decrypt before use so
|
||||
# the remote sees the real password (decrypt is a no-op on legacy
|
||||
# plaintext). The pull path src/caldav_sync.py already does this.
|
||||
pw = decrypt(cfg.get("password") or "")
|
||||
if not (url and user and pw):
|
||||
from core.database import CalendarCal, SessionLocal
|
||||
|
||||
accounts = _load_caldav_accounts(owner)
|
||||
if not accounts:
|
||||
return {"skipped": "caldav not configured"}
|
||||
|
||||
# Find which account owns this calendar.
|
||||
acc = None
|
||||
if len(accounts) > 1:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
cal_row = db.query(CalendarCal).filter(CalendarCal.id == calendar_id).first()
|
||||
cal_account_id = cal_row.account_id if cal_row else None
|
||||
finally:
|
||||
db.close()
|
||||
if cal_account_id:
|
||||
acc = next((a for a in accounts if a.get("id") == cal_account_id), None)
|
||||
# Fall back to first account (covers single-account and legacy rows with
|
||||
# no account_id stamped).
|
||||
if acc is None:
|
||||
acc = accounts[0]
|
||||
|
||||
url = (acc.get("url") or "").strip()
|
||||
user = (acc.get("username") or "").strip()
|
||||
pw = decrypt(acc.get("password") or "")
|
||||
if not (url and user and pw):
|
||||
return {"skipped": "caldav account credentials incomplete"}
|
||||
from src.caldav_sync import validate_caldav_url
|
||||
try:
|
||||
url = validate_caldav_url(url)
|
||||
except ValueError as e:
|
||||
logger.warning("CalDAV write-back URL rejected: %s", e)
|
||||
return {"ok": False, "error": str(e)[:200]}
|
||||
result = await asyncio.to_thread(_writeback_blocking, calendar_id, ev, delete, url, user, pw)
|
||||
acc_id = acc.get("id") or ""
|
||||
result = await asyncio.to_thread(
|
||||
_writeback_blocking, calendar_id, ev, delete, url, user, pw, owner, acc_id
|
||||
)
|
||||
if not result.get("ok"):
|
||||
logger.warning("CalDAV write-back did not apply: %s", result.get("error") or result)
|
||||
return result
|
||||
|
||||
+7
-1
@@ -19,6 +19,12 @@ UPLOAD_DIR = os.path.join(DATA_DIR, "uploads")
|
||||
FEATURES_FILE = os.path.join(DATA_DIR, "features.json")
|
||||
SETTINGS_FILE = os.path.join(DATA_DIR, "settings.json")
|
||||
|
||||
# Agent tool output limits (single source of truth — imported by tool_execution.py,
|
||||
# tool_implementations.py, agent_tools.py, and any other module that needs them)
|
||||
MAX_OUTPUT_CHARS = 10_000 # cap for bash/python/web_search/web_fetch output
|
||||
MAX_READ_CHARS = 20_000 # cap for read_file / document preview
|
||||
MAX_DIFF_LINES = 400 # cap for edit_file unified-diff display
|
||||
|
||||
# API Configuration
|
||||
MAX_CONTEXT_MESSAGES = 90
|
||||
REQUEST_TIMEOUT = 20
|
||||
@@ -28,7 +34,7 @@ OPENAI_COMPAT_PATH = "/v1/chat/completions"
|
||||
DEFAULT_HOST = os.getenv("LLM_HOST", "localhost")
|
||||
LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()]
|
||||
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
|
||||
SEARXNG_INSTANCE = os.getenv('SEARXNG_INSTANCE', 'http://localhost:8080')
|
||||
SEARXNG_INSTANCE = os.getenv("SEARXNG_INSTANCE", "http://localhost:8080")
|
||||
|
||||
|
||||
# Cleanup configuration
|
||||
|
||||
@@ -5,6 +5,7 @@ Auto-compacts conversation history when approaching context window limits.
|
||||
Summarizes older messages via the same LLM, preserving key context.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -146,15 +147,53 @@ def _truncate_text_to_token_budget(text: str, token_budget: int) -> str:
|
||||
return text[:head_len].rstrip() + notice + "\n\n" + text[-tail_len:].lstrip()
|
||||
|
||||
|
||||
def _truncate_tool_call_args(msg: Dict[str, Any], token_budget: int) -> Dict[str, Any]:
|
||||
"""Shrink oversized assistant ``tool_calls`` arguments to fit ``token_budget``.
|
||||
|
||||
A tool-only turn persists ``content=None`` with its whole payload in
|
||||
``tool_calls[].function.arguments`` (e.g. a large create_document body), which
|
||||
the text-content truncation can't reach — so the message could stay over
|
||||
budget and the upstream call would 400. Replace each argument string that
|
||||
overflows its share of the budget with a small valid-JSON placeholder,
|
||||
preserving ``id``/``type``/``function.name`` so tool/result pairing and
|
||||
provider validation are unaffected. Returns msg unchanged when there is
|
||||
nothing oversized.
|
||||
"""
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if not isinstance(tool_calls, list) or not tool_calls:
|
||||
return msg
|
||||
# Budget left after whatever content survived (estimate_tokens counts tool
|
||||
# arguments too, so measure content alone here).
|
||||
content_tokens = estimate_tokens([{"role": msg.get("role", "assistant"), "content": msg.get("content")}])
|
||||
per_call = max(16, (max(0, token_budget - content_tokens)) // len(tool_calls))
|
||||
new_calls = []
|
||||
changed = False
|
||||
for tc in tool_calls:
|
||||
fn = tc.get("function") if isinstance(tc, dict) else None
|
||||
args = fn.get("arguments") if isinstance(fn, dict) else None
|
||||
if isinstance(args, str) and int(len(args) * 0.3) > per_call:
|
||||
new_fn = dict(fn)
|
||||
new_fn["arguments"] = json.dumps({"_truncated_for_context": len(args)})
|
||||
new_tc = dict(tc)
|
||||
new_tc["function"] = new_fn
|
||||
new_calls.append(new_tc)
|
||||
changed = True
|
||||
else:
|
||||
new_calls.append(tc)
|
||||
if not changed:
|
||||
return msg
|
||||
out = dict(msg)
|
||||
out["tool_calls"] = new_calls
|
||||
return out
|
||||
|
||||
|
||||
def _truncate_message_to_token_budget(msg: Dict[str, Any], token_budget: int) -> Dict[str, Any]:
|
||||
"""Return a copy of msg whose text content fits inside token_budget."""
|
||||
"""Return a copy of msg whose text content (and tool-call args) fit token_budget."""
|
||||
out = dict(msg)
|
||||
content = out.get("content", "")
|
||||
if isinstance(content, str):
|
||||
out["content"] = _truncate_text_to_token_budget(content, token_budget)
|
||||
return out
|
||||
|
||||
if isinstance(content, list):
|
||||
elif isinstance(content, list):
|
||||
remaining = token_budget
|
||||
new_content = []
|
||||
for item in content:
|
||||
@@ -168,7 +207,9 @@ def _truncate_message_to_token_budget(msg: Dict[str, Any], token_budget: int) ->
|
||||
new_content.append(cloned)
|
||||
remaining -= _message_text_token_estimate(truncated)
|
||||
out["content"] = new_content
|
||||
return out
|
||||
# A tool-only turn (content=None) carries its payload in tool_calls args,
|
||||
# which the branches above can't shrink — handle it so the message can fit.
|
||||
return _truncate_tool_call_args(out, token_budget)
|
||||
|
||||
|
||||
def trim_for_context(messages: List[Dict], context_length: int, reserve_tokens: int = 512) -> List[Dict]:
|
||||
|
||||
@@ -107,7 +107,7 @@ You are deciding whether a research report is comprehensive enough.
|
||||
**Current report:**
|
||||
{report}
|
||||
|
||||
**Rounds completed:** {round_num}
|
||||
**Rounds completed:** {round_num} of {max_rounds}
|
||||
|
||||
Based on the report so far, do we have enough information to answer the question \
|
||||
comprehensively? Consider:
|
||||
@@ -115,6 +115,9 @@ comprehensively? Consider:
|
||||
- Are there obvious gaps or unanswered sub-questions?
|
||||
- Is the evidence sufficient and from multiple sources?
|
||||
|
||||
If rounds completed is well below the target, prefer continuing unless the \
|
||||
report is already exhaustive.
|
||||
|
||||
Reply with ONLY "YES" or "NO" followed by a brief one-sentence reason.
|
||||
Example: "YES — The report covers all major aspects with evidence from multiple sources."
|
||||
Example: "NO — We still lack information about the economic impact."
|
||||
@@ -698,6 +701,7 @@ class DeepResearcher:
|
||||
question=question,
|
||||
report=report,
|
||||
round_num=round_num,
|
||||
max_rounds=self.max_rounds,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -100,6 +100,19 @@ INTEGRATION_PRESETS: Dict[str, Dict[str, Any]] = {
|
||||
" GET /{topic}/json?poll=1 — poll for messages"
|
||||
),
|
||||
},
|
||||
"discord_webhook": {
|
||||
"name": "Discord Webhook",
|
||||
"auth_type": "none",
|
||||
"description": (
|
||||
"Discord Incoming Webhook. Paste the full webhook URL (including the token) as the Base URL.\n"
|
||||
"To get a URL: Discord server -> Server Settings -> Integrations -> Webhooks -> New Webhook -> Copy Webhook URL.\n"
|
||||
"The secret is embedded in the URL — leave auth type as None.\n\n"
|
||||
"Use this integration as the target in Settings -> Reminders -> Webhook channel.\n"
|
||||
"Payload template examples:\n"
|
||||
" Simple: {\"content\": \"{{title}}: {{message}}\"}\n"
|
||||
" Embed: {\"embeds\": [{\"title\": \"{{title}}\", \"description\": \"{{message}}\", \"color\": 5793266}]}"
|
||||
),
|
||||
},
|
||||
"vaultwarden": {
|
||||
"name": "Vaultwarden",
|
||||
"auth_type": "header",
|
||||
|
||||
+6
-1
@@ -1690,7 +1690,12 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl
|
||||
if func.get("name"):
|
||||
_tc_acc[idx]["name"] = func["name"]
|
||||
if "arguments" in func:
|
||||
_tc_acc[idx]["arguments"] += func["arguments"]
|
||||
# Guard against a null arguments delta: `func` can be
|
||||
# {"arguments": None} (JSON null), and a raw `+= None`
|
||||
# raises TypeError that the broad except swallows,
|
||||
# silently dropping the rest of the chunk. Matches the
|
||||
# Anthropic accumulator (`partial = ... or ""`) above.
|
||||
_tc_acc[idx]["arguments"] += func["arguments"] or ""
|
||||
# Stream tool arg deltas for doc tools
|
||||
if func["arguments"] and _tc_acc[idx].get("name") in ("create_document", "update_document", "edit_document"):
|
||||
yield f'data: {json.dumps({"type": "tool_call_delta", "index": idx, "name": _tc_acc[idx]["name"], "arg_delta": func["arguments"]})}\n\n'
|
||||
|
||||
@@ -777,7 +777,7 @@ class ResearchHandler:
|
||||
llm_model=llm_model,
|
||||
llm_headers=llm_headers,
|
||||
max_rounds=max_rounds,
|
||||
min_rounds=min(3, max_rounds),
|
||||
min_rounds=max(2, max_rounds - 2),
|
||||
max_time=max_time,
|
||||
max_report_tokens=_max_report_tokens,
|
||||
extraction_timeout=_extraction_timeout,
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Shared session transcript search for UI and agent tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from core.database import ChatMessage as DBChatMessage
|
||||
from core.database import Session as DBSession
|
||||
from core.database import SessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SEARCH_ROLES = ("user", "assistant")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionSearchResult:
|
||||
message_id: str
|
||||
session_id: str
|
||||
session_name: str
|
||||
role: str
|
||||
content: str
|
||||
content_snippet: str
|
||||
timestamp: str | None
|
||||
context_before: list[dict[str, Any]]
|
||||
context_after: list[dict[str, Any]]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"message_id": self.message_id,
|
||||
"session_id": self.session_id,
|
||||
"session_name": self.session_name,
|
||||
"role": self.role,
|
||||
"content_snippet": self.content_snippet,
|
||||
"timestamp": self.timestamp,
|
||||
"context_before": self.context_before,
|
||||
"context_after": self.context_after,
|
||||
}
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if value else None
|
||||
|
||||
|
||||
def _message_to_context(msg: DBChatMessage) -> dict[str, Any]:
|
||||
return {
|
||||
"message_id": msg.id,
|
||||
"role": msg.role,
|
||||
"content": msg.content or "",
|
||||
"timestamp": _iso(msg.timestamp),
|
||||
}
|
||||
|
||||
|
||||
def _escape_like(value: str) -> str:
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
def _snippet(content: str, query: str, radius: int = 60) -> str:
|
||||
content = content or ""
|
||||
query = query or ""
|
||||
if not query:
|
||||
return content[: radius * 2]
|
||||
|
||||
idx = content.lower().find(query.lower())
|
||||
if idx == -1:
|
||||
return content[: radius * 2]
|
||||
|
||||
start = max(0, idx - radius)
|
||||
end = min(len(content), idx + len(query) + radius)
|
||||
return ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
|
||||
|
||||
|
||||
def _sanitize_fts_query(query: str) -> str | None:
|
||||
"""Convert free text into a conservative FTS5 MATCH query.
|
||||
|
||||
User input can contain FTS5 operators or punctuation that raises
|
||||
sqlite3.OperationalError. For transcript search we do not need advanced
|
||||
syntax in v1, so keep only words and balanced quoted phrases.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
for match in re.finditer(r'"([^"]+)"|[\w][\w._-]*', query, flags=re.UNICODE):
|
||||
phrase = match.group(1)
|
||||
if phrase is not None:
|
||||
phrase = phrase.strip()
|
||||
if phrase:
|
||||
parts.append('"' + phrase.replace('"', '""') + '"')
|
||||
continue
|
||||
|
||||
token = match.group(0).strip("._-")
|
||||
if not token:
|
||||
continue
|
||||
if any(ch in token for ch in "._-"):
|
||||
parts.append('"' + token.replace('"', '""') + '"')
|
||||
else:
|
||||
parts.append(token)
|
||||
|
||||
if not parts:
|
||||
return None
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _is_sqlite_session(db) -> bool:
|
||||
try:
|
||||
bind = db.get_bind()
|
||||
return getattr(getattr(bind, "dialect", None), "name", None) == "sqlite"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _has_fts_table(db) -> bool:
|
||||
if not _is_sqlite_session(db):
|
||||
return False
|
||||
try:
|
||||
row = db.execute(
|
||||
text("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chat_messages_fts' LIMIT 1")
|
||||
).first()
|
||||
return row is not None
|
||||
except Exception as e:
|
||||
logger.debug("chat_messages_fts availability check failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def _owner_filter(query, owner: str | None, include_legacy_owner: bool):
|
||||
if owner is None:
|
||||
return query.filter(DBSession.owner.is_(None))
|
||||
if not include_legacy_owner:
|
||||
return query.filter(DBSession.owner == owner)
|
||||
return query.filter((DBSession.owner == owner) | (DBSession.owner.is_(None)))
|
||||
|
||||
|
||||
def _context_for_message(db, msg: DBChatMessage, count: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if count <= 0 or not msg.timestamp:
|
||||
return [], []
|
||||
|
||||
before_rows = (
|
||||
db.query(DBChatMessage)
|
||||
.filter(
|
||||
DBChatMessage.session_id == msg.session_id,
|
||||
DBChatMessage.role.in_(SEARCH_ROLES),
|
||||
DBChatMessage.timestamp < msg.timestamp,
|
||||
)
|
||||
.order_by(DBChatMessage.timestamp.desc())
|
||||
.limit(count)
|
||||
.all()
|
||||
)
|
||||
after_rows = (
|
||||
db.query(DBChatMessage)
|
||||
.filter(
|
||||
DBChatMessage.session_id == msg.session_id,
|
||||
DBChatMessage.role.in_(SEARCH_ROLES),
|
||||
DBChatMessage.timestamp > msg.timestamp,
|
||||
)
|
||||
.order_by(DBChatMessage.timestamp.asc())
|
||||
.limit(count)
|
||||
.all()
|
||||
)
|
||||
before = [_message_to_context(row) for row in reversed(before_rows)]
|
||||
after = [_message_to_context(row) for row in after_rows]
|
||||
return before, after
|
||||
|
||||
|
||||
def _rows_to_results(db, rows: Iterable[tuple[DBChatMessage, str, str]], query: str, context_messages: int) -> list[SessionSearchResult]:
|
||||
results: list[SessionSearchResult] = []
|
||||
for msg, session_name, snippet in rows:
|
||||
before, after = _context_for_message(db, msg, context_messages)
|
||||
content = msg.content or ""
|
||||
results.append(
|
||||
SessionSearchResult(
|
||||
message_id=msg.id,
|
||||
session_id=msg.session_id,
|
||||
session_name=session_name or "Untitled",
|
||||
role=msg.role,
|
||||
content=content,
|
||||
content_snippet=snippet or _snippet(content, query),
|
||||
timestamp=_iso(msg.timestamp),
|
||||
context_before=before,
|
||||
context_after=after,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _search_like(
|
||||
db,
|
||||
query: str,
|
||||
limit: int,
|
||||
owner: str | None,
|
||||
include_archived: bool,
|
||||
context_messages: int,
|
||||
restrict_owner: bool,
|
||||
include_legacy_owner: bool,
|
||||
) -> list[SessionSearchResult]:
|
||||
safe_q = _escape_like(query)
|
||||
q = (
|
||||
db.query(DBChatMessage, DBSession.name)
|
||||
.join(DBSession, DBChatMessage.session_id == DBSession.id)
|
||||
.filter(
|
||||
DBChatMessage.content.ilike(f"%{safe_q}%", escape="\\"),
|
||||
DBChatMessage.role.in_(SEARCH_ROLES),
|
||||
)
|
||||
)
|
||||
if not include_archived:
|
||||
q = q.filter(DBSession.archived == False)
|
||||
if restrict_owner:
|
||||
q = _owner_filter(q, owner, include_legacy_owner)
|
||||
rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
|
||||
shaped = ((msg, session_name, _snippet(msg.content or "", query)) for msg, session_name in rows)
|
||||
return _rows_to_results(db, shaped, query, context_messages)
|
||||
|
||||
|
||||
def _search_fts(
|
||||
db,
|
||||
query: str,
|
||||
limit: int,
|
||||
owner: str | None,
|
||||
include_archived: bool,
|
||||
context_messages: int,
|
||||
restrict_owner: bool,
|
||||
include_legacy_owner: bool,
|
||||
) -> list[SessionSearchResult] | None:
|
||||
fts_query = _sanitize_fts_query(query)
|
||||
if not fts_query or not _has_fts_table(db):
|
||||
return None
|
||||
|
||||
archived_clause = "" if include_archived else "AND s.archived = 0"
|
||||
if not restrict_owner:
|
||||
owner_clause = ""
|
||||
elif owner is None:
|
||||
owner_clause = "AND s.owner IS NULL"
|
||||
elif not include_legacy_owner:
|
||||
owner_clause = "AND s.owner = :owner"
|
||||
else:
|
||||
owner_clause = "AND (s.owner = :owner OR s.owner IS NULL)"
|
||||
params: dict[str, Any] = {"fts_query": fts_query, "limit": limit}
|
||||
if restrict_owner and owner is not None:
|
||||
params["owner"] = owner
|
||||
|
||||
sql = text(
|
||||
f"""
|
||||
SELECT
|
||||
m.id AS message_id,
|
||||
snippet(chat_messages_fts, 0, '', '', '...', 24) AS content_snippet
|
||||
FROM chat_messages_fts
|
||||
JOIN chat_messages m ON m.id = chat_messages_fts.message_id
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE chat_messages_fts MATCH :fts_query
|
||||
{archived_clause}
|
||||
{owner_clause}
|
||||
AND m.role IN ('user', 'assistant')
|
||||
ORDER BY bm25(chat_messages_fts), m.timestamp DESC
|
||||
LIMIT :limit
|
||||
"""
|
||||
)
|
||||
|
||||
try:
|
||||
hits = db.execute(sql, params).fetchall()
|
||||
except Exception as e:
|
||||
logger.debug("FTS session search failed; falling back to LIKE: %s", e)
|
||||
return None
|
||||
|
||||
if not hits:
|
||||
return None
|
||||
|
||||
rows = []
|
||||
for hit in hits:
|
||||
message_id = hit[0]
|
||||
snippet = hit[1] or ""
|
||||
row = (
|
||||
db.query(DBChatMessage, DBSession.name)
|
||||
.join(DBSession, DBChatMessage.session_id == DBSession.id)
|
||||
.filter(DBChatMessage.id == message_id)
|
||||
.first()
|
||||
)
|
||||
if row:
|
||||
msg, session_name = row
|
||||
rows.append((msg, session_name, snippet))
|
||||
return _rows_to_results(db, rows, query, context_messages)
|
||||
|
||||
|
||||
def search_session_messages(
|
||||
query: str,
|
||||
limit: int = 20,
|
||||
owner: str | None = None,
|
||||
include_archived: bool = False,
|
||||
context_messages: int = 1,
|
||||
restrict_owner: bool = True,
|
||||
include_legacy_owner: bool = True,
|
||||
db=None,
|
||||
) -> list[SessionSearchResult]:
|
||||
"""Search session transcripts using FTS5 when available.
|
||||
|
||||
`owner=None` is deliberately treated as legacy/null-owner scope rather
|
||||
than global access.
|
||||
"""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
limit = max(1, min(int(limit or 20), 100))
|
||||
context_messages = max(0, min(int(context_messages or 0), 3))
|
||||
|
||||
owns_db = db is None
|
||||
if owns_db:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
fts_results = _search_fts(
|
||||
db,
|
||||
query,
|
||||
limit,
|
||||
owner,
|
||||
include_archived,
|
||||
context_messages,
|
||||
restrict_owner,
|
||||
include_legacy_owner,
|
||||
)
|
||||
if fts_results is not None:
|
||||
like_results = _search_like(
|
||||
db,
|
||||
query,
|
||||
limit,
|
||||
owner,
|
||||
include_archived,
|
||||
context_messages,
|
||||
restrict_owner,
|
||||
include_legacy_owner,
|
||||
)
|
||||
merged: list[SessionSearchResult] = []
|
||||
seen: set[str] = set()
|
||||
for result in [*fts_results, *like_results]:
|
||||
if result.message_id in seen:
|
||||
continue
|
||||
seen.add(result.message_id)
|
||||
merged.append(result)
|
||||
if len(merged) >= limit:
|
||||
break
|
||||
return merged
|
||||
return _search_like(
|
||||
db,
|
||||
query,
|
||||
limit,
|
||||
owner,
|
||||
include_archived,
|
||||
context_messages,
|
||||
restrict_owner,
|
||||
include_legacy_owner,
|
||||
)
|
||||
finally:
|
||||
if owns_db:
|
||||
db.close()
|
||||
+8
-1
@@ -141,10 +141,17 @@ DEFAULT_SETTINGS = {
|
||||
# library can grow beyond this; cleanup/retirement is an explicit review flow.
|
||||
"skill_max_injected": 3,
|
||||
# Reminders
|
||||
"reminder_channel": "browser", # "browser" | "email" | "ntfy"
|
||||
"reminder_channel": "browser", # "browser" | "email" | "ntfy" | "webhook"
|
||||
"reminder_llm_synthesis": False,
|
||||
"reminder_ntfy_topic": "Reminders",
|
||||
"reminder_email_to": "",
|
||||
# Generic outbound webhook channel: pick any saved Integration as the
|
||||
# target and supply a JSON payload template. Use {{title}} and {{message}}
|
||||
# as placeholders — they are JSON-escaped before substitution, so the
|
||||
# rendered string is always valid JSON. Works with Discord, Slack, Teams,
|
||||
# ntfy (JSON mode), or any service that accepts a POST with a JSON body.
|
||||
"reminder_webhook_integration_id": "",
|
||||
"reminder_webhook_payload_template": "",
|
||||
# Email triage scanner rules. Running/paused state and schedule live in
|
||||
# Tasks via the built-in `check_email_urgency` task.
|
||||
"urgent_email_prompt": (
|
||||
|
||||
+19
-26
@@ -19,6 +19,7 @@ import time
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Tuple
|
||||
|
||||
from src.tool_security import is_public_blocked_tool, owner_is_admin_or_single_user
|
||||
from src.constants import MAX_OUTPUT_CHARS, MAX_READ_CHARS, MAX_DIFF_LINES
|
||||
|
||||
# Persistent working directory for agent subprocesses.
|
||||
# Resolves to <repo_root>/data, which is the bind-mounted volume in Docker
|
||||
@@ -27,10 +28,6 @@ from src.tool_security import is_public_blocked_tool, owner_is_admin_or_single_u
|
||||
# in ephemeral container layers that are lost on the next rebuild.
|
||||
_AGENT_WORKDIR = str(pathlib.Path(__file__).parent.parent / "data")
|
||||
|
||||
MAX_OUTPUT_CHARS = 10_000
|
||||
MAX_READ_CHARS = 20_000
|
||||
MAX_DIFF_LINES = 400 # cap unified-diff size returned to the UI
|
||||
|
||||
|
||||
def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Build a unified diff of a file write for display in the chat.
|
||||
@@ -50,8 +47,8 @@ def _unified_diff(old: str, new: str, path: str) -> Optional[Dict[str, Any]]:
|
||||
fromfile=f"a/{label}", tofile=f"b/{label}",
|
||||
lineterm="",
|
||||
))
|
||||
added = sum(1 for l in diff_lines if l.startswith("+") and not l.startswith("+++"))
|
||||
removed = sum(1 for l in diff_lines if l.startswith("-") and not l.startswith("---"))
|
||||
added = sum(1 for line in diff_lines if line.startswith("+") and not line.startswith("+++"))
|
||||
removed = sum(1 for line in diff_lines if line.startswith("-") and not line.startswith("---"))
|
||||
truncated = False
|
||||
if len(diff_lines) > MAX_DIFF_LINES:
|
||||
diff_lines = diff_lines[:MAX_DIFF_LINES]
|
||||
@@ -555,7 +552,7 @@ def _parse_write_file(content: str) -> Dict:
|
||||
return {"path": lines[0].strip(), "content": lines[1] if len(lines) > 1 else ""}
|
||||
|
||||
|
||||
_MCP_ARG_PARSERS: Dict[str, callable] = {
|
||||
_MCP_ARG_PARSERS: Dict[str, Callable[[str], Dict[str, str]]] = {
|
||||
"bash": lambda c: {"command": c},
|
||||
"python": lambda c: {"code": c},
|
||||
"web_search": lambda c: {"query": c.split("\n")[0].strip()},
|
||||
@@ -660,8 +657,6 @@ async def _direct_fallback(
|
||||
are still running, with `{elapsed_s, tail}` payloads. Other tools
|
||||
ignore it.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
# Inherit env + force a sane terminal so subprocesses that touch
|
||||
# terminfo (anything calling `clear`, `tput`, `os.system("clear")`,
|
||||
# or scripts that probe $TERM) don't spam "TERM environment variable
|
||||
@@ -735,11 +730,11 @@ async def _direct_fallback(
|
||||
_stripped = content.strip()
|
||||
if _stripped.startswith("{"):
|
||||
try:
|
||||
_a = _json.loads(_stripped)
|
||||
_a = json.loads(_stripped)
|
||||
raw_path = str(_a.get("path", "")).strip()
|
||||
offset = int(_a.get("offset") or 0)
|
||||
limit = int(_a.get("limit") or 0)
|
||||
except (_json.JSONDecodeError, TypeError, ValueError):
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
pass
|
||||
try:
|
||||
path = (_resolve_tool_path_in_workspace(workspace, raw_path)
|
||||
@@ -824,8 +819,8 @@ async def _direct_fallback(
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
args = _json.loads(_s)
|
||||
except _json.JSONDecodeError:
|
||||
args = json.loads(_s)
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
else:
|
||||
args = {"pattern": _s}
|
||||
@@ -915,8 +910,8 @@ async def _direct_fallback(
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
args = _json.loads(_s)
|
||||
except _json.JSONDecodeError:
|
||||
args = json.loads(_s)
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
else:
|
||||
args = {"pattern": _s}
|
||||
@@ -965,8 +960,8 @@ async def _direct_fallback(
|
||||
_s = (content or "").strip()
|
||||
if _s.startswith("{"):
|
||||
try:
|
||||
raw_path = str(_json.loads(_s).get("path", "")).strip()
|
||||
except _json.JSONDecodeError:
|
||||
raw_path = str(json.loads(_s).get("path", "")).strip()
|
||||
except json.JSONDecodeError:
|
||||
raw_path = ""
|
||||
else:
|
||||
raw_path = _s.split("\n", 1)[0].strip()
|
||||
@@ -1016,7 +1011,7 @@ async def _direct_fallback(
|
||||
# Allow JSON-shaped args: {"query": "...", "time_filter": "day", "max_pages": 7}
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = _json.loads(raw)
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict) and "query" in parsed:
|
||||
query = str(parsed.get("query", "")).strip()
|
||||
tf = parsed.get("time_filter") or parsed.get("freshness")
|
||||
@@ -1025,7 +1020,7 @@ async def _direct_fallback(
|
||||
mp = parsed.get("max_pages")
|
||||
if isinstance(mp, int) and 1 <= mp <= 10:
|
||||
max_pages = mp
|
||||
except _json.JSONDecodeError:
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if not query:
|
||||
query = raw.split("\n")[0].strip()
|
||||
@@ -1055,7 +1050,7 @@ async def _direct_fallback(
|
||||
)
|
||||
output = text[:MAX_OUTPUT_CHARS] if len(text) > MAX_OUTPUT_CHARS else text
|
||||
if sources:
|
||||
output += "\n\n<!-- SOURCES:" + _json.dumps(sources) + " -->"
|
||||
output += "\n\n<!-- SOURCES:" + json.dumps(sources) + " -->"
|
||||
return {"output": output, "exit_code": 0}
|
||||
|
||||
if tool == "web_fetch":
|
||||
@@ -1068,10 +1063,10 @@ async def _direct_fallback(
|
||||
# Accept either a JSON arg ({"url": "..."}) or a plain URL/domain.
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = _json.loads(raw)
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict):
|
||||
url = str(parsed.get("url") or "").strip()
|
||||
except _json.JSONDecodeError:
|
||||
except json.JSONDecodeError:
|
||||
url = ""
|
||||
if not url:
|
||||
# Non-JSON (or JSON without a usable url): take the first line
|
||||
@@ -1169,8 +1164,7 @@ async def execute_tool_block(
|
||||
# Return a helpful error so the model retries with the correct format.
|
||||
if tool in ("python", "json", "xml") and content.strip().startswith("{") and content.strip().endswith("}"):
|
||||
try:
|
||||
import json as _json
|
||||
parsed = _json.loads(content.strip())
|
||||
parsed = json.loads(content.strip())
|
||||
if isinstance(parsed, dict):
|
||||
desc = f"{tool}: misformatted tool call"
|
||||
result = {
|
||||
@@ -1222,11 +1216,10 @@ async def execute_tool_block(
|
||||
# into an `ask_user` SSE event and then ENDS the turn, so the chat waits for
|
||||
# the user's selection (their choice arrives as the next message).
|
||||
if tool == "ask_user":
|
||||
import json as _json
|
||||
question, options, multi = "", [], False
|
||||
raw = (content or "").strip()
|
||||
try:
|
||||
parsed = _json.loads(raw) if raw else {}
|
||||
parsed = json.loads(raw) if raw else {}
|
||||
except (ValueError, TypeError):
|
||||
parsed = {}
|
||||
if isinstance(parsed, dict):
|
||||
|
||||
+20
-46
@@ -12,8 +12,7 @@ import os
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
MAX_OUTPUT_CHARS = 10_000
|
||||
MAX_READ_CHARS = 20_000
|
||||
from src.constants import MAX_OUTPUT_CHARS, MAX_READ_CHARS
|
||||
|
||||
|
||||
def get_mcp_manager():
|
||||
@@ -549,7 +548,7 @@ async def do_suggest_document(content: str, doc_id: str = None, owner: Optional[
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def do_search_chats(query: str, limit: int = 20, owner: str | None = None) -> Dict:
|
||||
"""Search past chat messages for the calling user's sessions only.
|
||||
"""Search past session transcripts for the calling user's sessions only.
|
||||
|
||||
Without an owner filter this used to leak EVERY user's chat history
|
||||
into the agent's `search_chats` results (v2 review HIGH-11). The
|
||||
@@ -557,63 +556,36 @@ async def do_search_chats(query: str, limit: int = 20, owner: str | None = None)
|
||||
through; legacy callers without owner pass through as before but
|
||||
will only see legacy/null-owner rows.
|
||||
"""
|
||||
from src.database import SessionLocal, ChatMessage as DBChatMessage, Session as DBSession
|
||||
# Escape LIKE wildcards in the user-supplied query so a stray % or _
|
||||
# doesn't widen the match (and to keep the response deterministic).
|
||||
safe_q = query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
q = (
|
||||
db.query(DBChatMessage, DBSession.id, DBSession.name)
|
||||
.join(DBSession, DBChatMessage.session_id == DBSession.id)
|
||||
.filter(
|
||||
DBSession.archived == False,
|
||||
DBChatMessage.content.ilike(f"%{safe_q}%", escape="\\"),
|
||||
DBChatMessage.role.in_(["user", "assistant"]),
|
||||
)
|
||||
)
|
||||
if owner is not None:
|
||||
# Restrict to this user's sessions plus legacy null-owner
|
||||
# rows (so single-user upgrades keep seeing their own data).
|
||||
q = q.filter((DBSession.owner == owner) | (DBSession.owner.is_(None)))
|
||||
rows = q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all()
|
||||
from src.session_search import search_session_messages
|
||||
|
||||
if not rows:
|
||||
results = search_session_messages(query, limit=limit, owner=owner)
|
||||
if not results:
|
||||
return {"results": f"No chats found matching \"{query}\"."}
|
||||
|
||||
# Group by session to avoid duplicate links
|
||||
seen_sessions = {}
|
||||
for msg, session_id, session_name in rows:
|
||||
if session_id not in seen_sessions:
|
||||
content = msg.content or ""
|
||||
lower_content = content.lower()
|
||||
idx = lower_content.find(query.lower())
|
||||
if idx == -1:
|
||||
snippet = content[:150]
|
||||
else:
|
||||
start = max(0, idx - 60)
|
||||
end = min(len(content), idx + len(query) + 60)
|
||||
snippet = ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
|
||||
seen_sessions[session_id] = {
|
||||
"name": session_name or "Untitled",
|
||||
"snippet": snippet,
|
||||
"role": msg.role,
|
||||
"timestamp": msg.timestamp.isoformat() if msg.timestamp else None,
|
||||
}
|
||||
for result in results:
|
||||
if result.session_id not in seen_sessions:
|
||||
seen_sessions[result.session_id] = result
|
||||
|
||||
lines = [f"Found {len(seen_sessions)} session(s) matching \"{query}\":\n"]
|
||||
for sid, info in seen_sessions.items():
|
||||
lines.append(f"- **{info['name']}** (#{sid})")
|
||||
for sid, result in seen_sessions.items():
|
||||
lines.append(f"- **{result.session_name}** (#{sid})")
|
||||
lines.append(f" Link: [Open chat](#{sid})")
|
||||
lines.append(f" > {info['snippet']}")
|
||||
lines.append(f" Match ({result.role}): {result.content_snippet}")
|
||||
if result.context_before:
|
||||
before = result.context_before[-1]
|
||||
lines.append(f" Before ({before['role']}): {before['content'][:180]}")
|
||||
if result.context_after:
|
||||
after = result.context_after[0]
|
||||
lines.append(f" After ({after['role']}): {after['content'][:180]}")
|
||||
lines.append("")
|
||||
|
||||
return {"results": "\n".join(lines)}
|
||||
except Exception as e:
|
||||
logger.error(f"search_chats failed: {e}")
|
||||
return {"error": str(e), "exit_code": 1}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1566,6 +1538,8 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
"image gen": "image_gen_enabled", "image generation": "image_gen_enabled",
|
||||
"reminder channel": "reminder_channel", "reminders": "reminder_channel",
|
||||
"ntfy topic": "reminder_ntfy_topic",
|
||||
"webhook integration": "reminder_webhook_integration_id",
|
||||
"webhook template": "reminder_webhook_payload_template", "webhook payload": "reminder_webhook_payload_template",
|
||||
"agent tool calls": "agent_max_tool_calls", "max tool calls": "agent_max_tool_calls",
|
||||
"agent timeout": "agent_stream_timeout_seconds", "stream timeout": "agent_stream_timeout_seconds",
|
||||
"token budget": "agent_input_token_budget", "input budget": "agent_input_token_budget",
|
||||
@@ -1581,7 +1555,7 @@ async def do_manage_settings(content: str, owner: Optional[str] = None) -> Dict:
|
||||
|
||||
_ENUMS = {
|
||||
"image_quality": ["low", "medium", "high"],
|
||||
"reminder_channel": ["browser", "email", "ntfy"],
|
||||
"reminder_channel": ["browser", "email", "ntfy", "webhook"],
|
||||
}
|
||||
def _coerce(value, default):
|
||||
if isinstance(default, bool):
|
||||
|
||||
+1
-1
@@ -115,7 +115,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = {
|
||||
"create_session": "Create a new chat with a name and model.",
|
||||
"list_sessions": "List all chats with their metadata (the UI calls these 'chats'). Use for 'list my chats', 'rename all my chats' (list first, then manage_session to rename each).",
|
||||
"send_to_session": "Send a message to another chat. Cross-chat communication.",
|
||||
"search_chats": "Search through chat history across all sessions.",
|
||||
"search_chats": "Search past session transcripts across chats.",
|
||||
"ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.",
|
||||
"update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.",
|
||||
"ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel <name>`. Use `open_email_reply <uid> <folder> reply` to open an email reply draft document without sending. Also switches between chat/agent modes, changes the current model, and applies/creates themes.",
|
||||
|
||||
+5
-4
@@ -354,14 +354,15 @@ def parse_tool_blocks(text: str) -> List[ToolBlock]:
|
||||
# If a code block's content is an <invoke> XML call (some models wrap
|
||||
# tool calls in ```python or ```xml fences), parse the invoke instead.
|
||||
if '<invoke' in content:
|
||||
invoked = False
|
||||
for inv in _XML_INVOKE_RE.finditer(content):
|
||||
block = _parse_xml_invoke(inv)
|
||||
if block:
|
||||
blocks.append(block)
|
||||
invoked = True
|
||||
if invoked:
|
||||
continue
|
||||
# This fenced block is <invoke> markup, not literal code. Whether or
|
||||
# not any call converted, never fall through to append the raw XML as
|
||||
# a python/bash block — e.g. a hyphenated/namespaced tool name that
|
||||
# _XML_INVOKE_RE's \w+ can't match would otherwise be executed as code.
|
||||
continue
|
||||
blocks.append(ToolBlock(tag, content))
|
||||
|
||||
# Pattern 2: [TOOL_CALL] blocks (only if no fenced blocks found)
|
||||
|
||||
+1
-1
@@ -258,7 +258,7 @@ FUNCTION_TOOL_SCHEMAS = [
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_chats",
|
||||
"description": "Search the user's past chat conversations by keyword. Use when the user asks about previous chats, past conversations, or wants to find a discussion they had before. Returns matching sessions with clickable links.",
|
||||
"description": "Search the user's past session transcripts by keyword. Use when the user asks about previous chats, past conversations, or when direct transcript evidence is better than persistent memory. Returns matching sessions with clickable links and nearby context.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
+10
-1
@@ -1957,6 +1957,7 @@
|
||||
<option value="browser">Browser notification (default)</option>
|
||||
<option value="email" id="set-reminder-channel-email-opt">Email</option>
|
||||
<option value="ntfy" id="set-reminder-channel-ntfy-opt">ntfy</option>
|
||||
<option value="webhook" id="set-reminder-channel-webhook-opt">Webhook</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="set-reminder-email-from-row" class="settings-row" style="display:none">
|
||||
@@ -1971,13 +1972,21 @@
|
||||
<label class="settings-label">ntfy topic</label>
|
||||
<input id="set-reminder-ntfy-topic" class="settings-select" type="text" placeholder="reminders" />
|
||||
</div>
|
||||
<div id="set-reminder-webhook-intg-row" class="settings-row" style="display:none">
|
||||
<label class="settings-label">Integration</label>
|
||||
<select id="set-reminder-webhook-intg" class="settings-select"></select>
|
||||
</div>
|
||||
<div id="set-reminder-webhook-template-row" class="settings-row" style="display:none;align-items:flex-start">
|
||||
<label class="settings-label" style="padding-top:6px">Payload</label>
|
||||
<textarea id="set-reminder-webhook-template" class="settings-select" rows="3" style="font-family:inherit;resize:vertical;flex:1" placeholder='{"content": "{{title}}: {{message}}"}'></textarea>
|
||||
</div>
|
||||
<div id="set-reminder-channel-hint" style="font-size:11px;opacity:0.6;"></div>
|
||||
<div style="font-size:11px;opacity:0.6;margin-top:4px;">Configure email account, ntfy server, etc. in <a href="#" id="set-reminders-open-integrations" style="color:var(--accent, var(--red));text-decoration:none;font-weight:600;">Integrations</a>.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="admin-card">
|
||||
<h2 style="display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="margin-right:1px;opacity:0.6;flex-shrink:0"><path d="M12 0L14.59 8.41L23 12L14.59 15.59L12 24L9.41 15.59L1 12L9.41 8.41Z"/></svg>AI Synthesis<span style="flex:1"></span><label class="admin-switch" title="Use the utility model to write reminder messages"><input type="checkbox" id="set-reminder-llm-toggle"><span class="admin-slider"></span></label></h2>
|
||||
<div class="admin-toggle-sub" style="margin-bottom:8px">When on, the utility model writes a short, warm one-line reminder for browser, email, AND ntfy reminders instead of just the raw note content.</div>
|
||||
<div class="admin-toggle-sub" style="margin-bottom:8px">When on, the utility model writes a short, warm one-line reminder for browser, email, ntfy, AND webhook reminders instead of just the raw note content.</div>
|
||||
</div>
|
||||
<div class="admin-card">
|
||||
<h2><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"/><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"/></svg>Public App URL</h2>
|
||||
|
||||
+19
-12
@@ -87,8 +87,11 @@ async function loadUsers() {
|
||||
<input type="number" min="0" value="${maxMsg}" data-priv="max_messages_per_day" data-user="${esc(u.username)}" style="width:70px;padding:4px 6px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--fg);font-size:12px;text-align:center;">
|
||||
</div>`;
|
||||
// Allowed models — checkbox list
|
||||
const allowedSet = new Set((u.privileges && u.privileges.allowed_models) || []);
|
||||
const allEmpty = allowedSet.size === 0;
|
||||
const allowedModels = Array.isArray(u.privileges && u.privileges.allowed_models)
|
||||
? u.privileges.allowed_models
|
||||
: [];
|
||||
const allowedSet = new Set(allowedModels);
|
||||
const modelsRestricted = !!(u.privileges && u.privileges.allowed_models_restricted);
|
||||
html += `<div style="padding:4px 0;">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;">
|
||||
<span style="font-size:12px;">Allowed models</span>
|
||||
@@ -97,7 +100,7 @@ async function loadUsers() {
|
||||
<a href="#" class="priv-models-none" data-user="${esc(u.username)}" style="font-size:10px;opacity:0.5;">None</a>
|
||||
</div>
|
||||
</div>
|
||||
<div style="font-size:10px;opacity:0.4;margin-bottom:4px;">${allEmpty ? 'All models allowed (no restrictions)' : allowedSet.size + ' model(s) allowed'}</div>
|
||||
<div style="font-size:10px;opacity:0.4;margin-bottom:4px;">${!modelsRestricted ? 'All models allowed (no restrictions)' : (allowedSet.size === 0 ? 'No models allowed' : allowedSet.size + ' model(s) allowed')}</div>
|
||||
<div class="priv-models-list" data-user="${esc(u.username)}">
|
||||
<span style="opacity:0.4;font-size:11px;">Loading models...</span>
|
||||
</div>
|
||||
@@ -119,7 +122,7 @@ async function loadUsers() {
|
||||
// Load models list on first expand
|
||||
if (!_modelsLoaded && !privPanel.classList.contains('hidden')) {
|
||||
_modelsLoaded = true;
|
||||
_loadModelsForUser(u.username, allowedSet, privPanel);
|
||||
_loadModelsForUser(u.username, allowedSet, modelsRestricted, privPanel);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -199,7 +202,7 @@ async function loadUsers() {
|
||||
} catch (e) { list.innerHTML = '<div class="admin-error">Failed to load users</div>'; }
|
||||
}
|
||||
|
||||
async function _loadModelsForUser(username, allowedSet, privPanel) {
|
||||
async function _loadModelsForUser(username, allowedSet, modelsRestricted, privPanel) {
|
||||
const listEl = privPanel.querySelector(`.priv-models-list[data-user="${username}"]`);
|
||||
if (!listEl) return;
|
||||
try {
|
||||
@@ -216,9 +219,9 @@ async function _loadModelsForUser(username, allowedSet, privPanel) {
|
||||
listEl.innerHTML = '<span style="opacity:0.4;font-size:11px;">No models available</span>';
|
||||
return;
|
||||
}
|
||||
const allEmpty = allowedSet.size === 0;
|
||||
let restricted = modelsRestricted;
|
||||
listEl.innerHTML = sortModelObjects(allModels).map(m => {
|
||||
const checked = allEmpty || allowedSet.has(m.mid) ? 'checked' : '';
|
||||
const checked = !restricted || allowedSet.has(m.mid) ? 'checked' : '';
|
||||
return `<label>
|
||||
<input type="checkbox" class="priv-model-cb" data-mid="${esc(m.mid)}" ${checked}>
|
||||
<span>${esc(m.display)}</span>
|
||||
@@ -232,14 +235,15 @@ async function _loadModelsForUser(username, allowedSet, privPanel) {
|
||||
listEl.querySelectorAll('.priv-model-cb').forEach(cb => {
|
||||
if (cb.checked) checked.push(cb.dataset.mid);
|
||||
});
|
||||
// If all are checked, send empty array (= no restrictions)
|
||||
const value = checked.length === allModels.length ? [] : checked;
|
||||
// All checked means unrestricted; zero checked means explicitly no models.
|
||||
restricted = checked.length !== allModels.length;
|
||||
const value = restricted ? checked : [];
|
||||
const hint = privPanel.querySelector('.priv-models-list[data-user]')?.previousElementSibling?.querySelector('div[style*="opacity"]');
|
||||
if (hint) hint.textContent = value.length === 0 ? 'All models allowed (no restrictions)' : value.length + ' model(s) allowed';
|
||||
if (hint) hint.textContent = !restricted ? 'All models allowed (no restrictions)' : (value.length === 0 ? 'No models allowed' : value.length + ' model(s) allowed');
|
||||
fetch(`/api/auth/users/${encodeURIComponent(username)}/privileges`, {
|
||||
method: 'PUT', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ allowed_models: value }),
|
||||
body: JSON.stringify({ allowed_models: value, allowed_models_restricted: restricted }),
|
||||
}).catch(() => {});
|
||||
}
|
||||
listEl.querySelectorAll('.priv-model-cb').forEach(cb => cb.addEventListener('change', _saveModels));
|
||||
@@ -413,6 +417,9 @@ async function loadEndpoints() {
|
||||
const justAddedClass = (_recentlyAddedEpId && String(ep.id) === _recentlyAddedEpId) ? ' adm-ep-just-added' : '';
|
||||
const category = ep.category || (_isLocalEndpoint(ep.base_url) ? 'local' : 'api');
|
||||
const kindLabel = ep.endpoint_kind && ep.endpoint_kind !== 'auto' ? ep.endpoint_kind.toUpperCase() : '';
|
||||
const keyLabel = ep.has_key
|
||||
? (ep.api_key_fingerprint ? ` (key ${esc(ep.api_key_fingerprint)})` : ' (key set)')
|
||||
: '';
|
||||
return `
|
||||
<div class="admin-user-row${ep.is_enabled ? '' : ' admin-ep-disabled'}${justAddedClass}" data-adm-ep-id="${ep.id}">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;${hasModels ? 'cursor:pointer;' : ''}padding:4px 0;" data-adm-ep-header="${ep.id}">
|
||||
@@ -430,7 +437,7 @@ async function loadEndpoints() {
|
||||
${hasModels ? '<svg class="admin-user-chevron" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="opacity:0.3;transition:transform 0.2s,opacity 0.2s;"><polyline points="6 9 12 15 18 9"/></svg>' : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="admin-ep-detail">${esc(ep.base_url)}${category === 'local' ? `<button type="button" class="admin-ep-copy-btn" data-adm-copy-url="${esc(ep.base_url)}" title="Copy URL" aria-label="Copy URL" style="background:none;border:none;padding:0 2px;margin-left:6px;cursor:pointer;color:inherit;opacity:0.45;vertical-align:-2px;line-height:1;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button>` : ''}${ep.has_key ? ' (key set)' : ''}</div>
|
||||
<div class="admin-ep-detail">${esc(ep.base_url)}${category === 'local' ? `<button type="button" class="admin-ep-copy-btn" data-adm-copy-url="${esc(ep.base_url)}" title="Copy URL" aria-label="Copy URL" style="background:none;border:none;padding:0 2px;margin-left:6px;cursor:pointer;color:inherit;opacity:0.45;vertical-align:-2px;line-height:1;"><svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg></button>` : ''}${keyLabel}</div>
|
||||
${hasModels ? `<div class="mcp-tools-panel hidden" data-adm-ep-models-panel="${ep.id}"></div>` : ''}
|
||||
</div>`;
|
||||
});
|
||||
|
||||
@@ -188,6 +188,19 @@ export function _isMetal() {
|
||||
return ['metal', 'mps', 'apple'].includes(String(_hwfitCache?.system?.backend || '').toLowerCase());
|
||||
}
|
||||
|
||||
const GEMMA4_THINKING_CHAT_TEMPLATE = `{% for message in messages %}{% if message['role'] == 'system' %}<|turn>system\n<|think|>{{ message['content'] }}<turn|>\n{% elif message['role'] == 'user' %}<|turn>user\n{{ message['content'] }}<turn|>\n{% elif message['role'] == 'assistant' %}<|turn>model\n{{ message['content'] }}<turn|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|turn>model\n<|channel>thought{% endif %}`;
|
||||
|
||||
function _isGemma4ThinkingModel(modelName) {
|
||||
const n = (modelName || '').toLowerCase();
|
||||
return n.includes('gemma-4') || n.includes('gemma4');
|
||||
}
|
||||
|
||||
function _gemma4ThinkingChatTemplateArg(modelName) {
|
||||
return _isGemma4ThinkingModel(modelName)
|
||||
? _shellQuote(GEMMA4_THINKING_CHAT_TEMPLATE)
|
||||
: '';
|
||||
}
|
||||
|
||||
/** Detect model-specific vLLM optimizations */
|
||||
function _detectModelOptimizations(modelName) {
|
||||
const n = (modelName || '').toLowerCase();
|
||||
@@ -388,6 +401,8 @@ export function _buildServeCmd(f, modelName, backend) {
|
||||
const _extraEnv = (f.extra_env ?? '').toString().replace(/\s+/g, ' ').trim();
|
||||
if (_extraEnv) cmd += _extraEnv + ' ';
|
||||
cmd += `${_vllmBin} serve ${modelName} --host 0.0.0.0 --port ${f.port || '8000'}`;
|
||||
const _gemma4ChatTemplate = _gemma4ThinkingChatTemplateArg(modelName);
|
||||
if (_gemma4ChatTemplate) cmd += ` --chat-template ${_gemma4ChatTemplate}`;
|
||||
cmd += ` --tensor-parallel-size ${f.tp || '1'}`;
|
||||
cmd += ` --max-model-len ${f.ctx || '8192'}`;
|
||||
cmd += ` --gpu-memory-utilization ${f.gpu_mem || '0.90'}`;
|
||||
@@ -418,6 +433,8 @@ export function _buildServeCmd(f, modelName, backend) {
|
||||
const _extraEnv = (f.extra_env ?? '').toString().replace(/\s+/g, ' ').trim();
|
||||
if (_extraEnv) cmd += _extraEnv + ' ';
|
||||
cmd += `${_py3Bin} -m sglang.launch_server --model-path ${modelName} --host 0.0.0.0 --port ${f.port || '30000'}`;
|
||||
const _gemma4ChatTemplate = _gemma4ThinkingChatTemplateArg(modelName);
|
||||
if (_gemma4ChatTemplate) cmd += ` --chat-template ${_gemma4ChatTemplate}`;
|
||||
if (f.tp && f.tp !== '1') cmd += ` --tp ${f.tp}`;
|
||||
if (f.ctx) cmd += ` --context-length ${f.ctx}`;
|
||||
if (f.gpu_mem && f.gpu_mem !== '0.90') cmd += ` --mem-fraction-static ${f.gpu_mem}`;
|
||||
|
||||
+169
-48
@@ -2244,6 +2244,7 @@ async function initReminderSettings() {
|
||||
const channelSel = el('set-reminder-channel');
|
||||
const emailOpt = el('set-reminder-channel-email-opt');
|
||||
const ntfyOpt = el('set-reminder-channel-ntfy-opt');
|
||||
const webhookOpt = el('set-reminder-channel-webhook-opt');
|
||||
const hint = el('set-reminder-channel-hint');
|
||||
const llmToggle = el('set-reminder-llm-toggle');
|
||||
// "Integrations" link in the channel-hint copy. Jumps to the
|
||||
@@ -2306,12 +2307,33 @@ async function initReminderSettings() {
|
||||
ntfyOpt.textContent = 'ntfy (add in Integrations first)';
|
||||
}
|
||||
|
||||
// Webhook: available whenever at least one integration with a base_url exists.
|
||||
// The user picks which integration to target and supplies a payload template.
|
||||
let allIntegrations = [];
|
||||
let webhookConfigured = false;
|
||||
try {
|
||||
const res = await fetch('/api/auth/integrations', { credentials: 'same-origin' });
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
allIntegrations = (data.integrations || []).filter(i => i.base_url && i.enabled !== false);
|
||||
webhookConfigured = allIntegrations.length > 0;
|
||||
}
|
||||
} catch (_) {}
|
||||
if (!webhookConfigured && webhookOpt) {
|
||||
webhookOpt.disabled = true;
|
||||
webhookOpt.textContent = 'Webhook (add an Integration first)';
|
||||
}
|
||||
|
||||
const emailFromRow = el('set-reminder-email-from-row');
|
||||
const emailAcctSel = el('set-reminder-email-account');
|
||||
const emailToRow = el('set-reminder-email-to-row');
|
||||
const emailToIn = el('set-reminder-email-to');
|
||||
const ntfyTopicRow = el('set-reminder-ntfy-topic-row');
|
||||
const ntfyTopicIn = el('set-reminder-ntfy-topic');
|
||||
const webhookIntgRow = el('set-reminder-webhook-intg-row');
|
||||
const webhookIntgSel = el('set-reminder-webhook-intg');
|
||||
const webhookTemplateRow = el('set-reminder-webhook-template-row');
|
||||
const webhookTemplateIn = el('set-reminder-webhook-template');
|
||||
|
||||
function populateReminderEmailAccounts(selectedId = '') {
|
||||
if (!emailAcctSel) return;
|
||||
@@ -2322,6 +2344,14 @@ async function initReminderSettings() {
|
||||
emailAcctSel.value = (selectedId && emailAccounts.some(a => a.id === selectedId)) ? selectedId : fallback;
|
||||
}
|
||||
|
||||
function populateWebhookIntegrations(selectedId = '') {
|
||||
if (!webhookIntgSel) return;
|
||||
webhookIntgSel.innerHTML = allIntegrations.length
|
||||
? allIntegrations.map(i => `<option value="${esc(i.id)}">${esc(i.name || i.id)}</option>`).join('')
|
||||
: '<option value="">No integrations configured</option>';
|
||||
if (selectedId && allIntegrations.some(i => i.id === selectedId)) webhookIntgSel.value = selectedId;
|
||||
}
|
||||
|
||||
function applyReminderChannelAvailability() {
|
||||
if (emailOpt) {
|
||||
emailOpt.disabled = !smtpConfigured;
|
||||
@@ -2331,11 +2361,16 @@ async function initReminderSettings() {
|
||||
ntfyOpt.disabled = !ntfyConfigured;
|
||||
ntfyOpt.textContent = ntfyConfigured ? 'ntfy' : 'ntfy (add in Integrations first)';
|
||||
}
|
||||
if (webhookOpt) {
|
||||
webhookOpt.disabled = !webhookConfigured;
|
||||
webhookOpt.textContent = webhookConfigured ? 'Webhook' : 'Webhook (add an Integration first)';
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshReminderChannelAvailability() {
|
||||
const currentChannel = channelSel.value || 'browser';
|
||||
const currentEmailAccount = emailAcctSel?.value || '';
|
||||
const currentWebhookIntg = webhookIntgSel?.value || '';
|
||||
try {
|
||||
const res = await fetch('/api/email/accounts', { credentials: 'same-origin' });
|
||||
if (res.ok) {
|
||||
@@ -2353,6 +2388,8 @@ async function initReminderSettings() {
|
||||
ntfyConfigured = (data.integrations || []).some(
|
||||
i => (i.preset === 'ntfy' || (i.name || '').toLowerCase() === 'ntfy') && i.enabled !== false && i.base_url
|
||||
);
|
||||
allIntegrations = (data.integrations || []).filter(i => i.base_url && i.enabled !== false);
|
||||
webhookConfigured = allIntegrations.length > 0;
|
||||
}
|
||||
} catch (_) {}
|
||||
if (!ntfyConfigured) {
|
||||
@@ -2365,8 +2402,10 @@ async function initReminderSettings() {
|
||||
|
||||
applyReminderChannelAvailability();
|
||||
populateReminderEmailAccounts(currentEmailAccount);
|
||||
populateWebhookIntegrations(currentWebhookIntg);
|
||||
if (currentChannel === 'email' && !smtpConfigured) channelSel.value = 'browser';
|
||||
else if (currentChannel === 'ntfy' && !ntfyConfigured) channelSel.value = 'browser';
|
||||
else if (currentChannel === 'webhook' && !webhookConfigured) channelSel.value = 'browser';
|
||||
else channelSel.value = currentChannel;
|
||||
if (hint) hint.textContent = CHANNEL_HINTS[channelSel.value] || '';
|
||||
syncChannelRows();
|
||||
@@ -2377,9 +2416,12 @@ async function initReminderSettings() {
|
||||
|
||||
function syncChannelRows() {
|
||||
const isEmail = channelSel.value === 'email';
|
||||
const isWebhook = channelSel.value === 'webhook';
|
||||
if (emailFromRow) emailFromRow.style.display = (isEmail && emailAccounts.length > 1) ? 'flex' : 'none';
|
||||
if (emailToRow) emailToRow.style.display = isEmail ? 'flex' : 'none';
|
||||
if (ntfyTopicRow) ntfyTopicRow.style.display = channelSel.value === 'ntfy' ? 'flex' : 'none';
|
||||
if (webhookIntgRow) webhookIntgRow.style.display = isWebhook ? 'flex' : 'none';
|
||||
if (webhookTemplateRow) webhookTemplateRow.style.display = isWebhook ? 'flex' : 'none';
|
||||
}
|
||||
|
||||
// Browser notifications fire on EVERY reminder (see
|
||||
@@ -2390,6 +2432,7 @@ async function initReminderSettings() {
|
||||
browser: 'Reminders appear as browser notifications inside Odysseus.',
|
||||
email: 'Reminders are emailed AND shown as a browser notification.',
|
||||
ntfy: 'Reminders are pushed via ntfy AND shown as a browser notification.',
|
||||
webhook: 'Reminders are POSTed to the selected integration AND shown as a browser notification. Use {{title}} and {{message}} in the payload template.',
|
||||
};
|
||||
|
||||
applyReminderChannelAvailability();
|
||||
@@ -2400,16 +2443,36 @@ async function initReminderSettings() {
|
||||
});
|
||||
}
|
||||
|
||||
// Default payload templates for known presets — auto-filled when the user
|
||||
// picks a matching integration so they don't have to write JSON from scratch.
|
||||
// Defined here (before the load block) so both the load path and the change
|
||||
// handler can reference it.
|
||||
const WEBHOOK_PRESET_TEMPLATES = {
|
||||
discord_webhook: '{"embeds": [{"title": "{{title}}", "description": "{{message}}", "color": 5793266}]}',
|
||||
};
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/auth/settings', { credentials: 'same-origin' });
|
||||
const s = await res.json();
|
||||
let savedChannel = s.reminder_channel || 'browser';
|
||||
if (savedChannel === 'email' && !smtpConfigured) savedChannel = 'browser';
|
||||
if (savedChannel === 'ntfy' && !ntfyConfigured) savedChannel = 'browser';
|
||||
if (savedChannel === 'webhook' && !webhookConfigured) savedChannel = 'browser';
|
||||
channelSel.value = savedChannel;
|
||||
llmToggle.checked = !!s.reminder_llm_synthesis;
|
||||
if (emailToIn) emailToIn.value = s.reminder_email_to || '';
|
||||
if (ntfyTopicIn) ntfyTopicIn.value = s.reminder_ntfy_topic || 'Reminders';
|
||||
populateWebhookIntegrations(s.reminder_webhook_integration_id || '');
|
||||
if (webhookTemplateIn) {
|
||||
webhookTemplateIn.value = s.reminder_webhook_payload_template || '';
|
||||
// If an integration is already selected but no template was ever saved,
|
||||
// auto-fill with the preset default so the first test works out of the box.
|
||||
if (!webhookTemplateIn.value && webhookIntgSel?.value) {
|
||||
const intg = allIntegrations.find(i => i.id === webhookIntgSel.value);
|
||||
const tpl = WEBHOOK_PRESET_TEMPLATES[intg?.preset] || '';
|
||||
if (tpl) { webhookTemplateIn.value = tpl; save({ reminder_webhook_payload_template: tpl }); }
|
||||
}
|
||||
}
|
||||
// Restore the previously-picked email account (if any), otherwise
|
||||
// default to the account flagged is_default in the integrations
|
||||
// list. Falls through to the first option if neither exists.
|
||||
@@ -2459,6 +2522,28 @@ async function initReminderSettings() {
|
||||
topicDebounce = setTimeout(() => save({ reminder_ntfy_topic: ntfyTopicIn.value.trim() || 'reminders' }), 600);
|
||||
});
|
||||
}
|
||||
if (webhookIntgSel) {
|
||||
webhookIntgSel.addEventListener('change', () => {
|
||||
save({ reminder_webhook_integration_id: webhookIntgSel.value || '' });
|
||||
// If the template is empty and we recognise the integration's preset,
|
||||
// pre-fill with a sensible default so users can test immediately.
|
||||
if (webhookTemplateIn && !webhookTemplateIn.value.trim()) {
|
||||
const intg = allIntegrations.find(i => i.id === webhookIntgSel.value);
|
||||
const tpl = WEBHOOK_PRESET_TEMPLATES[intg?.preset] || '';
|
||||
if (tpl) {
|
||||
webhookTemplateIn.value = tpl;
|
||||
save({ reminder_webhook_payload_template: tpl });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (webhookTemplateIn) {
|
||||
let templateDebounce;
|
||||
webhookTemplateIn.addEventListener('input', () => {
|
||||
clearTimeout(templateDebounce);
|
||||
templateDebounce = setTimeout(() => save({ reminder_webhook_payload_template: webhookTemplateIn.value.trim() }), 600);
|
||||
});
|
||||
}
|
||||
// Dim the whole AI Synthesis card when off (matches Vision/Utility/etc.).
|
||||
function syncSynthesisDim() {
|
||||
const card = llmToggle.closest('.admin-card');
|
||||
@@ -2495,6 +2580,11 @@ async function initReminderSettings() {
|
||||
note_id: 'test-' + Date.now(),
|
||||
title: 'Test Reminder',
|
||||
body: 'This is a test reminder to verify your settings are working.',
|
||||
channel: channelSel.value,
|
||||
...(channelSel.value === 'webhook' ? {
|
||||
webhook_integration_id: webhookIntgSel?.value || '',
|
||||
webhook_payload_template: webhookTemplateIn?.value.trim() || '',
|
||||
} : {}),
|
||||
}),
|
||||
});
|
||||
const data = await res.json();
|
||||
@@ -2505,10 +2595,15 @@ async function initReminderSettings() {
|
||||
if (channelSel.value === 'ntfy' && !data.ntfy_sent) {
|
||||
throw new Error(data.ntfy_error || 'ntfy reminder was not sent');
|
||||
}
|
||||
if (channelSel.value === 'webhook' && !data.webhook_sent) {
|
||||
const activeChannel = data.channel ? ` (server used channel: "${data.channel}")` : '';
|
||||
throw new Error((data.webhook_error || 'Webhook reminder was not sent') + activeChannel);
|
||||
}
|
||||
let status = 'Delivered via ' + channelSel.value;
|
||||
if (data.synthesis) status += ' (AI: "' + data.synthesis.slice(0, 60) + '...")';
|
||||
if (data.email_sent) status += ' — email sent';
|
||||
if (data.ntfy_sent) status += ' — ntfy sent';
|
||||
if (data.webhook_sent) status += ' — webhook sent';
|
||||
if (testMsg) { testMsg.textContent = status; testMsg.style.color = 'var(--green, #50fa7b)'; }
|
||||
// Also fire a browser notification so user can see it
|
||||
if ('Notification' in window && Notification.permission === 'granted') {
|
||||
@@ -2932,12 +3027,18 @@ async function initIntegrations() {
|
||||
let editingId = null;
|
||||
let presets = {};
|
||||
|
||||
// Toggle auth header row visibility
|
||||
// Presets where the secret is embedded in the URL — no separate key or
|
||||
// auth header is used, so hiding those fields avoids confusion.
|
||||
const URL_AUTH_PRESETS = ['discord_webhook'];
|
||||
|
||||
// Toggle auth header + key row visibility based on auth type and preset.
|
||||
function syncAuthRow() {
|
||||
const v = authTypeSel.value;
|
||||
authHeaderRow.style.display = (v === 'header' || v === 'query') ? 'flex' : 'none';
|
||||
if (v === 'query') authHeaderIn.placeholder = 'api_key';
|
||||
else authHeaderIn.placeholder = 'X-Auth-Token';
|
||||
const keyRow = keyIn?.closest('.settings-row');
|
||||
if (keyRow) keyRow.style.display = URL_AUTH_PRESETS.includes(presetSel?.value) ? 'none' : '';
|
||||
}
|
||||
authTypeSel.addEventListener('change', syncAuthRow);
|
||||
|
||||
@@ -3199,7 +3300,7 @@ async function initUnifiedIntegrations() {
|
||||
async function fetchAll() {
|
||||
const [apiRes, calRes, cardRes, contactsRes, emailAccountsRes, mcpRes, vaultRes, tokenRes, calendarsRes] = await Promise.all([
|
||||
fetch('/api/auth/integrations', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { integrations: [] }).catch(() => ({ integrations: [] })),
|
||||
fetch('/api/calendar/config', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : {}).catch(() => ({})),
|
||||
fetch('/api/calendar/config/accounts', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { accounts: [] }).catch(() => ({ accounts: [] })),
|
||||
fetch('/api/contacts/config', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : {}).catch(() => ({})),
|
||||
fetch('/api/contacts/list', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { contacts: [], count: 0 }).catch(() => ({ contacts: [], count: 0 })),
|
||||
fetch('/api/email/accounts', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { accounts: [] }).catch(() => ({ accounts: [] })),
|
||||
@@ -3213,15 +3314,9 @@ async function initUnifiedIntegrations() {
|
||||
for (const intg of (apiRes.integrations || [])) {
|
||||
items.push({ type: 'api', id: intg.id, name: intg.name || 'Unnamed', detail: intg.base_url || '', enabled: intg.enabled !== false, data: intg });
|
||||
}
|
||||
// CalDAV — one card per synced calendar collection; fall back to the
|
||||
// server-level entry if calendars haven't been synced yet.
|
||||
const caldavCals = (calendarsRes.calendars || []).filter(c => c.source === 'caldav');
|
||||
if (caldavCals.length > 0) {
|
||||
for (const cal of caldavCals) {
|
||||
items.push({ type: 'caldav', id: cal.href, name: cal.name, detail: calRes.url || 'CalDAV', enabled: true, data: { ...cal, serverData: calRes } });
|
||||
}
|
||||
} else if (calRes.url) {
|
||||
items.push({ type: 'caldav', id: '__caldav__', name: 'Calendar (CalDAV)', detail: calRes.url, enabled: true, data: calRes });
|
||||
// CalDAV — one card per account
|
||||
for (const acc of (calRes.accounts || [])) {
|
||||
items.push({ type: 'caldav', id: acc.id, name: acc.label || 'Calendar (CalDAV)', detail: acc.url, enabled: true, data: acc });
|
||||
}
|
||||
// Contacts import first, then the optional CardDAV sync account.
|
||||
const contactCount = Number(contactsRes.count || (contactsRes.contacts || []).length || 0);
|
||||
@@ -3334,14 +3429,7 @@ async function initUnifiedIntegrations() {
|
||||
const id = btn.dataset.intgId;
|
||||
try {
|
||||
if (type === 'api') await fetch(`/api/auth/integrations/${id}`, { method: 'DELETE', credentials: 'same-origin' });
|
||||
else if (type === 'caldav') {
|
||||
if (id === '__caldav__') {
|
||||
// Fallback card: server configured but never synced — clear credentials
|
||||
await fetch('/api/calendar/config', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: '', username: '', password: '' }) });
|
||||
} else {
|
||||
await fetch(`/api/calendar/calendars/${id}`, { method: 'DELETE', credentials: 'same-origin' });
|
||||
}
|
||||
}
|
||||
else if (type === 'caldav') await fetch(`/api/calendar/config/accounts/${id}`, { method: 'DELETE', credentials: 'same-origin' });
|
||||
else if (type === 'contacts') {
|
||||
await fetch('/api/contacts/clear', { method: 'DELETE', credentials: 'same-origin' });
|
||||
}
|
||||
@@ -3363,7 +3451,7 @@ async function initUnifiedIntegrations() {
|
||||
function showForm(type, editId) {
|
||||
formEl.style.display = '';
|
||||
if (type === 'api') showApiForm(editId);
|
||||
else if (type === 'caldav') showCalDavForm();
|
||||
else if (type === 'caldav') showCalDavForm(editId);
|
||||
else if (type === 'contacts' || type === 'carddav') showCardDavForm();
|
||||
else if (type === 'email') showEmailForm(editId);
|
||||
else if (type === 'mcp') showMcpForm(editId);
|
||||
@@ -3500,6 +3588,7 @@ async function initUnifiedIntegrations() {
|
||||
const _applyPreset = () => {
|
||||
const p = presets[preset.value];
|
||||
const isNtfy = preset.value === 'ntfy' || (p && (p.name || '').toLowerCase() === 'ntfy');
|
||||
const isUrlAuth = preset.value === 'discord_webhook'; // secret embedded in URL — no key/auth fields needed
|
||||
if (ntfyHint) {
|
||||
ntfyHint.style.display = isNtfy ? 'block' : 'none';
|
||||
if (isNtfy) {
|
||||
@@ -3507,8 +3596,16 @@ async function initUnifiedIntegrations() {
|
||||
}
|
||||
}
|
||||
if (url) {
|
||||
url.placeholder = isNtfy ? 'http://127.0.0.1:8091' : 'http://localhost:8080';
|
||||
url.placeholder = isNtfy ? 'http://127.0.0.1:8091' : isUrlAuth ? 'https://discord.com/api/webhooks/...' : 'http://localhost:8080';
|
||||
}
|
||||
// For presets that embed the secret in the URL, hide auth/key/header rows
|
||||
// so users aren't confused into thinking they need to fill them in.
|
||||
const keyRow = key?.closest('.settings-row');
|
||||
const authRow = auth?.closest('.settings-row');
|
||||
const headerRow = el('uf-api-header-row');
|
||||
if (keyRow) keyRow.style.display = isUrlAuth ? 'none' : '';
|
||||
if (authRow) authRow.style.display = isUrlAuth ? 'none' : '';
|
||||
if (headerRow) headerRow.style.display = isUrlAuth ? 'none' : '';
|
||||
if (!p) return;
|
||||
name.value = p.name || '';
|
||||
auth.value = p.auth_type || 'none';
|
||||
@@ -3555,33 +3652,43 @@ async function initUnifiedIntegrations() {
|
||||
});
|
||||
}
|
||||
|
||||
// ── CalDAV form ──
|
||||
async function showCalDavForm() {
|
||||
// ── CalDAV form (supports add + edit per account) ──
|
||||
async function showCalDavForm(editId) {
|
||||
const isNew = !editId || editId === 'new';
|
||||
formEl.innerHTML = `
|
||||
<div class="admin-card" style="margin-top:8px">
|
||||
<h2 style="font-size:13px">Calendar (CalDAV)</h2>
|
||||
<h2 style="font-size:13px;display:flex;align-items:center;gap:6px;"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="color:var(--accent, var(--red));flex-shrink:0;"><rect x="3" y="4" width="18" height="18" rx="2"/><line x1="16" y1="2" x2="16" y2="6"/><line x1="8" y1="2" x2="8" y2="6"/><line x1="3" y1="10" x2="21" y2="10"/></svg>${isNew ? 'Add CalDAV Calendar' : 'Edit CalDAV Calendar'}</h2>
|
||||
<div class="settings-col">
|
||||
<div class="settings-row"><label class="settings-label">Server URL</label><input id="uf-caldav-url" class="settings-input" placeholder="http://localhost:5232/user"></div>
|
||||
<div class="settings-row"><label class="settings-label">Username</label><input id="uf-caldav-user" class="settings-input"></div>
|
||||
<div class="settings-row"><label class="settings-label">Password</label><input id="uf-caldav-pass" class="settings-input" type="password"></div>
|
||||
<div class="settings-row" style="margin-top:4px"><button class="admin-btn-sm" id="uf-caldav-save">Save</button><button class="admin-btn-sm" id="uf-caldav-test" style="opacity:0.7">Test</button><button class="admin-btn-sm" id="uf-caldav-cancel" style="opacity:0.7">Cancel</button><span id="uf-caldav-msg" style="font-size:11px"></span></div>
|
||||
<div class="settings-row"><label class="settings-label">Label</label><input id="uf-caldav-label" class="settings-input" placeholder="e.g. Work, Personal"></div>
|
||||
<div class="settings-row"><label class="settings-label">Server URL</label><input id="uf-caldav-url" class="settings-input" placeholder="https://www.google.com/calendar/dav/you@gmail.com/user/"></div>
|
||||
<div class="settings-row"><label class="settings-label">Username</label><input id="uf-caldav-user" class="settings-input" placeholder="you@example.com"></div>
|
||||
<div class="settings-row"><label class="settings-label">Password</label><input id="uf-caldav-pass" class="settings-input" type="password" placeholder="${isNew ? '' : 'Leave blank to keep existing'}"></div>
|
||||
<div class="settings-row" style="margin-top:4px"><button class="admin-btn-sm" id="uf-caldav-save">Save</button><button class="admin-btn-sm" id="uf-caldav-test" style="opacity:0.7">Test</button><button class="admin-btn-sm" id="uf-caldav-cancel" style="opacity:0.7">Cancel</button><span id="uf-caldav-msg" style="font-size:11px;margin-left:6px"></span></div>
|
||||
</div>
|
||||
</div>`;
|
||||
try {
|
||||
const r = await fetch('/api/calendar/config', { credentials: 'same-origin' }); const d = await r.json();
|
||||
el('uf-caldav-url').value = d.url || ''; el('uf-caldav-user').value = d.username || '';
|
||||
} catch (_) {}
|
||||
|
||||
if (!isNew) {
|
||||
try {
|
||||
const r = await fetch('/api/calendar/config/accounts', { credentials: 'same-origin' });
|
||||
const d = await r.json();
|
||||
const acc = (d.accounts || []).find(a => a.id === editId);
|
||||
if (acc) {
|
||||
el('uf-caldav-label').value = acc.label || '';
|
||||
el('uf-caldav-url').value = acc.url || '';
|
||||
el('uf-caldav-user').value = acc.username || '';
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
el('uf-caldav-cancel').addEventListener('click', () => { formEl.style.display = 'none'; });
|
||||
|
||||
// Run a PROPFIND with the form's current url+user+pass. Used by
|
||||
// both the Test button (visible result only) and by Save (refuse
|
||||
// to persist a broken config). Returns the parsed {ok, error?}.
|
||||
const _runCalDavTest = async () => {
|
||||
const body = {
|
||||
url: el('uf-caldav-url').value.trim(),
|
||||
username: el('uf-caldav-user').value.trim(),
|
||||
password: el('uf-caldav-pass').value,
|
||||
};
|
||||
if (!isNew && !body.password) body.account_id = editId;
|
||||
try {
|
||||
const r = await fetch('/api/calendar/test', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
@@ -3593,6 +3700,7 @@ async function initUnifiedIntegrations() {
|
||||
return { ok: false, error: 'Network error: ' + e.message };
|
||||
}
|
||||
};
|
||||
|
||||
const _setCalDavMsg = (text, ok) => {
|
||||
const msg = el('uf-caldav-msg');
|
||||
msg.textContent = text;
|
||||
@@ -3600,10 +3708,6 @@ async function initUnifiedIntegrations() {
|
||||
};
|
||||
|
||||
el('uf-caldav-save').addEventListener('click', async () => {
|
||||
// Pre-validate by hitting the server with the same PROPFIND the
|
||||
// Test button uses. If the CalDAV server rejects the creds/URL
|
||||
// we won't persist garbage — the user gets the actual error
|
||||
// (HTTP 401, "Not found", "Connection refused", etc.) in red.
|
||||
_setCalDavMsg('Testing…', true);
|
||||
el('uf-caldav-msg').style.color = '';
|
||||
const d = await _runCalDavTest();
|
||||
@@ -3612,15 +3716,31 @@ async function initUnifiedIntegrations() {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await fetch('/api/calendar/config', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: el('uf-caldav-url').value,
|
||||
username: el('uf-caldav-user').value,
|
||||
password: el('uf-caldav-pass').value,
|
||||
}),
|
||||
});
|
||||
const payload = {
|
||||
label: el('uf-caldav-label').value.trim(),
|
||||
url: el('uf-caldav-url').value.trim(),
|
||||
username: el('uf-caldav-user').value.trim(),
|
||||
password: el('uf-caldav-pass').value,
|
||||
};
|
||||
let resp;
|
||||
if (isNew) {
|
||||
resp = await fetch('/api/calendar/config/accounts', {
|
||||
method: 'POST', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
} else {
|
||||
resp = await fetch(`/api/calendar/config/accounts/${editId}`, {
|
||||
method: 'PUT', credentials: 'same-origin',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
}
|
||||
if (!resp.ok) {
|
||||
const err = await resp.json().catch(() => ({}));
|
||||
_setCalDavMsg(err.detail || 'Save failed', false);
|
||||
return;
|
||||
}
|
||||
_setCalDavMsg('Saved', true);
|
||||
formEl.style.display = 'none';
|
||||
await renderList();
|
||||
@@ -3629,6 +3749,7 @@ async function initUnifiedIntegrations() {
|
||||
_setCalDavMsg('Save failed', false);
|
||||
}
|
||||
});
|
||||
|
||||
el('uf-caldav-test').addEventListener('click', async () => {
|
||||
_setCalDavMsg('Testing…', true);
|
||||
el('uf-caldav-msg').style.color = '';
|
||||
|
||||
@@ -88,6 +88,7 @@ def test_sync_caldav_decrypts_stored_password_and_validates_url(monkeypatch):
|
||||
"_resolve_caldav_host_ips",
|
||||
lambda host: [ipaddress.ip_address("93.184.216.34")],
|
||||
)
|
||||
saved = {}
|
||||
prefs_mod = types.ModuleType("routes.prefs_routes")
|
||||
prefs_mod._load_for_user = lambda owner: {
|
||||
"caldav": {
|
||||
@@ -96,6 +97,7 @@ def test_sync_caldav_decrypts_stored_password_and_validates_url(monkeypatch):
|
||||
"password": "enc:stored",
|
||||
}
|
||||
}
|
||||
prefs_mod._save_for_user = lambda owner, prefs: saved.update({"owner": owner, "prefs": prefs})
|
||||
monkeypatch.setitem(sys.modules, "routes.prefs_routes", prefs_mod)
|
||||
|
||||
secret_mod = types.ModuleType("src.secret_storage")
|
||||
@@ -104,7 +106,7 @@ def test_sync_caldav_decrypts_stored_password_and_validates_url(monkeypatch):
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_sync_blocking(owner, url, username, password):
|
||||
def fake_sync_blocking(owner, url, username, password, account_id=""):
|
||||
captured.update(
|
||||
{
|
||||
"owner": owner,
|
||||
@@ -136,7 +138,7 @@ def test_calendar_routes_use_hardened_caldav_client_and_secret_storage():
|
||||
text = Path("routes/calendar_routes.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "validate_caldav_url(body.get(\"url\", \"\"))" in text
|
||||
assert "cfg[\"password\"] = encrypt(body[\"password\"])" in text
|
||||
assert "encrypt(body[\"password\"])" in text
|
||||
assert "pw = decrypt(pw)" in text
|
||||
assert "follow_redirects=False, trust_env=False" in text
|
||||
assert "Redirects are not followed for CalDAV safety" in text
|
||||
|
||||
@@ -151,7 +151,8 @@ def test_writeback_validates_saved_url_before_remote_call(monkeypatch):
|
||||
captured["validated_url"] = url
|
||||
return "https://dav.example.com/calendars/home"
|
||||
|
||||
def fake_writeback_blocking(local_cal_id, ev, delete, url, username, password):
|
||||
def fake_writeback_blocking(local_cal_id, ev, delete, url, username, password,
|
||||
owner="", account_id=""):
|
||||
captured.update(
|
||||
{
|
||||
"local_cal_id": local_cal_id,
|
||||
@@ -207,7 +208,8 @@ def test_writeback_rejects_unsafe_saved_url_before_remote_call(monkeypatch):
|
||||
def fake_validate(_url):
|
||||
raise ValueError("CalDAV URL host is not allowed")
|
||||
|
||||
def fake_writeback_blocking(*_args, **_kwargs):
|
||||
def fake_writeback_blocking(local_cal_id, ev, delete, url, username, password,
|
||||
owner="", account_id=""):
|
||||
nonlocal called
|
||||
called = True
|
||||
return {"ok": True}
|
||||
|
||||
@@ -1,5 +1,67 @@
|
||||
import pytest
|
||||
from routes.chat_helpers import clean_thinking_for_save, needs_auto_name
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routes.chat_helpers import _enforce_chat_privileges, clean_thinking_for_save, needs_auto_name
|
||||
|
||||
|
||||
class _AuthManager:
|
||||
def __init__(self, privileges):
|
||||
self._privileges = privileges
|
||||
|
||||
def get_privileges(self, username):
|
||||
assert username == "alice"
|
||||
return self._privileges
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, privileges):
|
||||
self.app = type("App", (), {})()
|
||||
self.app.state = type("State", (), {"auth_manager": _AuthManager(privileges)})()
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
|
||||
def test_allowed_models_legacy_empty_list_remains_unrestricted(monkeypatch):
|
||||
monkeypatch.setattr("routes.chat_helpers.get_current_user", lambda request: "alice")
|
||||
|
||||
_enforce_chat_privileges(
|
||||
_Request({"allowed_models": [], "max_messages_per_day": 0}),
|
||||
_Session("provider/model-a"),
|
||||
)
|
||||
|
||||
|
||||
def test_allowed_models_explicit_empty_restricted_list_blocks_all_models(monkeypatch):
|
||||
monkeypatch.setattr("routes.chat_helpers.get_current_user", lambda request: "alice")
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
_enforce_chat_privileges(
|
||||
_Request({
|
||||
"allowed_models": [],
|
||||
"allowed_models_restricted": True,
|
||||
"max_messages_per_day": 0,
|
||||
}),
|
||||
_Session("provider/model-a"),
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 403
|
||||
assert "provider/model-a" in exc.value.detail
|
||||
|
||||
|
||||
def test_allowed_models_nonempty_list_still_restricts_without_new_flag(monkeypatch):
|
||||
monkeypatch.setattr("routes.chat_helpers.get_current_user", lambda request: "alice")
|
||||
|
||||
_enforce_chat_privileges(
|
||||
_Request({"allowed_models": ["provider/model-a"], "max_messages_per_day": 0}),
|
||||
_Session("provider/model-a"),
|
||||
)
|
||||
with pytest.raises(HTTPException):
|
||||
_enforce_chat_privileges(
|
||||
_Request({"allowed_models": ["provider/model-a"], "max_messages_per_day": 0}),
|
||||
_Session("provider/model-b"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,expected", [
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Issue #2947 — _truncate_message_to_token_budget must shrink oversized tool_calls
|
||||
arguments, not just text content.
|
||||
|
||||
A tool-only assistant turn persists content=None with its whole payload in
|
||||
tool_calls[].function.arguments. The text-content truncation can't reach it, so
|
||||
trim_for_context's last-resort message shrink left the message over budget and the
|
||||
upstream call 400'd. This pins that oversized args are bounded (so the message
|
||||
fits) while id/type/function.name are preserved, and that small args / plain text
|
||||
are untouched.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
for mod in [
|
||||
'sqlalchemy', 'sqlalchemy.orm', 'sqlalchemy.ext', 'sqlalchemy.ext.declarative',
|
||||
'sqlalchemy.ext.hybrid', 'sqlalchemy.sql', 'sqlalchemy.sql.expression',
|
||||
'src.database',
|
||||
'core.models', 'core.database',
|
||||
]:
|
||||
if mod not in sys.modules:
|
||||
sys.modules[mod] = MagicMock()
|
||||
|
||||
from src.context_compactor import _truncate_message_to_token_budget # noqa: E402
|
||||
from src.model_context import estimate_tokens # noqa: E402
|
||||
|
||||
|
||||
def _tool_msg(arg_len):
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{
|
||||
"id": "c1", "type": "function",
|
||||
"function": {"name": "create_document", "arguments": "x" * arg_len},
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def test_oversized_tool_call_args_are_truncated_to_fit_budget():
|
||||
budget = 200
|
||||
out = _truncate_message_to_token_budget(_tool_msg(40000), budget)
|
||||
# The message now fits the budget (before the fix it stayed ~12k tokens).
|
||||
assert estimate_tokens([out]) <= budget, estimate_tokens([out])
|
||||
tc = out["tool_calls"][0]
|
||||
# Structure preserved so tool/result pairing + provider validation still hold.
|
||||
assert tc["id"] == "c1" and tc["type"] == "function"
|
||||
assert tc["function"]["name"] == "create_document"
|
||||
# Arguments remain valid JSON, just bounded.
|
||||
parsed = json.loads(tc["function"]["arguments"])
|
||||
assert parsed.get("_truncated_for_context") == 40000
|
||||
|
||||
|
||||
def test_small_tool_call_args_are_left_untouched():
|
||||
out = _truncate_message_to_token_budget(_tool_msg(20), 500)
|
||||
assert out["tool_calls"][0]["function"]["arguments"] == "x" * 20
|
||||
|
||||
|
||||
def test_plain_text_content_still_truncates():
|
||||
out = _truncate_message_to_token_budget({"role": "user", "content": "y" * 40000}, 200)
|
||||
assert len(out["content"]) < 2000 # truncated, not left at 40k
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Regression coverage for issue #2929: Gemma 4 thinking chat template.
|
||||
|
||||
Gemma 4 thinking models need the `<|think|>` control token in the system
|
||||
instruction, while the generation prompt should start the model turn with the
|
||||
thought channel. Cookbook serve commands should supply that template for
|
||||
OpenAI-compatible servers instead of relying on a generic chat template that
|
||||
cannot toggle thinking mode.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
SRC = Path(__file__).resolve().parent.parent / "static/js/cookbook.js"
|
||||
|
||||
|
||||
def test_gemma4_thinking_template_uses_google_documented_thinking_placement():
|
||||
text = SRC.read_text(encoding="utf-8")
|
||||
|
||||
assert "GEMMA4_THINKING_CHAT_TEMPLATE" in text
|
||||
assert "<|turn>system\\n<|think|>{{ message['content'] }}<turn|>" in text
|
||||
assert "<|turn>user" in text
|
||||
assert "<|turn>model" in text
|
||||
assert "<|turn>model\\n<|channel>thought" in text
|
||||
assert "<|turn>model\\n<|think|><|channel>thought" not in text
|
||||
|
||||
|
||||
def test_vllm_and_sglang_apply_gemma4_thinking_template():
|
||||
text = SRC.read_text(encoding="utf-8")
|
||||
|
||||
assert "function _isGemma4ThinkingModel" in text
|
||||
assert "const _gemma4ChatTemplate" in text
|
||||
assert "if (_gemma4ChatTemplate) cmd += ` --chat-template ${_gemma4ChatTemplate}`;" in text
|
||||
assert text.count("_gemma4ThinkingChatTemplateArg(modelName)") >= 2
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Issue #2925 — a fenced ```python/```bash block wrapping an <invoke> call that
|
||||
can't be converted (e.g. a hyphenated/namespaced tool name that _XML_INVOKE_RE's
|
||||
\\w+ won't match, or an unknown tool) must NOT fall through and ship the raw XML
|
||||
to the code executor as if it were python/bash.
|
||||
"""
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
for mod in ['src.agent_tools', 'src.tool_parsing', 'src.tool_schemas', 'src.tool_execution']:
|
||||
sys.modules.pop(mod, None)
|
||||
for mod in [
|
||||
'sqlalchemy', 'sqlalchemy.orm', 'sqlalchemy.ext', 'sqlalchemy.ext.declarative',
|
||||
'sqlalchemy.ext.hybrid', 'sqlalchemy.sql', 'sqlalchemy.sql.expression',
|
||||
'src.database', 'core.models', 'core.database', 'core.auth'
|
||||
]:
|
||||
if mod not in sys.modules:
|
||||
sys.modules[mod] = MagicMock()
|
||||
|
||||
import src.agent_tools # noqa: E402, F401
|
||||
from src.tool_parsing import parse_tool_blocks # noqa: E402
|
||||
|
||||
|
||||
def test_unconvertible_invoke_in_fence_is_not_executed_as_code():
|
||||
text = '```python\n<invoke name="foo-bar">\n<parameter name="x">1</parameter>\n</invoke>\n```'
|
||||
blocks = parse_tool_blocks(text)
|
||||
# the hyphenated name can't match _XML_INVOKE_RE, so nothing converts —
|
||||
# the raw XML must not be appended as a python/bash code block.
|
||||
assert not any(
|
||||
b.tool_type in ("python", "bash") and "<invoke" in b.content for b in blocks
|
||||
), blocks
|
||||
|
||||
|
||||
def test_plain_fenced_python_block_still_parses_as_code():
|
||||
# No regression: an ordinary fenced python block (no <invoke>) still works.
|
||||
blocks = parse_tool_blocks('```python\nprint("hi")\n```')
|
||||
assert any(b.tool_type == "python" and 'print("hi")' in b.content for b in blocks), blocks
|
||||
@@ -149,3 +149,23 @@ def test_sparse_integer_indices_then_null_do_not_collide(monkeypatch):
|
||||
events = _drive(monkeypatch, lines)
|
||||
calls = next(e["calls"] for e in events if e.get("type") == "tool_calls")
|
||||
assert sorted(c["name"] for c in calls) == ["f0", "f2", "fn"], f"collision: {calls}"
|
||||
|
||||
|
||||
def test_null_arguments_delta_does_not_drop_sibling_calls(monkeypatch):
|
||||
# A gateway can emit a tool_call delta whose `arguments` is JSON null. The
|
||||
# accumulator did `"" += None`, raising TypeError caught by the broad except
|
||||
# that wraps the whole chunk — so it abandoned the rest of the tool_calls
|
||||
# loop, silently dropping every LATER call in the same delta. Here the first
|
||||
# call has arguments: null; the second (same delta) must still survive.
|
||||
lines = [
|
||||
_sse({"tool_calls": [
|
||||
{"index": 0, "id": "a", "type": "function",
|
||||
"function": {"name": "first", "arguments": None}},
|
||||
{"index": 1, "id": "b", "type": "function",
|
||||
"function": {"name": "second", "arguments": "{}"}},
|
||||
]}),
|
||||
"data: [DONE]",
|
||||
]
|
||||
events = _drive(monkeypatch, lines, model="gpt-4o-test")
|
||||
calls = next(e["calls"] for e in events if e.get("type") == "tool_calls")
|
||||
assert sorted(c["name"] for c in calls) == ["first", "second"], calls
|
||||
|
||||
@@ -37,6 +37,7 @@ from routes.model_routes import (
|
||||
_curate_models,
|
||||
_visible_models,
|
||||
_normalize_model_ids,
|
||||
_api_key_fingerprint,
|
||||
_is_chat_model,
|
||||
_classify_endpoint,
|
||||
_effective_endpoint_kind,
|
||||
@@ -754,6 +755,16 @@ def test_visible_models_handles_malformed_strings():
|
||||
assert _visible_models("only-cached", None, None) == ["only-cached"]
|
||||
|
||||
|
||||
def test_api_key_fingerprint_is_stable_and_non_secret():
|
||||
fp_one = _api_key_fingerprint("key-one")
|
||||
|
||||
assert _api_key_fingerprint("") == ""
|
||||
assert fp_one == _api_key_fingerprint(" key-one ")
|
||||
assert fp_one != _api_key_fingerprint("key-two")
|
||||
assert len(fp_one) == 8
|
||||
assert "key-one" not in fp_one
|
||||
|
||||
|
||||
def _create_form_kwargs(**overrides):
|
||||
"""Defaults for every Form() param create_model_endpoint reads directly.
|
||||
|
||||
@@ -791,6 +802,29 @@ def _patch_create_deps(monkeypatch, db):
|
||||
monkeypatch.setattr(auth_helpers, "get_current_user", lambda req: None)
|
||||
|
||||
|
||||
def test_list_model_endpoints_returns_key_fingerprint(monkeypatch):
|
||||
endpoint_with_key = _make_endpoint(
|
||||
api_key="key-one",
|
||||
cached_models=json.dumps(["m1"]),
|
||||
)
|
||||
endpoint_without_key = _make_endpoint(
|
||||
id="ep2",
|
||||
api_key=None,
|
||||
cached_models=json.dumps(["m2"]),
|
||||
)
|
||||
db = _PinnedFakeDb([endpoint_with_key, endpoint_without_key])
|
||||
monkeypatch.setattr(model_routes, "SessionLocal", lambda: db)
|
||||
monkeypatch.setattr(model_routes, "require_admin", lambda request: None)
|
||||
endpoint = _get_route("/api/model-endpoints", "GET")
|
||||
|
||||
result = endpoint(_PinnedFakeRequest())
|
||||
|
||||
assert result[0]["has_key"] is True
|
||||
assert result[0]["api_key_fingerprint"] == _api_key_fingerprint("key-one")
|
||||
assert result[1]["has_key"] is False
|
||||
assert result[1]["api_key_fingerprint"] == ""
|
||||
|
||||
|
||||
def test_post_creates_endpoint_with_pinned_models(monkeypatch):
|
||||
db = _PinnedFakeDb([]) # no existing row → fresh create path
|
||||
_patch_create_deps(monkeypatch, db)
|
||||
@@ -856,6 +890,53 @@ def test_post_dedupe_existing_does_not_clobber_pinned_when_omitted(monkeypatch):
|
||||
assert json.loads(existing.pinned_models) == ["keep-me"]
|
||||
assert result["pinned_models"] == ["keep-me"]
|
||||
assert db.committed == 0 # nothing to persist
|
||||
|
||||
|
||||
def test_post_same_base_url_different_api_key_creates_distinct_endpoint(monkeypatch):
|
||||
existing = _make_endpoint(
|
||||
base_url="https://api.example.test/v1",
|
||||
api_key="key-one",
|
||||
)
|
||||
db = _PinnedFakeDb([existing])
|
||||
_patch_create_deps(monkeypatch, db)
|
||||
create = _get_route("/api/model-endpoints", "POST")
|
||||
|
||||
result = create(
|
||||
_PinnedFakeRequest(),
|
||||
base_url="https://api.example.test/v1",
|
||||
**_create_form_kwargs(api_key="key-two"),
|
||||
)
|
||||
|
||||
assert result.get("existing") is not True
|
||||
assert result["has_key"] is True
|
||||
assert result["api_key_fingerprint"] == _api_key_fingerprint("key-two")
|
||||
assert len(db.added) == 1
|
||||
assert db.added[0].base_url == "https://api.example.test/v1"
|
||||
assert db.added[0].api_key == "key-two"
|
||||
|
||||
|
||||
def test_post_same_base_url_same_api_key_still_dedupes(monkeypatch):
|
||||
existing = _make_endpoint(
|
||||
base_url="https://api.example.test/v1",
|
||||
api_key="key-one",
|
||||
)
|
||||
db = _PinnedFakeDb([existing])
|
||||
_patch_create_deps(monkeypatch, db)
|
||||
create = _get_route("/api/model-endpoints", "POST")
|
||||
|
||||
result = create(
|
||||
_PinnedFakeRequest(),
|
||||
base_url="https://api.example.test/v1",
|
||||
**_create_form_kwargs(api_key="key-one"),
|
||||
)
|
||||
|
||||
assert result["existing"] is True
|
||||
assert result["id"] == existing.id
|
||||
assert result["has_key"] is True
|
||||
assert result["api_key_fingerprint"] == _api_key_fingerprint("key-one")
|
||||
assert db.added == []
|
||||
|
||||
|
||||
class _RouteQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = list(rows)
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_unauthenticated_still_403(monkeypatch):
|
||||
sm = SimpleNamespace(sessions={"ghost": SimpleNamespace(owner=None)})
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
SR._verify_session_owner(_req(api_token=False, current_user=None), "ghost", sm)
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.status_code == 401
|
||||
|
||||
|
||||
# --- manager layer: delete_session clears memory-only ghosts ---------------
|
||||
|
||||
@@ -136,4 +136,4 @@ def test_unauthenticated_caller_rejected(monkeypatch):
|
||||
req = _req(api_token=False, current_user=None)
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
SR._verify_session_owner(req, "sid")
|
||||
assert exc.value.status_code == 403
|
||||
assert exc.value.status_code == 401
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
from datetime import datetime, timedelta
|
||||
import asyncio
|
||||
import sqlite3
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from core.database import Base
|
||||
from core.database import ChatMessage as DbChatMessage
|
||||
from core.database import Session as DbSession
|
||||
from src.session_search import SessionSearchResult, search_session_messages
|
||||
|
||||
|
||||
def _db(with_fts=True):
|
||||
engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
db = sessionmaker(bind=engine)()
|
||||
if with_fts:
|
||||
db.connection().exec_driver_sql(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE chat_messages_fts USING fts5(
|
||||
content,
|
||||
message_id UNINDEXED,
|
||||
session_id UNINDEXED,
|
||||
role UNINDEXED
|
||||
)
|
||||
"""
|
||||
)
|
||||
return db
|
||||
|
||||
|
||||
def _add_session(db, sid, owner="alice", archived=False, name=None):
|
||||
db.add(
|
||||
DbSession(
|
||||
id=sid,
|
||||
name=name or sid,
|
||||
endpoint_url="http://example.test",
|
||||
model="test-model",
|
||||
owner=owner,
|
||||
archived=archived,
|
||||
message_count=0,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _add_message(db, sid, mid, role, content, when):
|
||||
db.add(DbChatMessage(id=mid, session_id=sid, role=role, content=content, timestamp=when))
|
||||
if _has_fts(db):
|
||||
db.connection().exec_driver_sql(
|
||||
"INSERT INTO chat_messages_fts(content, message_id, session_id, role) VALUES (?, ?, ?, ?)",
|
||||
(content, mid, sid, role),
|
||||
)
|
||||
|
||||
|
||||
def _has_fts(db):
|
||||
return (
|
||||
db.connection()
|
||||
.exec_driver_sql("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chat_messages_fts'")
|
||||
.first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def test_session_search_uses_fts_and_returns_context():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "s1", owner="alice", name="Jazz planning")
|
||||
_add_message(db, "s1", "m1", "user", "Before context about music", base)
|
||||
_add_message(db, "s1", "m2", "assistant", "We talked about modal jazz theory", base + timedelta(minutes=1))
|
||||
_add_message(db, "s1", "m3", "user", "After context about tasks", base + timedelta(minutes=2))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("modal jazz", owner="alice", db=db)
|
||||
|
||||
assert [r.message_id for r in results] == ["m2"]
|
||||
assert results[0].session_name == "Jazz planning"
|
||||
assert results[0].context_before[0]["message_id"] == "m1"
|
||||
assert results[0].context_after[0]["message_id"] == "m3"
|
||||
assert "modal" in results[0].content_snippet.lower()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_escapes_like_wildcards_in_fallback():
|
||||
db = _db(with_fts=False)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "s1", owner="alice")
|
||||
_add_message(db, "s1", "literal", "user", "The literal token is foo_bar.", base)
|
||||
_add_message(db, "s1", "wild", "user", "The wildcard-looking token is fooXbar.", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("foo_bar", owner="alice", db=db)
|
||||
|
||||
assert [r.message_id for r in results] == ["literal"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_owner_scope_includes_legacy_and_excludes_other_users():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "alice", owner="alice")
|
||||
_add_session(db, "legacy", owner=None)
|
||||
_add_session(db, "bob", owner="bob")
|
||||
_add_message(db, "alice", "m-alice", "user", "shared recall target", base)
|
||||
_add_message(db, "legacy", "m-legacy", "user", "shared recall target", base + timedelta(minutes=1))
|
||||
_add_message(db, "bob", "m-bob", "user", "shared recall target", base + timedelta(minutes=2))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("shared recall target", owner="alice", db=db)
|
||||
|
||||
assert {r.message_id for r in results} == {"m-alice", "m-legacy"}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_can_exclude_legacy_rows_for_authenticated_ui_scope():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "alice", owner="alice")
|
||||
_add_session(db, "legacy", owner=None)
|
||||
_add_message(db, "alice", "m-alice", "user", "exact owner target", base)
|
||||
_add_message(db, "legacy", "m-legacy", "user", "exact owner target", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages(
|
||||
"exact owner target",
|
||||
owner="alice",
|
||||
include_legacy_owner=False,
|
||||
db=db,
|
||||
)
|
||||
|
||||
assert [r.message_id for r in results] == ["m-alice"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_ownerless_call_only_sees_legacy_rows():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "alice", owner="alice")
|
||||
_add_session(db, "legacy", owner=None)
|
||||
_add_message(db, "alice", "m-alice", "user", "ownerless search target", base)
|
||||
_add_message(db, "legacy", "m-legacy", "user", "ownerless search target", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("ownerless search target", owner=None, db=db)
|
||||
|
||||
assert [r.message_id for r in results] == ["m-legacy"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_falls_back_to_like_when_fts_has_no_substring_hits():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "s1", owner="alice")
|
||||
_add_message(db, "s1", "m1", "user", "We discussed customidentifier routing.", base)
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("identifier", owner="alice", db=db)
|
||||
|
||||
assert [r.message_id for r in results] == ["m1"]
|
||||
assert "identifier" in results[0].content_snippet
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_merges_like_substring_hits_with_fts_hits():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "s1", owner="alice")
|
||||
_add_message(db, "s1", "m-token", "user", "The identifier token is standalone.", base)
|
||||
_add_message(db, "s1", "m-substring", "assistant", "We also discussed customidentifier routing.", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("identifier", owner="alice", db=db)
|
||||
|
||||
assert {r.message_id for r in results} == {"m-token", "m-substring"}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_can_preserve_unrestricted_no_auth_route_scope():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "owned", owner="admin")
|
||||
_add_session(db, "legacy", owner=None)
|
||||
_add_message(db, "owned", "m-owned", "user", "no auth search target", base)
|
||||
_add_message(db, "legacy", "m-legacy", "user", "no auth search target", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages(
|
||||
"no auth search target",
|
||||
owner=None,
|
||||
restrict_owner=False,
|
||||
db=db,
|
||||
)
|
||||
|
||||
assert {r.message_id for r in results} == {"m-owned", "m-legacy"}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_session_search_excludes_archived_by_default():
|
||||
db = _db(with_fts=True)
|
||||
try:
|
||||
base = datetime(2026, 1, 1, 12, 0, 0)
|
||||
_add_session(db, "active", owner="alice")
|
||||
_add_session(db, "archived", owner="alice", archived=True)
|
||||
_add_message(db, "active", "m-active", "user", "archive filter target", base)
|
||||
_add_message(db, "archived", "m-archived", "user", "archive filter target", base + timedelta(minutes=1))
|
||||
db.commit()
|
||||
|
||||
results = search_session_messages("archive filter target", owner="alice", db=db)
|
||||
|
||||
assert [r.message_id for r in results] == ["m-active"]
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def test_chat_messages_fts_migration_backfills_and_tracks_inserts(tmp_path, monkeypatch):
|
||||
from core import database as cdb
|
||||
|
||||
db_path = tmp_path / "app.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE chat_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
content TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO chat_messages(id, session_id, role, content)
|
||||
VALUES ('m1', 's1', 'user', 'backfilled transcript search');
|
||||
"""
|
||||
)
|
||||
conn.close()
|
||||
|
||||
monkeypatch.setattr(cdb, "DATABASE_URL", f"sqlite:///{db_path}")
|
||||
|
||||
cdb._migrate_chat_messages_fts()
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
backfilled = conn.execute(
|
||||
"SELECT message_id FROM chat_messages_fts WHERE chat_messages_fts MATCH 'backfilled'"
|
||||
).fetchall()
|
||||
assert backfilled == [("m1",)]
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO chat_messages(id, session_id, role, content) VALUES (?, ?, ?, ?)",
|
||||
("m2", "s1", "assistant", "triggered transcript search"),
|
||||
)
|
||||
triggered = conn.execute(
|
||||
"SELECT message_id FROM chat_messages_fts WHERE chat_messages_fts MATCH 'triggered'"
|
||||
).fetchall()
|
||||
assert triggered == [("m2",)]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_search_chats_formats_shared_results(monkeypatch):
|
||||
from src import session_search
|
||||
from src.tool_implementations import do_search_chats
|
||||
|
||||
def fake_search(query, limit=20, owner=None, include_archived=False, context_messages=1, db=None):
|
||||
return [
|
||||
SessionSearchResult(
|
||||
message_id="m2",
|
||||
session_id="s1",
|
||||
session_name="Design notes",
|
||||
role="assistant",
|
||||
content="We discussed session search.",
|
||||
content_snippet="We discussed session search.",
|
||||
timestamp="2026-01-01T12:00:00",
|
||||
context_before=[{"message_id": "m1", "role": "user", "content": "Can you find old chats?", "timestamp": None}],
|
||||
context_after=[{"message_id": "m3", "role": "user", "content": "That helps.", "timestamp": None}],
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(session_search, "search_session_messages", fake_search)
|
||||
|
||||
out = asyncio.run(do_search_chats("session search", owner="alice"))
|
||||
|
||||
assert "Design notes" in out["results"]
|
||||
assert "Match (assistant): We discussed session search." in out["results"]
|
||||
assert "Before (user): Can you find old chats?" in out["results"]
|
||||
assert "After (user): That helps." in out["results"]
|
||||
Reference in New Issue
Block a user