feat(notes): attach selected Notes/Todos as Agent context (#6495)

* feat: add Notes and Todos as agent chat context

- Send selected Note/Todo IDs from the Notes UI
- Load selected notes with owner scoping in chat routes
- Format note content and checklist items for the model
- Inject selected notes as untrusted chat context
- Add a selected-context indicator beside Shell Access
- Keep selected context associated with the active chat session

* Improved selected notes context handling and discouragin redundant note lookup and permission requests

* Fix selected notes UI when switching chats

* fix(notes): harden selected context lifecycle and validation

---------

Co-authored-by: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com>
This commit is contained in:
Beyrem Weslati
2026-10-13 18:57:19 +01:00
committed by GitHub
co-authored by Alexandre Teixeira
parent a8c147b238
commit 8feb50e894
9 changed files with 324 additions and 2 deletions
+18
View File
@@ -1010,6 +1010,7 @@ async def build_chat_context(
chat_processor,
message: str,
session_id: str,
selected_notes_context: str | None = None,
preset_id=None,
att_ids: list = None,
use_web=None,
@@ -1182,6 +1183,23 @@ async def build_chat_context(
if search_context and allow_tool_preprocessing and not casual_low_signal:
preface.append(untrusted_context_message("prefetched search context", search_context))
# Inject selected notes as context
if selected_notes_context:
selected_notes_context = (
"The following selected notes and todos are reference context explicitly chosen "
"by the user. Use them directly when relevant to the user's request. Do not "
"ask for permission to use this context, and do not call note-management "
"tools just to retrieve the same notes; use those tools only for a requested "
"change or when this context is insufficient.\n\n"
+ selected_notes_context
)
preface.append(
untrusted_context_message(
"Selected notes and todos",
selected_notes_context,
)
)
# YouTube transcripts
for transcript in preprocessed.youtube_transcripts:
preface.append(untrusted_context_message("youtube transcript", transcript))
+82 -1
View File
@@ -58,7 +58,7 @@ from src.auth_helpers import (
from routes.session_routes import _verify_session_owner
from routes.document_helpers import _owner_session_filter
from core.database import SessionLocal, get_session_mode, set_session_mode
from core.database import Session as DBSession, ChatMessage as DBChatMessage
from core.database import Session as DBSession, ChatMessage as DBChatMessage, Note
from core.database import Document as DBDocument, ModelEndpoint
from core.log_safety import redact_url
from routes.research_routes import _resolve_research_endpoint
@@ -2153,6 +2153,68 @@ def _set_user_time_from_request(request: Request) -> None:
except Exception:
pass
def _parse_selected_note_ids(raw) -> list[str]:
"""Validate user-selected Note IDs before passing them to SQLAlchemy."""
if raw is None or raw == "":
return []
if not isinstance(raw, str) or len(raw) > 8192:
raise HTTPException(400, "Invalid selected_note_ids payload")
try:
parsed = json.loads(raw)
except (TypeError, ValueError) as exc:
raise HTTPException(400, "Invalid selected_note_ids JSON") from exc
if (
not isinstance(parsed, list)
or len(parsed) > 50
or any(
not isinstance(note_id, str)
or not note_id.strip()
or len(note_id) > 128
for note_id in parsed
)
):
raise HTTPException(400, "selected_note_ids must be an array of up to 50 non-empty string IDs")
return list(dict.fromkeys(parsed))
def _format_selected_notes_context(notes) -> str:
"""Format selected Notes/Todos as reference context for the model."""
sections = []
for index, note in enumerate(notes, start=1):
lines = [
f"--- Selected Note {index} ---",
f"Title: {note.title or '(untitled)'}",
f"Type: {note.note_type or 'note'}",
]
if note.content:
lines.extend([
"Content:",
str(note.content),
])
if note.items:
try:
items = json.loads(note.items)
except (TypeError, json.JSONDecodeError):
items = []
if isinstance(items, list):
lines.append("Items:")
for item in items:
if not isinstance(item, dict):
continue
status = "done" if item.get("done") else "pending"
text = str(item.get("text") or "").strip()
if text:
lines.append(f"- [{status}] {text}")
sections.append("\n".join(lines))
return "\n\n".join(sections)
def _resolve_prompt_thinking_mode(explicit_mode, preset_id, preset_manager):
"""Use an explicit request override, then fall back to the active preset."""
@@ -2423,6 +2485,8 @@ def setup_chat_routes(
form_data = await request.form()
message = form_data.get("message")
session = form_data.get("session")
selected_note_ids = _parse_selected_note_ids(form_data.get("selected_note_ids"))
attachments = form_data.get("attachments")
use_web = form_data.get("use_web")
use_research = form_data.get("use_research")
@@ -2730,6 +2794,22 @@ def setup_chat_routes(
current_rejected=workspace_rejected,
)
owner = effective_user(request)
selected_notes = []
if selected_note_ids:
notes_db = SessionLocal()
try:
selected_notes = (
notes_db.query(Note)
.filter(
Note.id.in_(selected_note_ids),
Note.owner == owner,
)
.all()
)
finally:
notes_db.close()
selected_notes_context = _format_selected_notes_context(selected_notes)
if tool_approval_id:
_reject_delegated_tool_approval(request)
from src.agent_runtime.authority import require_user_approval_request
@@ -3001,6 +3081,7 @@ def setup_chat_routes(
sess, request, chat_handler, chat_processor,
message=message,
session_id=session,
selected_notes_context=selected_notes_context,
preset_id=preset_id,
att_ids=att_ids,
use_web=use_web,
+3 -1
View File
@@ -6933,6 +6933,7 @@ _AGENT_RULES = """\
- If the user asks for multiple specific emails and you call `read_email` more than once, your final answer MUST include every successfully read email, clearly separated and linked by UID. Do not answer with only the last email you read.
- Multiple email accounts: if tool output says "Other accounts" or the user asks "my Gmail?", "other inbox?", "work mail?", "custom domain mail?", or names any mailbox/account, DO NOT answer from memory. Call `list_email_accounts` if needed, then call `list_emails`/`read_email`/`bulk_email` with the exact `account` value for that mailbox. Account names are user-defined labels; if the user typo-matches a known account, use the closest listed account instead of claiming it does not exist. NEVER use `app_api` or `/api/email/accounts` to discover email accounts; that route is owner-filtered in tool context and can falsely return empty.
- User identity facts/preferences ("my name is <name>", "I live in <place>", "I prefer concise replies", "call me <name>") → use `manage_memory` with action=add. NEVER use `manage_contact` for facts about the user unless the user explicitly says to create/update a contact and provides contact details such as an email or phone.
- When the prompt contains "Selected notes and todos", treat that content as already-provided reference context. Use it directly when it answers the request; do not call `manage_notes` to list/search/view the same notes, and do not ask for permission merely to use the selected context. Call `manage_notes` only when the user requests a note change or the selected context is insufficient and a fresh lookup is needed.
- "Create/add/write a note" / "notes" / "todos" / "remind me to X at <time>" → use `manage_notes`. Do NOT store notes in `manage_memory`; memory is for persistent facts/preferences about the user, not note content. For reminders, include a `due_date`; for todos, use `note_type=checklist` when appropriate.
- "Do X every morning / daily / on a schedule / automatically" (e.g. "summarize my inbox every morning") → this is a request to CREATE A SCHEDULED TASK, not to do X once right now. Call `manage_tasks` with action=create (prompt = what to do, schedule + cron/time). Do NOT just perform the action inline this turn — the user wants it to recur. After creating, return a clickable `[Task name](#task-<id>)` link and tell them it'll run on schedule and show in the Tasks panel. If you also want to show a sample of this run, do that AFTER creating the task, not instead of it.
- There is NO generic sleep / auto-wakeup / resume-after-this-turn primitive. Background jobs and subagent-style work should return a job/task id and notify the session automatically when finished; do not sleep or poll for their progress. If the user explicitly asks you to retry or do something later, call `manage_tasks` with action=create, `task_type=llm`, `schedule=once`, and a self-contained prompt. NEVER claim you will wake up, retry later, or handle something later unless a `manage_tasks` create call succeeds.
@@ -7041,6 +7042,7 @@ _AGENT_RULES = """\
- After a tool fails, retry with a concrete fix or state what is blocking you.
- Finish only when the user's concrete request is actually done, or clearly state that you are blocked.
- User identity facts/preferences ("my name is X", "call me X", "I live in X") use `manage_memory`, not contacts.
- When the prompt contains "Selected notes and todos", treat that content as already-provided reference context. Use it directly when it answers the request; do not call `manage_notes` to list/search/view the same notes, and do not ask for permission merely to use the selected context. Call `manage_notes` only when the user requests a note change or the selected context is insufficient and a fresh lookup is needed.
"""
_API_AGENT_RULES = """\
@@ -7099,7 +7101,7 @@ _DOMAIN_RULES = {
- After a successful serve, verify with `list_served_models`; if an external server is running but invisible, use `adopt_served_model`.""",
"notes_calendar_tasks": """\
## Notes/calendar/tasks rules
- Notes/todos/reminders use `manage_notes`, not memory.
- Notes/todos/reminders use `manage_notes`, not memory. However, when the prompt contains "Selected notes and todos", those notes are already available as reference context: use them directly and do not list/search/view them again or ask permission to use them. Use `manage_notes` only for a requested change or when a fresh lookup is genuinely needed.
- Calendar create/update/delete should call `manage_calendar` with `action=list_calendars` first.
- Recurring/automatic/scheduled requests create a `manage_tasks` task; do not just perform the action once.""",
"memory": """\
+15
View File
@@ -1178,6 +1178,21 @@
<polyline points="4 17 10 11 4 5"/><line x1="12" y1="19" x2="20" y2="19"/>
</svg>
</button>
<!-- Selected Notes/Todos indicator -->
<button
type="button"
class="input-icon-btn tool-indicator"
title="Selected Notes/Todos context — click to clear"
id="selected-notes-context-btn"
aria-label="Clear selected Notes/Todos context"
style="display:none;"
>
<span id="selected-notes-context-count" style="font-size:11px;margin-left:2px;"></span>
<svg class="tool-indicator-x" width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" aria-hidden="true">
<line x1="6" y1="6" x2="18" y2="18"/>
<line x1="18" y1="6" x2="6" y2="18"/>
</svg>
</button>
<!-- Workspace indicator (hidden until a folder is set) -->
<button type="button" class="input-icon-btn tool-indicator" title="Workspace - click to clear" id="workspace-indicator-btn" aria-label="Clear workspace" style="display:none;">
<svg class="workspace-indicator-icon" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 7a2 2 0 0 1 2-2h4l2 2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2z"/></svg>
+68
View File
@@ -65,6 +65,8 @@ import { invalidateSettings } from './appConfig.js';
let _contextHeaderAnchorEl = null;
let _contextHeaderPopupCleanup = null;
let _pendingToolApproval = null;
const _selectedNoteIdsBySession = new Map();
let _pendingSelectedNoteIds = [];
let _lastPrivateBrowserUrl = '';
function _isPrivateBrowserTool(tool) {
const name = String(tool || '').toLowerCase();
@@ -240,6 +242,46 @@ import { invalidateSettings } from './appConfig.js';
_submitToolApprovalWhenIdle(_pendingToolApproval.approval_id);
});
window.addEventListener('odysseus:notes-context-selected', (event) => {
const noteIds = event.detail?.noteIds;
if (!Array.isArray(noteIds) || noteIds.length === 0) return;
const sessionId = sessionModule.getCurrentSessionId?.();
if (!sessionId && !sessionModule.hasPendingChat?.()) {
console.warn('Cannot attach Notes context: no active or pending chat');
return;
}
if (sessionId) {
_selectedNoteIdsBySession.set(sessionId, [...noteIds]);
} else {
_pendingSelectedNoteIds = [...noteIds];
}
if (event.detail) event.detail.accepted = true;
_renderSelectedNotesIndicator(noteIds);
});
window.addEventListener('odysseus:session-changed', (event) => {
const sessionId = event.detail?.sessionId;
if (sessionId && event.detail?.materialized && _pendingSelectedNoteIds.length) {
_selectedNoteIdsBySession.set(sessionId, _pendingSelectedNoteIds);
}
_pendingSelectedNoteIds = [];
_renderSelectedNotesIndicator(
sessionId ? (_selectedNoteIdsBySession.get(sessionId) || []) : []
);
});
document.getElementById('selected-notes-context-btn')?.addEventListener('click', () => {
const sessionId = sessionModule.getCurrentSessionId?.();
if (sessionId) {
_selectedNoteIdsBySession.delete(sessionId);
} else {
_pendingSelectedNoteIds = [];
}
_renderSelectedNotesIndicator([]);
});
function _fmtContextNumber(n) {
const v = Number(n || 0);
return v ? v.toLocaleString() : '?';
@@ -290,6 +332,26 @@ import { invalidateSettings } from './appConfig.js';
return Math.max(50, Math.min(95, Math.round(n)));
}
function _renderSelectedNotesIndicator(noteIds){
const button = document.getElementById('selected-notes-context-btn');
const count = document.getElementById('selected-notes-context-count');
if (!button || !count) return;
const total = Array.isArray(noteIds) ? noteIds.length : 0;
if(!total){
button.style.display = 'none';
count.textContent = '';
return;
}
count.textContent = `${total} note${total === 1 ? '' : 's'} selected`;
button.style.display = '';
button.title = `${total} selected Note/Todo${total === 1 ? '' : 's'} — click to clear`;
button.setAttribute(
'aria-label',
`Clear ${total} selected Note/Todo${total === 1 ? '' : 's'} context`
);
}
function _liveSessionModule() {
return (window.sessionModule && window.sessionModule.getCurrentSessionId)
? window.sessionModule
@@ -2497,6 +2559,12 @@ import { invalidateSettings } from './appConfig.js';
const fd = new FormData();
fd.append('message', approvalForSend ? '' : _finalMsgWithInject);
fd.append('session', streamSessionId);
const selectedNoteIds = _selectedNoteIdsBySession.get(streamSessionId) || [];
if(selectedNoteIds.length){
fd.append('selected_note_ids', JSON.stringify(selectedNoteIds))
}
if (approvalForSend) {
fd.append('tool_approval_id', approvalForSend.approval_id);
fd.append('tool_approval_decision', approvalForSend.decision);
+14
View File
@@ -1230,6 +1230,7 @@ export function openPanel() {
<label class="memory-bulk-check-all"><input type="checkbox" id="notes-select-all" /> All</label>
<span id="notes-selected-count">0 Selected</span>
<span style="flex:1"></span>
<button id="notes-send-agent" class="memory-toolbar-btn" disabled><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:5px;opacity:0.6"><path d="M14.7 6.3a1 1 0 0 0 0 1.4l1.6 1.6a1 1 0 0 0 1.4 0l3.77-3.77a6 6 0 0 1-7.94 7.94l-6.91 6.91a2.12 2.12 0 0 1-3-3l6.91-6.91a6 6 0 0 1 7.94-7.94l-3.76 3.76z"></path></svg>Send to Agent</button>
<button id="notes-bulk-archive" class="memory-toolbar-btn" disabled>
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="vertical-align:-2px;margin-right:4px;"><rect x="2" y="3" width="20" height="5" rx="1"/><path d="M4 8v11a2 2 0 002 2h12a2 2 0 002-2V8"/><path d="M10 12h4"/></svg>Archive
</button>
@@ -1379,6 +1380,17 @@ export function openPanel() {
_renderNotes();
_updateBulkBar();
});
document.getElementById('notes-send-agent').addEventListener('click', async () => {
const ids = [..._selectedIds];
if (!ids.length) return;
const detail = { noteIds: ids, accepted: false };
window.dispatchEvent(new CustomEvent('odysseus:notes-context-selected', { detail }));
if (!detail.accepted) {
uiModule.showToast('Choose a chat or model before attaching notes');
return;
}
_exitSelectMode();
})
document.getElementById('notes-bulk-archive').addEventListener('click', async () => {
const ids = [..._selectedIds];
if (!ids.length) return;
@@ -1511,10 +1523,12 @@ function _exitSelectMode() {
function _updateBulkBar() {
const count = _selectedIds.size;
const countEl = document.getElementById('notes-selected-count');
const sendAgentBtn = document.getElementById('notes-send-agent');
const archiveBtn = document.getElementById('notes-bulk-archive');
const deleteBtn = document.getElementById('notes-bulk-delete');
const allEl = document.getElementById('notes-select-all');
if (countEl) countEl.textContent = `${count} Selected`;
if (sendAgentBtn) sendAgentBtn.disabled = count === 0;
if (archiveBtn) archiveBtn.disabled = count === 0;
if (deleteBtn) deleteBtn.disabled = count === 0;
if (allEl) allEl.checked = _notes.length > 0 && _notes.every(n => _selectedIds.has(n.id));
+15
View File
@@ -351,6 +351,9 @@ let _sessionListFocused = false;
function _deselectCurrentSession(sid) {
if (currentSessionId !== sid) return;
currentSessionId = null;
window.dispatchEvent(new CustomEvent('odysseus:session-changed', {
detail: { sessionId: null },
}));
uiModule.el('chat-history').innerHTML = '';
uiModule.el('current-meta').textContent = 'Odysseus Chat';
Storage.remove('lastSessionId');
@@ -2065,6 +2068,9 @@ export async function selectSession(id, { keepSidebar = false, showLoading = tru
try { window.documentModule.clearSelection(); } catch {}
}
currentSessionId = id;
window.dispatchEvent(new CustomEvent('odysseus:session-changed', {
detail: { sessionId: id ? id : null },
}));
try { window.__odysseusLastSelectedSessionId = id; } catch (_) {}
// Identify Assistant / task-output sessions so we don't "trap" the user
// there on return. Skipped from both `lastSessionId` persistence and the
@@ -2445,6 +2451,9 @@ export function createDirectChat(url, modelId, endpointId, opts = {}) {
_skipAutoSelect = true;
_suppressNextSessionLoading = true;
currentSessionId = null;
window.dispatchEvent(new CustomEvent('odysseus:session-changed', {
detail: { sessionId: null },
}));
try { window.__odysseusLastSelectedSessionId = ''; } catch (_) {}
Storage.remove('lastSessionId');
history.replaceState(null, '', window.location.pathname);
@@ -2555,6 +2564,9 @@ export async function materializePendingSession() {
}
_pendingChat = null;
currentSessionId = payload.id;
window.dispatchEvent(new CustomEvent('odysseus:session-changed', {
detail: { sessionId: payload.id, materialized: true },
}));
if (!isIncognito) {
Storage.set('lastSessionId', payload.id);
history.replaceState(null, '', '#' + payload.id);
@@ -2626,6 +2638,9 @@ export function getCurrentEndpointUrl() {
export function setCurrentSessionId(id) {
_sessionNavToken++;
currentSessionId = id;
window.dispatchEvent(new CustomEvent('odysseus:session-changed', {
detail: { sessionId: id ? id : null },
}));
try { window.__odysseusLastSelectedSessionId = id || ''; } catch (_) {}
if (!id) {
_suppressNextSessionLoading = true;
+81
View File
@@ -0,0 +1,81 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { readFileSync } from 'node:fs';
import vm from 'node:vm';
const chatSource = readFileSync(new URL('../static/js/chat.js', import.meta.url), 'utf8');
const sessionSource = readFileSync(new URL('../static/js/sessions.js', import.meta.url), 'utf8');
const notesSource = readFileSync(new URL('../static/js/notes.js', import.meta.url), 'utf8');
function setup() {
const start = chatSource.indexOf(" window.addEventListener('odysseus:notes-context-selected'");
const end = chatSource.indexOf(' function _fmtContextNumber(n)', start);
assert.ok(start !== -1 && end > start, 'production event handlers must exist');
const handlers = new Map();
const rendered = [];
let sessionId = null;
let pending = false;
let clear;
const win = { addEventListener: (name, fn) => handlers.set(name, fn) };
const doc = {
getElementById: () => ({ addEventListener: (name, fn) => { if (name === 'click') clear = fn; } }),
};
const sessionModule = {
getCurrentSessionId: () => sessionId,
hasPendingChat: () => pending,
};
vm.runInNewContext(
`const _selectedNoteIdsBySession = new Map(); let _pendingSelectedNoteIds = [];
${chatSource.slice(start, end)}`,
{ window: win, document: doc, sessionModule, console, _renderSelectedNotesIndicator: ids => rendered.push(Array.from(ids)) },
);
return {
setSession(id, isPending = false) { sessionId = id; pending = isPending; },
emit(name, detail) { handlers.get(name)({ detail }); },
selected(ids) { const detail = { noteIds: ids, accepted: false }; handlers.get('odysseus:notes-context-selected')({ detail }); return detail; },
clear() { clear(); },
lastRendered() { return rendered.at(-1); },
};
}
test('pending selections transfer only to the materialized chat', () => {
const ctx = setup();
ctx.setSession(null, true);
assert.equal(ctx.selected(['a', 'b']).accepted, true);
assert.deepEqual(ctx.lastRendered(), ['a', 'b']);
ctx.setSession('new-chat');
ctx.emit('odysseus:session-changed', { sessionId: 'new-chat', materialized: true });
assert.deepEqual(ctx.lastRendered(), ['a', 'b']);
ctx.setSession('other-chat');
ctx.emit('odysseus:session-changed', { sessionId: 'other-chat' });
assert.deepEqual(ctx.lastRendered(), []);
ctx.setSession('new-chat');
ctx.emit('odysseus:session-changed', { sessionId: 'new-chat' });
assert.deepEqual(ctx.lastRendered(), ['a', 'b']);
ctx.clear();
assert.deepEqual(ctx.lastRendered(), []);
});
test('new chat resets old indicator and invalid selection is rejected', () => {
const ctx = setup();
assert.equal(ctx.selected(['a']).accepted, false);
ctx.setSession('old-chat');
assert.equal(ctx.selected(['a']).accepted, true);
ctx.setSession(null, true);
ctx.emit('odysseus:session-changed', { sessionId: null });
assert.deepEqual(ctx.lastRendered(), []);
assert.equal(ctx.selected(['b']).accepted, true);
ctx.emit('odysseus:session-changed', { sessionId: null });
assert.deepEqual(ctx.lastRendered(), []);
ctx.setSession('fresh');
ctx.emit('odysseus:session-changed', { sessionId: 'fresh', materialized: true });
assert.deepEqual(ctx.lastRendered(), []);
});
test('producer events and rejection acknowledgement are wired in source', () => {
const create = sessionSource.slice(sessionSource.indexOf('export function createDirectChat'), sessionSource.indexOf('export async function materializePendingSession'));
const materialize = sessionSource.slice(sessionSource.indexOf('export async function materializePendingSession'), sessionSource.indexOf('export function preMaterializePendingSession'));
assert.match(create, /currentSessionId = null;[\s\S]{0,180}odysseus:session-changed/);
assert.match(materialize, /currentSessionId = payload.id;[\s\S]{0,180}odysseus:session-changed/);
assert.match(notesSource, /if \(!detail\.accepted\)/);
});
+28
View File
@@ -0,0 +1,28 @@
"""Regression tests for untrusted selected-note ID parsing."""
import json
import pytest
from fastapi import HTTPException
from routes.chat_routes import _parse_selected_note_ids
@pytest.mark.parametrize("raw,expected", [
(None, []),
("", []),
("[]", []),
('["one", "two", "one"]', ["one", "two"]),
])
def test_selected_note_ids_valid(raw, expected):
assert _parse_selected_note_ids(raw) == expected
@pytest.mark.parametrize("raw", [
"42", "true", "null", '{"id":"a"}',
'["one", 5]', '["one", null]', '["one", false]',
'[""]', '[" "]', '["one", {}]',
"not json", json.dumps(["a"] * 51),
json.dumps(["a" * 129]), " " * 8193,
])
def test_selected_note_ids_rejected(raw):
with pytest.raises(HTTPException) as exc:
_parse_selected_note_ids(raw)
assert exc.value.status_code == 400