diff --git a/.env.example b/.env.example index d23276eb8..2d1be3373 100644 --- a/.env.example +++ b/.env.example @@ -189,6 +189,7 @@ SEARXNG_INSTANCE=http://localhost:8080 # ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400 # email compose attachment (25 MB) # ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB) # ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB) +# ODYSSEUS_TTS_CACHE_MAX_BYTES=524288000 # TTS cache (500 MB) # ============================================================ # Host Docker access (explicit opt-in) diff --git a/.github/scripts/check-issue-description.js b/.github/scripts/check-issue-description.js index a76ca29ab..63162b0d7 100644 --- a/.github/scripts/check-issue-description.js +++ b/.github/scripts/check-issue-description.js @@ -153,6 +153,16 @@ module.exports = async ({ github, context, core }) => { } } + const LABEL_BAD = 'needs more info'; + const LABEL_GOOD = 'ready for review'; + + // Closed issues are no longer awaiting review. + // This also prevents later edits to closed issues from restoring the label. + if (issue.state === 'closed') { + await dropLabel(LABEL_GOOD); + return; + } + // ── Find existing bot comment to update in-place ────────────────────────── const MARKER = ''; const { data: comments } = await github.rest.issues.listComments({ @@ -160,9 +170,6 @@ module.exports = async ({ github, context, core }) => { }); const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER)); - const LABEL_BAD = 'needs more info'; - const LABEL_GOOD = 'ready for review'; - if (failures.length === 0) { if (existing) { await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7d3659e8..e42c1a5d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [main] + branches: [main, dev] pull_request: # Least privilege: none of the jobs write to the repo. @@ -103,10 +103,7 @@ jobs: python-tests: name: Python tests (pytest) runs-on: ubuntu-latest - # Informational for now: the suite has known flaky / environment-dependent - # failures (test isolation + embedding-model assertions). Tracked under the - # ROADMAP "fresh install smoke tests" item; make this required once green. - continue-on-error: true + # Make Python test validation authoritative for the configured scope. steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: diff --git a/.github/workflows/issue-description-check.yml b/.github/workflows/issue-description-check.yml index 52e9dddae..5ce6037f0 100644 --- a/.github/workflows/issue-description-check.yml +++ b/.github/workflows/issue-description-check.yml @@ -2,7 +2,7 @@ name: ci / issue description check on: issues: - types: [opened, edited, reopened] + types: [opened, edited, reopened, closed] permissions: issues: write diff --git a/app.py b/app.py index e740ad518..bee4dae8f 100644 --- a/app.py +++ b/app.py @@ -692,7 +692,7 @@ from routes.history.history_routes import setup_history_routes app.include_router(setup_history_routes(session_manager, upload_handler=upload_handler)) # Search -from routes.search_routes import setup_search_routes +from routes.search.search_routes import setup_search_routes app.include_router(setup_search_routes(config)) # Presets @@ -739,7 +739,7 @@ app.include_router(setup_stt_routes(stt_service)) logger.info("STT service initialized (provider managed via settings)") # Documents (artifacts/canvas) -from routes.document_routes import setup_document_routes +from routes.document.document_routes import setup_document_routes document_router = setup_document_routes(session_manager, upload_handler) app.include_router(document_router) @@ -805,7 +805,7 @@ app.include_router(setup_font_routes()) # MCP (Model Context Protocol) from src.mcp_manager import McpManager from src.agent_tools import set_mcp_manager -from routes.mcp_routes import setup_mcp_routes +from routes.mcp.mcp_routes import setup_mcp_routes mcp_manager = McpManager() set_mcp_manager(mcp_manager) @@ -820,7 +820,7 @@ set_ai_rag_manager(rag_manager, personal_docs_mgr) logger.info("AI interaction tools initialized (session, memory, RAG, UI control)") # Webhooks -from routes.webhook_routes import setup_webhook_routes +from routes.webhook.webhook_routes import setup_webhook_routes app.include_router(setup_webhook_routes(webhook_manager, auth_manager, session_manager, api_key_manager)) # API Tokens @@ -852,7 +852,7 @@ app.include_router(setup_codex_routes( )) app.include_router(setup_claude_routes()) -from routes.vault_routes import setup_vault_routes +from routes.vault.vault_routes import setup_vault_routes app.include_router(setup_vault_routes()) # Contacts (CardDAV) diff --git a/core/atomic_io.py b/core/atomic_io.py index 81c640d8a..40a51adbe 100644 --- a/core/atomic_io.py +++ b/core/atomic_io.py @@ -15,17 +15,21 @@ from __future__ import annotations import json import os +import uuid from typing import Any, Optional def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> None: """Atomically persist `data` as JSON at `path`. - The temp file uses the live PID as a suffix so two processes saving the - same file (e.g. unit tests) don't collide on the rename target. + The temp file uses a random suffix so two concurrent writers saving the + same file don't collide on the rename target. A PID suffix does not do + this: the PID is constant for the life of a process, so two writers on + the same path within one process (or one single-process container, where + the PID never changes at all) still race for the same temp file. """ os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - tmp = f"{path}.tmp.{os.getpid()}" + tmp = f"{path}.tmp.{uuid.uuid4().hex}" with open(tmp, "w", encoding="utf-8") as f: json.dump(data, f, indent=indent) f.flush() @@ -37,7 +41,7 @@ def atomic_write_text(path: str, text: str) -> None: if not isinstance(text, str): raise TypeError("atomic_write_text expects a string") os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - tmp = f"{path}.tmp.{os.getpid()}" + tmp = f"{path}.tmp.{uuid.uuid4().hex}" with open(tmp, "w", encoding="utf-8") as f: f.write(text) f.flush() diff --git a/core/database.py b/core/database.py index a9ad90b8b..6eb529949 100644 --- a/core/database.py +++ b/core/database.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Optional from urllib.parse import unquote, urlparse -from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text +from sqlalchemy import DDL, event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, inspect, text from sqlalchemy.engine import Engine, make_url from sqlalchemy.types import TypeDecorator from sqlalchemy.ext.declarative import declarative_base, declared_attr @@ -430,6 +430,93 @@ class EmailAccount(TimestampMixin, Base): ) +class EmailAccountOwnerLock(Base): + """Durable per-owner mutex for email-account default mutations. + + Row-locking databases serialize mutations by locking this row before they + inspect or stage EmailAccount changes. SQLite uses ``BEGIN IMMEDIATE`` + instead, because it ignores ``SELECT ... FOR UPDATE``; keeping the table in + the shared metadata still makes the non-SQLite path available without a + separate migration. The empty key represents the normalized legacy / + unconfigured scope shared by ``owner IS NULL`` and ``owner = ''`` rows. + """ + __tablename__ = "email_account_owner_locks" + + owner_key = Column(String, primary_key=True) + + +_EMAIL_ACCOUNT_DEFAULT_INDEX = "ux_email_accounts_one_default_per_owner" +_EMAIL_ACCOUNT_DEFAULT_INDEX_DDL = { + "sqlite": ( + f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} " + "ON email_accounts (COALESCE(owner, '')) WHERE is_default = 1" + ), + "postgresql": ( + f"CREATE UNIQUE INDEX IF NOT EXISTS {_EMAIL_ACCOUNT_DEFAULT_INDEX} " + "ON email_accounts ((COALESCE(owner, ''))) WHERE is_default IS TRUE" + ), +} + + +# SQLAlchemy cannot express one portable partial, functional index across the +# two supported database families. Register dialect-specific DDL so fresh +# databases get the invariant as part of create_all(); the startup migration +# below installs the same index on existing databases after normalizing legacy +# duplicate rows. +for _dialect_name, _index_ddl in _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.items(): + event.listen( + EmailAccount.__table__, + "after_create", + DDL(_index_ddl).execute_if(dialect=_dialect_name), + ) + + +def lock_email_account_owner_mutations(db, *owners: str) -> None: + """Lock normalized email-account owner scopes in canonical order. + + ``NULL`` and the empty string are one legacy/single-user owner partition, + matching the unique default-account index. SQLite has only a database + writer reservation, while row-locking databases use durable mutex rows. + Sorting all requested owner keys keeps multi-owner operations such as user + rename from deadlocking with another mutation that requests the same keys + in the opposite order. + """ + from sqlalchemy.exc import IntegrityError + + owner_keys = sorted({owner or "" for owner in owners} or {""}) + if db.get_bind().dialect.name == "sqlite": + db.execute(text("BEGIN IMMEDIATE")) + return + + for owner_key in owner_keys: + lock_row = db.get( + EmailAccountOwnerLock, + owner_key, + with_for_update=True, + ) + if lock_row is not None: + continue + + inserted = False + try: + with db.begin_nested(): + db.add(EmailAccountOwnerLock(owner_key=owner_key)) + db.flush() + inserted = True + except IntegrityError: + # A competing transaction created the mutex row first. Once its + # insert commits, lock that durable row before touching accounts. + pass + + if not inserted: + ( + db.query(EmailAccountOwnerLock) + .filter(EmailAccountOwnerLock.owner_key == owner_key) + .with_for_update() + .one() + ) + + class ModelEndpoint(TimestampMixin, Base): """Admin-configured model endpoints. Models are auto-discovered via /v1/models.""" __tablename__ = "model_endpoints" @@ -1812,72 +1899,142 @@ class Integration(TimestampMixin, Base): -def _migrate_seed_email_account(): - """If email_accounts is empty and settings.json has legacy flat imap_host/smtp_host - keys, create a single default account from them so nothing breaks for users who - upgraded. Safe to run repeatedly — it short-circuits once any row exists.""" +def _migrate_email_account_default_invariant(): + """Normalize legacy duplicates and install durable at-most-one enforcement. + + Older databases only had a non-unique ``(owner, is_default)`` lookup index. + Keep the oldest default deterministically in each normalized owner scope, + then add the same partial functional unique index used for fresh schemas. + """ + dialect_name = engine.dialect.name + index_ddl = _EMAIL_ACCOUNT_DEFAULT_INDEX_DDL.get(dialect_name) + if index_ddl is None: + logger.warning( + "Email-account default uniqueness is not available for database " + "dialect %s; mutations remain serialized but are not protected by " + "a database constraint", + dialect_name, + ) + return + try: - with engine.connect() as conn: - tables = [r[0] for r in conn.execute(text( - "SELECT name FROM sqlite_master WHERE type='table' AND name='email_accounts'" - ))] - if "email_accounts" not in tables: - return - existing = conn.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0 - if existing > 0: + with engine.begin() as conn: + if not inspect(conn).has_table(EmailAccount.__tablename__): return + default_rows = conn.execute(text(""" + SELECT id, owner + FROM email_accounts + WHERE is_default IS TRUE + ORDER BY + COALESCE(owner, ''), + CASE WHEN created_at IS NULL THEN 1 ELSE 0 END, + created_at, + id + """)).mappings() + seen_owner_keys = set() + duplicate_ids = [] + for row in default_rows: + owner_key = row["owner"] or "" + if owner_key in seen_owner_keys: + duplicate_ids.append(row["id"]) + else: + seen_owner_keys.add(owner_key) - import json as _json - import uuid as _uuid - from pathlib import Path - settings_file = Path(SETTINGS_FILE) - if not settings_file.exists(): - return - try: - s = _json.loads(settings_file.read_text(encoding="utf-8")) - except Exception: - return + for account_id in duplicate_ids: + conn.execute( + text("UPDATE email_accounts SET is_default = :value WHERE id = :id"), + {"value": False, "id": account_id}, + ) + conn.execute(text(index_ddl)) - imap_host = (s.get("imap_host") or "").strip() - smtp_host = (s.get("smtp_host") or "").strip() - if not imap_host and not smtp_host: - return # nothing to migrate + if duplicate_ids: + logger.warning( + "Normalized %d duplicate default email account(s) before " + "installing %s", + len(duplicate_ids), + _EMAIL_ACCOUNT_DEFAULT_INDEX, + ) + except Exception: + # Starting without the constraint would silently retain the race this + # migration is intended to close. Fail startup so an operator sees and + # can repair an incompatible schema instead of accepting unsafe writes. + logger.exception("Failed to enforce the email-account default invariant") + raise + + +def _migrate_seed_email_account(): + """Atomically seed one legacy default account when no account exists. + + Reading settings is intentionally done before taking the owner mutex. The + decisive emptiness check and insert share one locked transaction, so two + application workers starting together cannot both seed a default row. + """ + import json as _json + import uuid as _uuid + + settings_file = Path(SETTINGS_FILE) + if not settings_file.exists(): + return + try: + s = _json.loads(settings_file.read_text(encoding="utf-8")) + except Exception: + return + + imap_host = (s.get("imap_host") or "").strip() + smtp_host = (s.get("smtp_host") or "").strip() + if not imap_host and not smtp_host: + return + + db = None + try: + if not inspect(engine).has_table(EmailAccount.__tablename__): + return + db = SessionLocal() + lock_email_account_owner_mutations(db, "") + existing = db.execute(text("SELECT COUNT(*) FROM email_accounts")).scalar() or 0 + if existing > 0: + return now = utcnow_naive() - with engine.begin() as conn: - conn.execute(text(""" - INSERT INTO email_accounts - (id, owner, name, is_default, enabled, - imap_host, imap_port, imap_user, imap_password, imap_starttls, - smtp_host, smtp_port, smtp_user, smtp_password, - from_address, created_at, updated_at) - VALUES - (:id, :owner, :name, :is_default, :enabled, - :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls, - :smtp_host, :smtp_port, :smtp_user, :smtp_password, - :from_address, :created_at, :updated_at) - """), { - "id": _uuid.uuid4().hex, - "owner": None, - "name": "Default", - "is_default": True, - "enabled": True, - "imap_host": imap_host, - "imap_port": int(s.get("imap_port") or 993), - "imap_user": s.get("imap_user") or "", - "imap_password": s.get("imap_password") or "", - "imap_starttls": bool(s.get("imap_starttls", True)), - "smtp_host": smtp_host, - "smtp_port": int(s.get("smtp_port") or 465), - "smtp_user": s.get("smtp_user") or "", - "smtp_password": s.get("smtp_password") or "", - "from_address": s.get("email_from") or "", - "created_at": now, - "updated_at": now, - }) - logging.getLogger(__name__).info("Seeded email_accounts 'Default' from settings.json") + db.execute(text(""" + INSERT INTO email_accounts + (id, owner, name, is_default, enabled, + imap_host, imap_port, imap_user, imap_password, imap_starttls, + smtp_host, smtp_port, smtp_user, smtp_password, + from_address, created_at, updated_at) + VALUES + (:id, :owner, :name, :is_default, :enabled, + :imap_host, :imap_port, :imap_user, :imap_password, :imap_starttls, + :smtp_host, :smtp_port, :smtp_user, :smtp_password, + :from_address, :created_at, :updated_at) + """), { + "id": _uuid.uuid4().hex, + "owner": None, + "name": "Default", + "is_default": True, + "enabled": True, + "imap_host": imap_host, + "imap_port": int(s.get("imap_port") or 993), + "imap_user": s.get("imap_user") or "", + "imap_password": s.get("imap_password") or "", + "imap_starttls": bool(s.get("imap_starttls", True)), + "smtp_host": smtp_host, + "smtp_port": int(s.get("smtp_port") or 465), + "smtp_user": s.get("smtp_user") or "", + "smtp_password": s.get("smtp_password") or "", + "from_address": s.get("email_from") or "", + "created_at": now, + "updated_at": now, + }) + db.commit() + logger.info("Seeded email_accounts 'Default' from settings.json") except Exception as e: - logging.getLogger(__name__).warning(f"seed email account migration: {e}") + if db is not None: + db.rollback() + logger.warning("seed email account migration: %s", e) + finally: + if db is not None: + db.close() # WARNING: Foreign-key enforcement is enabled globally for all SQLite connections. @@ -1960,6 +2117,7 @@ def init_db(): _migrate_add_crew_member_id() _migrate_add_assistant_columns() _migrate_add_email_smtp_security() + _migrate_email_account_default_invariant() _migrate_seed_email_account() _migrate_add_calendar_metadata() _migrate_add_calendar_is_utc() diff --git a/core/session_manager.py b/core/session_manager.py index 6eb493e95..6e65226d3 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -194,7 +194,12 @@ class SessionManager: is_important=getattr(db_session, 'is_important', False) or False, ) - session.message_count = getattr(db_session, 'message_count', len(history)) + # The rows just loaded are the whole transcript, so they — not the + # denormalized sessions.message_count column — are the truth for this + # cached object. get_session's hydration gate compares against this + # number; seeding it from a drifted column would ask for a reload that + # can never close the gap. + session.message_count = len(history) return session # ------------------------------------------------------------------ @@ -398,30 +403,50 @@ class SessionManager: # ------------------------------------------------------------------ def get_session(self, session_id: str) -> Session: - """Get a session by ID, loading from DB if needed. + """Get a session by ID, loading complete DB history when needed. - Sessions seeded by `load_sessions` start with empty history. The - first read here hydrates them with the message rows. + Sessions seeded by ``load_sessions`` start with empty history, and a + cached session can also become partially stale. Refresh metadata first, + then hydrate whenever the cached transcript is short of the stored rows. + Model-send routes enter through this method before building context, + while paginated display history reads SQLite directly. + + The gate compares against ``sync_session_metadata``'s reconciled count + (the real ``chat_messages`` total), never the denormalized column, so a + hydrate always closes the gap and the next read is a cache hit. """ if session_id not in self.sessions: self._load_session_from_db(session_id) - else: - cached = self.sessions[session_id] - # Lazy hydrate: metadata-only entries get their messages on first read. - if not cached.history and getattr(cached, "message_count", 0) > 0: - self._load_session_from_db(session_id) # Keep model/endpoint metadata fresh. Endpoint deletion can clear the - # DB row while a session object is still cached in RAM. + # DB row while a session object is still cached in RAM. Refreshing first + # also exposes the authoritative message count before completeness is + # checked. self.sync_session_metadata(session_id) + cached = self.sessions[session_id] + cached_count = len(cached.history or []) + stored_count = int(getattr(cached, "message_count", 0) or 0) + if cached_count < stored_count: + self._load_session_from_db(session_id) + # Update last_accessed self._touch_session(session_id) return self.sessions[session_id] def sync_session_metadata(self, session_id: str) -> bool: - """Refresh non-message session fields from the DB into the cached object.""" + """Refresh non-message session fields from the DB into the cached object. + + ``message_count`` is reconciled against the real ``chat_messages`` rows + rather than copied from the denormalized ``sessions.message_count`` + column. That column drifts in normal operation — ``_persist_message`` + swallows a failed insert but ``add_message`` has already appended in + memory, so the next successful persist writes rows+1, and a persist for + an uncached session writes 0. Hydration keys off this number: a + drifted-high column would reload the whole transcript on every warm + read, and a drifted-low one would leave the model a truncated one. + """ session = self.sessions.get(session_id) if session is None: return False @@ -444,7 +469,11 @@ class SessionManager: session.archived = db_session.archived session.owner = getattr(db_session, "owner", None) session.is_important = getattr(db_session, "is_important", False) or False - session.message_count = getattr(db_session, "message_count", session.message_count) or 0 + session.message_count = ( + db.query(DbChatMessage) + .filter(DbChatMessage.session_id == session_id) + .count() + ) return True except Exception as e: logger.error(f"Error syncing session metadata {session_id}: {e}") diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 91e223e05..9699fc038 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -67,6 +67,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index e8c2fd032..804a0a14e 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -66,6 +66,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docker-compose.yml b/docker-compose.yml index b1f2c37ee..b0efb4439 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ services: - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - ODYSSEUS_TTS_CACHE_MAX_BYTES=${ODYSSEUS_TTS_CACHE_MAX_BYTES} - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} diff --git a/docs/setup.md b/docs/setup.md index 53a6fb28c..171a195e7 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -309,6 +309,32 @@ container. Cookbook **Serve** is a separate workflow for serving downloaded models through Odysseus/llama.cpp, so Windows users with an existing Ollama install usually only need to add the endpoint in Settings. +**Tool calls not firing on a manually-added Ollama `/v1` endpoint.** By +design, a local Ollama `/v1` endpoint defaults to the conservative +text-based (fenced-block) tool-calling path rather than native structured +tool calls, since some locally-served models mishandle native schemas (see +#1567). This is correct for most local setups, but if you know your specific +model reliably supports native tool calling (check `ollama show ` for +`tools` under Capabilities), you can opt that endpoint in explicitly. There +is currently no UI control for this on manually-added endpoints (see #5192); +the flag can still be set directly against the existing API, from a browser +console on an authenticated admin session: + +```js +fetch('/api/model-endpoints/', { + method: 'PATCH', + credentials: 'same-origin', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({supports_tools: true}) +}).then(r => r.json()).then(console.log) +``` + +Find `` by inspecting the `/api/model-endpoints` response (or +your browser's network tab while Settings loads the endpoint list). Send +`supports_tools: false` to disable native structured tool calls and force the +conservative fenced/text path, or `supports_tools: null` to return the endpoint +to the Auto heuristic. + **Useful checks.** ```bash diff --git a/mcp_servers/memory_server.py b/mcp_servers/memory_server.py index fafbcfc2b..fd574fd1f 100644 --- a/mcp_servers/memory_server.py +++ b/mcp_servers/memory_server.py @@ -17,6 +17,8 @@ from mcp.types import Tool, TextContent sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from src.memory import MemoryStoreUnreadable + server = Server("memory") # Late-initialized managers (set during first tool call) @@ -29,6 +31,10 @@ _OWNER_SCOPE_ERROR = ( "Error: Memory MCP owner is not configured for an owner-scoped memory store. " "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool." ) +_UNREADABLE_STORE_ERROR = ( + "Error: Memory store is temporarily unreadable — nothing was saved. " + "Repair or restore memory.json, then retry." +) def _configured_owner() -> str | None: @@ -51,9 +57,21 @@ def _owner_scoped_store(entries: list[dict]) -> bool: return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict)) -def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]: - """Return configured owner, all entries, visible entries, and optional error.""" - entries = _memory_manager.load_all() +def _scope_entries(for_update: bool = False) -> tuple[str | None, list[dict], list[dict], str | None]: + """Return configured owner, all entries, visible entries, and optional error. + + ``for_update=True`` is for read-modify-write callers. They save the ``all + entries`` list back, so an unreadable store must be reported as an error + instead of degrading to ``[]`` — otherwise the save writes their one new + entry over the whole store (issue #5673). + """ + if for_update: + try: + entries = _memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + return None, [], [], f"{_UNREADABLE_STORE_ERROR} ({e})" + else: + entries = _memory_manager.load_all() owner = _configured_owner() if owner is None and _owner_scoped_store(entries): return None, entries, [], _OWNER_SCOPE_ERROR @@ -161,7 +179,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: category = arguments.get("category", "fact") if not text: return _text_result("Error: Memory text cannot be empty") - owner, memories, _visible, scope_error = _scope_entries() + owner, memories, _visible, scope_error = _scope_entries(for_update=True) if scope_error: return _text_result(scope_error) entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) diff --git a/requirements.txt b/requirements.txt index be5f5d450..3c5114f53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,10 @@ python-dateutil caldav cryptography bcrypt -mcp +# Built-in servers use the v1 low-level Server decorator API. MCP SDK v2 is a +# breaking rewrite, so keep fresh installs on the maintained v1 line until the +# servers are migrated together. +mcp<2 pyotp qrcode[pil] croniter diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 5c7a4e04a..c0c370561 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -345,9 +345,61 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: # docs, email accounts, tasks, etc. try: from sqlalchemy import func - from core.database import Base, SessionLocal + from core.database import ( + Base, + EmailAccount, + SessionLocal, + lock_email_account_owner_mutations, + ) db = SessionLocal() try: + # Email-account defaults are protected by per-owner mutex rows. + # A rename crosses two owner partitions, so lock both in the + # shared helper's canonical order before inspecting either. + lock_email_account_owner_mutations( + db, old_username, new_username + ) + + source_default_ids = [ + row[0] + for row in ( + db.query(EmailAccount.id) + .filter( + func.lower(EmailAccount.owner) == old_username, + EmailAccount.is_default == True, # noqa: E712 + ) + .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc()) + .all() + ) + ] + destination_default_ids = [ + row[0] + for row in ( + db.query(EmailAccount.id) + .filter( + func.lower(EmailAccount.owner) == new_username, + EmailAccount.is_default == True, # noqa: E712 + ) + .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc()) + .all() + ) + ] + if destination_default_ids: + clear_default_ids = ( + destination_default_ids[1:] + source_default_ids + ) + else: + clear_default_ids = source_default_ids[1:] + if clear_default_ids: + ( + db.query(EmailAccount) + .filter(EmailAccount.id.in_(clear_default_ids)) + .update( + {EmailAccount.is_default: False}, + synchronize_session=False, + ) + ) + for mapper in Base.registry.mappers: model = mapper.class_ if not hasattr(model, "owner"): diff --git a/routes/backup_routes.py b/routes/backup_routes.py index 313369370..4ecf4f165 100644 --- a/routes/backup_routes.py +++ b/routes/backup_routes.py @@ -6,6 +6,7 @@ from datetime import datetime from fastapi import APIRouter, HTTPException, Request, Response from core.middleware import require_admin +from services.memory import MemoryStoreUnreadable from src.auth_helpers import get_current_user from src.settings import load_settings, save_settings, load_features, save_features @@ -76,7 +77,15 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo # ── Memories ── if "memories" in body and isinstance(body["memories"], list): - existing = memory_manager.load_all() + # Strict load: importing on top of an unreadable store would write + # only the incoming rows back and drop everything already saved. + try: + existing = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to import memories: %s", e) + raise HTTPException( + 503, "Memory store is temporarily unreadable — nothing was imported." + ) # Dedup against THIS user's own memories only. Using every tenant's # rows (load_all) meant a memory whose text matched any other # user's was silently skipped, so the importing user lost their own diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index 6e0ee124c..b9c3b0a52 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -10,6 +10,7 @@ from typing import Optional, List from fastapi import APIRouter, HTTPException, Request, UploadFile, File from pydantic import BaseModel from sqlalchemy import or_, and_ +from sqlalchemy.exc import IntegrityError from dateutil.rrule import rrulestr from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent @@ -221,22 +222,125 @@ class EventUpdate(BaseModel): # ── Helpers ── +_DEFAULT_CALENDAR_NAMESPACE = uuid.UUID("4840613a-9847-4a3b-bd75-19e6bc5fc3ce") + + +def _default_calendar_id(owner: str, collision_index: int = 0) -> str: + """Return one stable primary-key candidate for an owner's lazy default. + + Slot zero preserves the original owner-derived identifier. Later slots + let a username be reused after its prior calendar was migrated to another + owner during a rename, without making concurrent first use choose random + and therefore divergent identifiers. + """ + if collision_index == 0: + candidate_name = owner + else: + candidate_name = json.dumps( + [owner, collision_index], + ensure_ascii=False, + separators=(",", ":"), + ) + return str(uuid.uuid5(_DEFAULT_CALENDAR_NAMESPACE, candidate_name)) + + +def _begin_sqlite_default_write(db) -> None: + """Serialize an absent-default check with other SQLite writers. + + SQLite's default deferred transactions allow two workers to both read an + empty calendar set before either writes. ``BEGIN IMMEDIATE`` acquires the + writer reservation before the second, authoritative lookup. We issue it + only when the driver has not already opened a write transaction; a caller + with a pending write already owns the required reservation. + """ + connection = db.connection() + dbapi_connection = connection.connection + driver_connection = getattr( + dbapi_connection, + "driver_connection", + dbapi_connection, + ) + if not getattr(driver_connection, "in_transaction", False): + connection.exec_driver_sql("BEGIN IMMEDIATE") + + def _ensure_default_calendar(db, owner: str = None) -> CalendarCal: - """Create default calendar if none exist for this owner.""" + """Return the owner's calendar, staging a default in the caller's transaction. + + A stable owner-derived primary key makes concurrent first-use inserts + converge on one row on every SQL backend. SQLite additionally serializes + the absent-row check because its deferred transactions otherwise permit + both workers to read the gap before either writes. Other backends recover + a lost insert race inside a savepoint so the caller's event transaction + remains usable and atomic. + """ owner = owner or FALLBACK_OWNER cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first() - if not cal: + if cal: + return cal + + dialect = db.get_bind().dialect.name + if dialect == "sqlite": + _begin_sqlite_default_write(db) + # Another worker may have committed while BEGIN IMMEDIATE waited. + cal = db.query(CalendarCal).filter(CalendarCal.owner == owner).first() + if cal: + return cal + + collision_index = 0 + while True: + default_id = _default_calendar_id(owner, collision_index) + + if dialect == "sqlite": + # BEGIN IMMEDIATE above makes this occupancy check authoritative: + # another SQLite writer cannot rename, delete, or claim this slot + # until the caller commits or rolls back. + occupant = db.query(CalendarCal).filter( + CalendarCal.id == default_id, + ).first() + if occupant is not None: + if occupant.owner == owner: + return occupant + collision_index += 1 + continue + cal = CalendarCal( - id=str(uuid.uuid4()), + id=default_id, owner=owner, name="Personal", color="#5b8abf", source="local", ) - db.add(cal) - db.commit() - db.refresh(cal) - return cal + + if dialect == "sqlite": + db.add(cal) + db.flush() + return cal + + try: + # A uniqueness failure rolls back only this savepoint, not an event + # or reminder already staged by the caller's outer transaction. + with db.begin_nested(): + db.add(cal) + db.flush() + return cal + except IntegrityError: + # Use a locking/current read so repeatable-read backends can observe + # the row that won after our transaction's original empty snapshot. + occupant = db.query(CalendarCal).filter( + CalendarCal.id == default_id, + ).with_for_update().first() + if occupant is None: + # Do not misclassify an unrelated integrity failure as an ID + # collision and loop forever. A concurrently deleted winner is + # safe for the caller to retry as a fresh transaction. + raise + if occupant.owner == owner: + return occupant + # A renamed calendar owns this deterministic slot. Advance to the + # next stable slot; concurrent callers for this owner will still + # converge there. + collision_index += 1 # Per-request user time context. chat_routes sets this from browser timezone @@ -1015,6 +1119,9 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: db = SessionLocal() try: _ensure_default_calendar(db, owner) + # Listing calendars intentionally lazily creates a durable default. + # Other callers commit it with the event they are creating. + db.commit() cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all() return {"calendars": [ {"name": c.name, "href": c.id, "color": c.color, "source": c.source} @@ -1023,6 +1130,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: except HTTPException: raise except Exception as e: + db.rollback() logger.error("Failed to list calendars: %s", e) raise HTTPException(500, "Failed to list calendars") finally: diff --git a/routes/chat_routes.py b/routes/chat_routes.py index b081d5f1c..6099e72fd 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -21,6 +21,7 @@ from src import agent_runs from src.model_context import estimate_tokens from src.chat_helpers import coerce_message_and_session from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_url +from src.foreground_model_routing import build_foreground_model_candidates from src.session_search import search_session_messages from src.prompt_security import untrusted_context_message from core.exceptions import SessionNotFoundError @@ -1399,14 +1400,14 @@ def setup_chat_routes( thinking_response = "" last_metrics = None - # Configured fallback chain for the default chat model. Tried in - # order if the session's primary model fails before producing - # output. Resolved once per request. - try: - from src.endpoint_resolver import resolve_chat_fallback_candidates - _fallback_candidates = resolve_chat_fallback_candidates(owner=_user) - except Exception: - _fallback_candidates = [] + # Foreground Chat and Agent requests use one owner-aware policy + # boundary. Legacy `default_model_fallbacks` data is not eligible. + _foreground_candidates = build_foreground_model_candidates( + sess.endpoint_url, + sess.model, + sess.headers, + owner=_user, + ) # Send model name early so the frontend can show it during streaming _model_suffix = "Research" if effective_do_research else None @@ -1522,9 +1523,8 @@ def setup_chat_routes( _actual_model = None # ── Chat mode: call stream_llm directly, NO tools, NO document access ── try: - _chat_candidates = [(sess.endpoint_url, sess.model, sess.headers)] + _fallback_candidates async for chunk in stream_llm_with_fallback( - _chat_candidates, + _foreground_candidates, messages, temperature=ctx.preset.temperature, # Respect the preset; 0/unset = let the server decide (no @@ -1710,7 +1710,7 @@ def setup_chat_routes( disabled_tools=disabled_tools if disabled_tools else None, tool_policy=tool_policy, owner=_user, - fallbacks=_fallback_candidates, + fallbacks=_foreground_candidates[1:], plan_mode=plan_mode, approved_plan=approved_plan or None, workspace=workspace or None, diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 1d79ba809..d51fb9a09 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -73,6 +73,30 @@ _HF_TOKEN_STATUS_SNIPPET = ( ) +def _windows_local_pid_record_line(pid_path: Path, ready_path: Path) -> str: + """Build the Git Bash prelude that records a Win32-stoppable PID. + + Python publishes the detached outer process's Win32 PID first, then touches + ``ready_path``. The inner Git Bash runner waits for that publication before + replacing the fallback with its own Win32 PID from /proc//winpid. + + Missing, malformed, or late mappings leave the valid outer PID untouched. + """ + pp = shlex.quote(pid_path.as_posix()) + rp = shlex.quote(ready_path.as_posix()) + return ( + "i=0; " + f"while [ ! -e {rp} ] && [ \"$i\" -lt 500 ]; do " + "i=$((i+1)); sleep 0.01; done; " + f"if [ -e {rp} ]; then " + "winpid=\"$(cat /proc/$$/winpid 2>/dev/null || true)\"; " + "case \"$winpid\" in ''|*[!0-9]*) ;; " + f"*) printf '%s\\n' \"$winpid\" > {pp} ;; esac; " + "fi; " + f"rm -f {rp}" + ) + + def _append_mlx_image_server_script(runner_lines: list[str]) -> None: """Write the MLX image API helper next to the tmux runner on remote hosts.""" script_path = Path(__file__).resolve().parents[1] / "scripts" / "mlx_image_server.py" @@ -978,15 +1002,18 @@ def setup_cookbook_routes() -> APIRouter: directly (simple commands only). Returns the launched job record.""" log_path = TMUX_LOG_DIR / f"{session_id}.log" pid_path = TMUX_LOG_DIR / f"{session_id}.pid" + pid_ready_path: Path | None = None bash = find_bash() if bash: # Run the existing bash wrapper verbatim through Git Bash, redirecting # all output to the log the poller reads. Paths handed to bash use # POSIX form + shell-quoting so drive paths / spaces survive. inner = TMUX_LOG_DIR / f"{session_id}_run.sh" - pp = shlex.quote(pid_path.as_posix()) + pid_ready_path = TMUX_LOG_DIR / f"{session_id}.pid.ready" + pid_ready_path.unlink(missing_ok=True) inner.write_text( - f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n", + _windows_local_pid_record_line(pid_path, pid_ready_path) + "\n" + + "\n".join(bash_lines) + "\n", encoding="utf-8", ) lp = shlex.quote(log_path.as_posix()) @@ -1020,7 +1047,18 @@ def setup_cookbook_routes() -> APIRouter: env=env, **detached_popen_kwargs(), ) + # Publish a valid Win32 ancestor first. The Git Bash runner may then + # replace it with its own Win32 pid, but never before this fallback exists. pid_path.write_text(str(proc.pid), encoding="utf-8") + if pid_ready_path is not None: + try: + pid_ready_path.touch() + except OSError as e: + logger.warning( + "Could not publish Windows local PID handoff for %s: %s", + session_id, + e, + ) return {"pid": proc.pid, "log_path": str(log_path)} @router.post("/api/model/download") diff --git a/routes/document/__init__.py b/routes/document/__init__.py new file mode 100644 index 000000000..7f79ce1bb --- /dev/null +++ b/routes/document/__init__.py @@ -0,0 +1,6 @@ +"""Document route domain package (slice 2m, #4082/#4071). + +Contains document_routes.py and document_helpers.py, migrated from the flat +routes/ directory. Backward-compat shims at routes/document_routes.py and +routes/document_helpers.py re-export from here. +""" diff --git a/routes/document/document_helpers.py b/routes/document/document_helpers.py new file mode 100644 index 000000000..a0c2d08eb --- /dev/null +++ b/routes/document/document_helpers.py @@ -0,0 +1,243 @@ +"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py.""" + +"""Document routes — CRUD for living documents with version history.""" + +import logging +import os +import re +from typing import Any, Dict, Optional + +from fastapi import HTTPException, Request +from pydantic import BaseModel + +from core.database import Document, DocumentVersion +from core.database import Session as DbSession +from src.auth_helpers import _auth_disabled +from src.upload_handler import UploadHandler + +logger = logging.getLogger(__name__) + + +# ---- Request schemas ---- + +class DocumentCreate(BaseModel): + session_id: Optional[str] = None + title: str = "Untitled" + language: Optional[str] = None + content: str = "" + +class DocumentUpdate(BaseModel): + content: str + summary: Optional[str] = None + force_version: bool = False + +class DocumentPatch(BaseModel): + title: Optional[str] = None + language: Optional[str] = None + session_id: Optional[str] = None # link/unlink document to a session + + +# ---- Helpers ---- + +def _doc_to_dict(doc: Document) -> Dict[str, Any]: + return { + "id": doc.id, + "session_id": doc.session_id, + "title": doc.title, + "language": doc.language, + "current_content": doc.current_content, + "version_count": doc.version_count, + "is_active": doc.is_active, + "archived": bool(getattr(doc, "archived", False)), + "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, + "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, + # Source-email provenance (set when doc was created from an email + # attachment) — drives the "Send signed reply" menu item. + "source_email_uid": getattr(doc, "source_email_uid", None), + "source_email_folder": getattr(doc, "source_email_folder", None), + "source_email_account_id": getattr(doc, "source_email_account_id", None), + "source_email_message_id": getattr(doc, "source_email_message_id", None), + } + +def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]: + return { + "id": v.id, + "document_id": v.document_id, + "version_number": v.version_number, + "content": v.content, + "summary": v.summary, + "source": v.source, + "created_at": v.created_at.isoformat() if v.created_at else None, + } + + +def _verify_doc_owner(db, doc: Document, user: str): + """Verify `user` owns this document. Raise 404 if not. + + Documents now carry their own `owner` column, so a doc whose session + was deleted (session_id → NULL) can still prove ownership and stay + openable / cloneable. We trust that column first and only fall back to + the session join for any not-yet-backfilled legacy row. + """ + if user is None: + if _auth_disabled(): + return # Single-user / no-auth mode: allow access + raise HTTPException(403, "Authentication required") + if doc.owner is not None: + if doc.owner != user: + raise HTTPException(404, "Document not found") + return + # Legacy fallback: derive ownership from the linked session. + if not doc.session_id: + raise HTTPException(404, "Document not found") + session = db.query(DbSession).filter(DbSession.id == doc.session_id).first() + if not session or session.owner != user: + raise HTTPException(404, "Document not found") + + +def _owner_session_filter(q, user): + """Restrict a documents query to those owned by `user`. + + Documents now carry their own `owner` column (backfilled at boot from + the linked session, or assigned to the admin user for legacy/orphaned + docs). We filter on that directly rather than on a session join, so a + document whose session was deleted (session_id → NULL) still shows up + for its owner instead of silently vanishing from the Library + search. + + The owner backfill runs in init_db before the app serves requests, so + by the time this filter is live there are no NULL-owner rows to leak; + we therefore match the owner strictly for authenticated callers.""" + if not user: + if user == "" or _auth_disabled(): + return q + return q.filter(False) + return q.filter(Document.owner == user) + + + +def _slug(name: str) -> str: + """Filesystem-friendly version of a document title. + + Whitespace becomes underscores; other unsafe punctuation is dropped. + Preserves letters, digits, dot, hyphen, underscore. Idempotent. + """ + import re as _re + s = (name or "").strip() + # Drop the trailing extension if the title happens to include one + s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE) + s = _re.sub(r'\s+', '_', s) + s = _re.sub(r'[^A-Za-z0-9._-]', '', s) + s = _re.sub(r'_+', '_', s).strip('_') + return s or "form" + + +# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units). +_PDF_RENDER_SCALE = 2.0 + + +def _upload_path_inside(upload_dir: str, path: str) -> bool: + base = os.path.realpath(upload_dir) + p = os.path.realpath(path) + try: + return os.path.commonpath([base, p]) == base + except Exception: + return False + + +def _resolve_user_upload_path( + upload_handler: Any, + upload_id: str, + owner: Optional[str], + auth_manager=None, +) -> Optional[str]: + """Resolve an upload id to a filesystem path the caller may read.""" + if upload_handler is None: + return None + resolved = upload_handler.resolve_upload( + upload_id, + owner=owner, + auth_manager=auth_manager, + ) + if not isinstance(resolved, dict) or not resolved: + return None + path = resolved.get("path") + upload_dir = getattr(upload_handler, "upload_dir", None) + if path and upload_dir and not _upload_path_inside(upload_dir, path): + logger.warning("Upload path outside upload directory: %s", path) + return None + return path + + +def _locate_upload( + upload_dir: str, + file_id: str, + owner: Optional[str] = None, + auth_manager=None, + upload_handler: Any = None, +): + """Find an upload by its filename ID via UploadHandler.resolve_upload.""" + if upload_handler is None: + from src.upload_handler import UploadHandler + + base_dir = os.path.dirname(os.path.abspath(upload_dir)) + upload_handler = UploadHandler(base_dir, upload_dir) + return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager) + + +def _assert_pdf_marker_upload_owned( + request: Request, + content: str, + user: Optional[str], + upload_handler: Any, +) -> None: + """Reject document content whose pdf_source marker points at another user's upload.""" + if upload_handler is None: + return + from src.pdf_form_doc import find_source_upload_id + + upload_id = find_source_upload_id(content or "") + if not upload_id: + return + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager): + raise HTTPException( + 400, + "Document PDF marker references an upload you do not own", + ) + + +def _derive_title(content: str) -> str: + """Derive a title from document content.""" + import re + if not isinstance(content, str): + return "Untitled" + text = content.strip() + if not text: + return "Untitled" + + # Markdown header + md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE) + if md: + title = md.group(1).strip() + if len(title) > 50: + title = title[:48] + "…" + return title + + # HTML heading + html = re.search(r']*>([^<]+)', text, re.IGNORECASE) + if html: + title = html.group(1).strip() + if len(title) > 50: + title = title[:48] + "…" + return title + + # First non-empty line (if short enough) + for line in text.split('\n'): + line = line.strip() + if line and 2 <= len(line) <= 60: + title = re.sub(r'[:#*`]+$', '', line).strip() + if title and len(title) > 50: + title = title[:48] + "…" + return title or "Untitled" + + return "Untitled" diff --git a/routes/document/document_routes.py b/routes/document/document_routes.py new file mode 100644 index 000000000..dae8b09fa --- /dev/null +++ b/routes/document/document_routes.py @@ -0,0 +1,1810 @@ +"""Document routes — CRUD for living documents with version history.""" + +import uuid +import logging +from datetime import datetime, timezone +from typing import Dict, Any, List, Optional + +from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form + +from sqlalchemy import case, func, or_ +from core.database import SessionLocal, Document, DocumentVersion +from core.database import Session as DbSession +from src.auth_helpers import get_current_user, _auth_disabled +from src.constants import MAIL_ATTACHMENTS_DIR +from src.upload_handler import reserve_upload_references + +logger = logging.getLogger(__name__) + + +def _get_session_or_404(db, session_id: str, user: Optional[str]): + session = db.query(DbSession).filter(DbSession.id == session_id).first() + if not session: + raise HTTPException(404, "Session not found") + if user and session.owner != user: + raise HTTPException(404, "Session not found") + return session + + +def _aggregate_language_facets(lang_rows): + """Sum document counts per display language for the library facet. + + NULL-language and explicit "text" rows share the "text" bucket (the + language filter treats them as one), so they must be ADDED. The old dict + comprehension keyed both to "text", silently overwriting one group and + undercounting the facet versus what the filter actually returns. + """ + out = {} + for lang, cnt in lang_rows: + key = lang or "text" + out[key] = out.get(key, 0) + cnt + return out + + +def _library_language_for_document(doc: Document) -> str: + """Return the display language used by the document library. + + PDF documents are stored as markdown wrappers so the editor can preserve + extracted text, form fields, and annotations. The library should still + identify them as PDFs instead of exposing that internal wrapper format. + """ + from src.pdf_form_doc import find_source_upload_id + + if find_source_upload_id(doc.current_content or ""): + return "pdf" + return doc.language or "text" + + +def _email_source_key(content: str) -> tuple[str, str]: + """Return the source email identity embedded in an email draft document.""" + import re + + text = content or "" + uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text) + folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text) + uid = (uid_m.group(1).strip() if uid_m else "") + folder = (folder_m.group(1).strip() if folder_m else "INBOX") + return uid, folder + + +from routes.document_helpers import ( + DocumentCreate, DocumentUpdate, DocumentPatch, + _doc_to_dict, _version_to_dict, + _verify_doc_owner, _owner_session_filter, + _slug, _resolve_user_upload_path, _assert_pdf_marker_upload_owned, _derive_title, + _PDF_RENDER_SCALE, +) + + +def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: + router = APIRouter(tags=["documents"]) + + def _reserve_document_uploads(user: Optional[str], content: str) -> None: + missing_id = reserve_upload_references(upload_handler, user, content) + if missing_id: + raise HTTPException( + 409, + f"Referenced upload is no longer available: {missing_id}", + ) + + def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]): + if upload_handler is None: + return None + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + return _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager) + + def _load_pdf_viewer_fitz(): + from src.pdf_runtime import load_pymupdf_for_pdf_viewer + + try: + return load_pymupdf_for_pdf_viewer() + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + + # ---- POST /api/document ---- + @router.post("/api/document") + async def create_document(request: Request, req: DocumentCreate) -> Dict[str, Any]: + from src.auth_helpers import require_privilege + user = require_privilege(request, "can_use_documents") + db = SessionLocal() + try: + # session_id is optional: a doc can be a session-less "library" doc + # (e.g. files imported from the library) — session_id is nullable and + # the doc is owner-stamped, so it lives in the library on its own. + session = None + if req.session_id: + # Match the lenient ownership model the rest of the app uses + # (see _owner_filter): only block when an AUTHENTICATED user is + # writing into a DIFFERENT user's session. In single-user / + # unconfigured / localhost-bypass mode, falsey users preserve + # the existing lenient path. + session = _get_session_or_404(db, req.session_id, user) + + # If no language was supplied (e.g. cloning a doc whose language + # was never set), detect it from the content rather than storing + # NULL — which made the editor fall back to plain text. Defaults + # to markdown for prose. + language = req.language + if not language: + from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content + language = _sniff_doc_language(req.content) + else: + from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content + if _looks_like_email_document(req.content, req.title): + language = "email" + + _reserve_document_uploads(user, req.content) + _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) + + # Reply drafts are keyed to the source email. If a UI/tool path tries + # to create a second draft for the same email in the same chat, + # update the existing draft instead so quoted thread history stays + # attached to the visible document. + if language == "email" and req.session_id: + source_uid, source_folder = _email_source_key(req.content) + if source_uid: + candidates = ( + db.query(Document) + .filter(Document.session_id == req.session_id) + .filter(Document.is_active == True) + .filter(Document.language == "email") + .order_by(Document.updated_at.desc()) + .limit(25) + .all() + ) + for existing in candidates: + old_uid, old_folder = _email_source_key(existing.current_content or "") + if old_uid != source_uid or old_folder != source_folder: + continue + merged = _coerce_email_document_content(existing.current_content or "", req.content) + if existing.current_content != merged: + new_ver = (existing.version_count or 1) + 1 + existing.current_content = merged + existing.title = req.title or existing.title + existing.version_count = new_ver + db.add(DocumentVersion( + id=str(uuid.uuid4()), + document_id=existing.id, + version_number=new_ver, + content=merged, + summary="Updated existing email draft", + source="user", + )) + db.commit() + db.refresh(existing) + return _doc_to_dict(existing) + + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + + doc = Document( + id=doc_id, + session_id=req.session_id, + title=req.title, + language=language, + current_content=req.content, + version_count=1, + is_active=True, + # Stamp ownership directly so the doc survives its session + # being deleted. Fall back to the session's owner when the + # request is unauthenticated (single-user / localhost bypass). + owner=user or (session.owner if session else None), + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=req.content, + summary="Initial version", + source="user", + ) + db.add(doc) + db.add(ver) + db.commit() + db.refresh(doc) + try: + from src.event_bus import fire_event + fire_event("document_created", doc.owner) + except Exception: + logger.debug("document_created event dispatch failed", exc_info=True) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"Failed to create document: {e}") + raise HTTPException(500, f"Failed to create document: {e}") + finally: + db.close() + + # ---- POST /api/documents/import-pdf ---- + @router.post("/api/documents/import-pdf") + async def import_pdf( + request: Request, + file: UploadFile = File(...), + session_id: Optional[str] = Form(None), + ) -> Dict[str, Any]: + """Upload a PDF and create the matching Document. + + Detects AcroForm fields — if any, creates a form-backed markdown doc + (clickable inputs in the PDF view). Otherwise creates a plain PDF doc + with a `pdf_source` marker so the viewer renders the pages without + overlays. + """ + from src.pdf_forms import has_form_fields, extract_fields + from src.pdf_form_doc import ( + save_field_sidecar, + create_form_markdown_document, + create_plain_pdf_document, + ) + from src.document_processor import _process_pdf, strip_pdf_content_marker + import os + + from src.auth_helpers import require_privilege + user = require_privilege(request, "can_use_documents") + + # session_id is optional — a library import isn't tied to a chat. When + # given, validate it; otherwise the PDF becomes a session-less library + # doc (the doc creators below already handle a missing session). + if session_id: + db = SessionLocal() + try: + _get_session_or_404(db, session_id, user) + finally: + db.close() + + if upload_handler is None: + raise HTTPException(500, "Upload handler not configured") + + client_ip = request.client.host if request.client else "unknown" + try: + meta = upload_handler.save_upload(file, client_ip, owner=user) + except HTTPException: + raise + except Exception as e: + logger.error(f"PDF import save_upload failed: {e}") + raise HTTPException(500, f"Upload failed: {e}") + + upload_id = meta["id"] + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(500, "Saved PDF could not be located") + + title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0] + try: + body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) + except Exception: + body_text = None + + is_form = False + try: + is_form = has_form_fields(pdf_path) + except Exception as e: + logger.warning(f"has_form_fields failed for {pdf_path}: {e}") + + if is_form: + fields = extract_fields(pdf_path) + save_field_sidecar(pdf_path, fields) + doc_id = create_form_markdown_document( + session_id=session_id, + fields=fields, + upload_id=upload_id, + title=title, + intro_text=body_text, + ) + else: + doc_id = create_plain_pdf_document( + session_id=session_id, + upload_id=upload_id, + title=title, + body_text=body_text, + ) + + if not doc_id: + raise HTTPException(500, "Failed to create document for PDF") + + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(500, "Created document not found") + # The PDF doc creators stamp owner from the session only; a + # session-less library import leaves owner NULL, which the Library's + # owner filter then hides. Stamp the requesting user so it shows. + if not doc.owner and user: + doc.owner = user + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + finally: + db.close() + + # ---- GET /api/documents/library ---- + @router.get("/api/documents/library") + async def documents_library( + request: Request, + search: Optional[str] = Query(None), + language: Optional[str] = Query(None), + sort: str = Query("recent"), + offset: int = Query(0, ge=0), + limit: int = Query(20, ge=1, le=50), + archived: bool = Query(False), + ) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + from sqlalchemy import or_ + pdf_marker_cond = or_( + Document.current_content.like('%\s*\n+#[^\n]*\n+)', re.MULTILINE) + head_match = head_re.match(content) + head = head_match.group(1) if head_match else (content.splitlines()[0] + "\n\n# " + (doc.title or "PDF") + "\n\n") + doc.current_content = head + body_text.strip() + "\n" + doc.version_count = (doc.version_count or 1) + 1 + db.add(DocumentVersion( + id=str(__import__("uuid").uuid4()), + document_id=doc_id, + version_number=doc.version_count, + content=doc.current_content, + summary="PDF text re-extracted (OCR)", + source="ocr", + )) + db.commit() + return {"ok": True, "id": doc_id, "extracted": True, "chars": len(body_text)} + finally: + db.close() + + # ---- POST /api/documents/export-zip — bundle selected docs into a .zip ---- + @router.post("/api/documents/export-zip") + async def documents_export_zip(request: Request): + """Zip the selected documents (each as a text file with the right + extension) — mirrors the gallery's bulk download-zip so multi-export + is one file instead of a blocked flood of individual downloads.""" + user = get_current_user(request) + try: + data = await request.json() + except Exception as e: + logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) + data = {} + ids = data.get("ids") or [] + if not ids: + raise HTTPException(400, "No documents specified") + _ext = { + "javascript": ".js", "python": ".py", "html": ".html", "css": ".css", + "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh", + "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c", + "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php", + "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini", + } + db = SessionLocal() + try: + import io + import re + import zipfile + from fastapi import Response + docs = db.query(Document).filter(Document.id.in_(ids)).all() + buf = io.BytesIO() + used = set() + wrote = 0 + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for doc in docs: + try: + _verify_doc_owner(db, doc, user) + except HTTPException: + continue # skip docs the user doesn't own + ext = _ext.get(doc.language or "text", ".txt") + base = (doc.title or "document").strip() or "document" + base = re.sub(r"[^\w\-. ]+", "", base)[:60].strip() or doc.id + name = base if "." in base else base + ext + i = 1 + while name in used: + name = f"{base}-{i}" + ("" if "." in base else ext) + i += 1 + used.add(name) + zf.writestr(name, doc.current_content or "") + wrote += 1 + if not wrote: + raise HTTPException(404, "No documents found") + return Response( + content=buf.getvalue(), + media_type="application/zip", + headers={"Content-Disposition": 'attachment; filename="documents.zip"'}, + ) + finally: + db.close() + + # ---- PUT /api/document/{doc_id} — user manual edit ---- + # Coalesce window: if the last user version was saved within this many + # seconds, update it in-place (user is still actively editing). + # Once the gap exceeds this, the next save creates a new version. + VERSION_COALESCE_SECONDS = 60 + + @router.put("/api/document/{doc_id}") + async def update_document(request: Request, doc_id: str, req: DocumentUpdate) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + incoming_content = req.content + from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document + is_email_doc = ( + (doc.language or "").lower() == "email" + or _looks_like_email_document(doc.current_content or "", doc.title or "") + or _looks_like_email_document(req.content or "", doc.title or "") + ) + if is_email_doc: + incoming_content = _coerce_email_document_content(doc.current_content or "", req.content) + doc.language = "email" + + # Skip if content is identical unless the caller explicitly wants + # a checkpoint version from the current editor state. + if doc.current_content == incoming_content and not req.force_version: + return _doc_to_dict(doc) + + _reserve_document_uploads(user, incoming_content) + _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler) + + # Check if we can coalesce with the latest version + latest_ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + ).order_by(DocumentVersion.version_number.desc()).first() + + now = datetime.now(timezone.utc) + coalesced = False + if latest_ver and latest_ver.source == "user" and not req.force_version: + ver_time = latest_ver.created_at + if ver_time.tzinfo is None: + ver_time = ver_time.replace(tzinfo=timezone.utc) + age = (now - ver_time).total_seconds() + if age < VERSION_COALESCE_SECONDS: + # Update the existing version in-place + latest_ver.content = incoming_content + latest_ver.created_at = now + if req.summary: + latest_ver.summary = req.summary + coalesced = True + + if not coalesced: + new_ver = doc.version_count + 1 + ver = DocumentVersion( + id=str(uuid.uuid4()), + document_id=doc_id, + version_number=new_ver, + content=incoming_content, + summary=req.summary or "Manual edit", + source="user", + ) + doc.version_count = new_ver + db.add(ver) + + doc.current_content = incoming_content + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, f"Failed to update document: {e}") + finally: + db.close() + + # ---- PATCH /api/document/{doc_id} — metadata only ---- + @router.patch("/api/document/{doc_id}") + async def patch_document(request: Request, doc_id: str, req: DocumentPatch) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + if req.title is not None: + doc.title = req.title + if req.language is not None: + doc.language = req.language + if req.session_id is not None: + # Empty string = unlink from session + if req.session_id: + _get_session_or_404(db, req.session_id, user) + doc.session_id = req.session_id if req.session_id else None + if not req.session_id: + # Tab closed / doc detached from its session — drop the + # in-memory active-doc pointer so the last-resort injection + # path doesn't re-surface this doc in a later chat (#1160). + try: + from src.agent_tools.document_tools import clear_active_document + clear_active_document(doc_id) + except Exception as e: + logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- DELETE /api/document/{doc_id} — soft delete ---- + @router.delete("/api/document/{doc_id}") + async def delete_document(request: Request, doc_id: str) -> Dict[str, str]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + doc.is_active = False + # Closed/deleted — drop the in-memory active-doc pointer so it isn't + # re-injected into a later, unrelated chat (#1160). + try: + from src.agent_tools.document_tools import clear_active_document + clear_active_document(doc_id) + except Exception: + pass + db.commit() + return {"status": "deleted", "id": doc_id} + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- GET /api/document/{doc_id}/versions ---- + @router.get("/api/document/{doc_id}/versions") + async def list_versions(request: Request, doc_id: str) -> List[Dict[str, Any]]: + user = get_current_user(request) + db = SessionLocal() + try: + # Verify ownership before listing versions + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + versions = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id + ).order_by(DocumentVersion.version_number.desc()).all() + return [{ + "id": v.id, + "version_number": v.version_number, + "content": v.content, + "summary": v.summary, + "source": v.source, + "created_at": v.created_at.isoformat() if v.created_at else None, + } for v in versions] + finally: + db.close() + + # ---- GET /api/document/{doc_id}/version/{num} ---- + @router.get("/api/document/{doc_id}/version/{num}") + async def get_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + # Verify ownership + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + DocumentVersion.version_number == num, + ).first() + if not ver: + raise HTTPException(404, "Version not found") + return _version_to_dict(ver) + finally: + db.close() + + # ---- POST /api/document/{doc_id}/restore/{num} ---- + @router.post("/api/document/{doc_id}/restore/{num}") + async def restore_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + old_ver = db.query(DocumentVersion).filter( + DocumentVersion.document_id == doc_id, + DocumentVersion.version_number == num, + ).first() + if not old_ver: + raise HTTPException(404, "Version not found") + + new_ver_num = doc.version_count + 1 + ver = DocumentVersion( + id=str(uuid.uuid4()), + document_id=doc_id, + version_number=new_ver_num, + content=old_ver.content, + summary=f"Restored from v{num}", + source="user", + ) + doc.current_content = old_ver.content + doc.version_count = new_ver_num + db.add(ver) + db.commit() + db.refresh(doc) + return _doc_to_dict(doc) + except HTTPException: + raise + except Exception as e: + db.rollback() + raise HTTPException(500, str(e)) + finally: + db.close() + + # ---- POST /api/documents/tidy — clean up broken/empty documents ---- + @router.post("/api/documents/tidy") + async def tidy_documents(request: Request) -> Dict[str, Any]: + """Fix empty titles and remove broken/empty documents (user's docs only).""" + user = get_current_user(request) + db = SessionLocal() + try: + q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True) + .filter((Document.archived == False) | (Document.archived.is_(None))) + ) + q = _owner_session_filter(q, user) + docs = q.all() + fixed_titles = 0 + deleted = 0 + + # Same junk-detection logic as the scheduled tidy_documents + # action (src/document_actions.py). Keep these two in sync. + import re as _re + from src.document_actions import _JUNK_TITLES + + to_delete = [] + now = datetime.now(timezone.utc) + for doc in docs: + created = doc.created_at + if created and created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + + # Skip freshly created documents to avoid deleting them while the user is actively editing + if created and (now - created).total_seconds() < 900: # 15 minutes + continue + + content = (doc.current_content or "").strip() + title_raw = (doc.title or "").strip() + title = title_raw.lower() + is_fresh_empty = ( + not content + and created is not None + and (now - created).total_seconds() < 1800 + ) + if is_fresh_empty: + continue + + # Strip markdown noise to get a "real" character count + stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE) + stripped = _re.sub(r"[*_`>\-=]+", "", stripped) + stripped = _re.sub(r"\s+", " ", stripped).strip() + real_len = len(stripped) + + # Detect email-scaffold stubs: "To: \nSubject: \n---\n" style + # bodies with nothing typed in. Stub = every meaningful line + # is a header label (To:/From:/Subject:/...) with no real + # value (blank, "empty", "(empty)", "-", "none", "n/a"). + _is_email_stub = False + _HEADER_RE = _re.compile(r"^(to|from|cc|bcc|subject|reply-to):\s*(.*)$", _re.I) + _PLACEHOLDER_VALS = {"", "empty", "(empty)", "-", "—", "none", "n/a", "na", "tbd"} + if title in ("new email", "new mail", "new message") or doc.language == "email": + body_lines = [ln.strip() for ln in content.split("\n") + if ln.strip() and ln.strip() != "---"] + def _is_filler(ln): + m = _HEADER_RE.match(ln) + if not m: + return False + val = (m.group(2) or "").strip().lower() + return val in _PLACEHOLDER_VALS + has_real_body = any(not _is_filler(ln) for ln in body_lines) + if body_lines and not has_real_body: + _is_email_stub = True + + # Hard-delete obviously empty / junk documents + if not content or content in ("", "# Untitled"): + to_delete.append(doc); deleted += 1; continue + if _is_email_stub: + to_delete.append(doc); deleted += 1; continue + if title in _JUNK_TITLES: + to_delete.append(doc); deleted += 1; continue + + # Fix empty or placeholder titles on survivors + if not title_raw or title_raw == "Untitled": + new_title = _derive_title(content) + if new_title and new_title != "Untitled": + doc.title = new_title + fixed_titles += 1 + + for doc in to_delete: + db.delete(doc) + + # Also clean up inactive empty docs from previous soft-deletes + inactive_q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == False) + .filter((Document.current_content == None) | (Document.current_content == "")) + ) + inactive_q = _owner_session_filter(inactive_q, user) + inactive_docs = inactive_q.all() + for doc in inactive_docs: + db.delete(doc) + deleted += len(inactive_docs) + + db.commit() + return { + "fixed_titles": fixed_titles, + "deleted": deleted, + "message": f"Fixed {fixed_titles} title{'s' if fixed_titles != 1 else ''}, removed {deleted} empty document{'s' if deleted != 1 else ''}", + } + except Exception as e: + db.rollback() + logger.error(f"Document tidy failed: {e}") + raise HTTPException(500, f"Tidy failed: {e}") + finally: + db.close() + + # ---- POST /api/documents/ai-tidy — AI-powered cleanup of junk/test documents ---- + @router.post("/api/documents/ai-tidy") + async def ai_tidy_documents(request: Request) -> Dict[str, Any]: + """Use AI to judge if documents are junk/test/accidental, then delete them. + Caches verdicts so previously-reviewed docs are skipped.""" + from src.task_endpoint import resolve_task_endpoint + from src.endpoint_resolver import resolve_endpoint + from src.llm_core import llm_call_async + + user = get_current_user(request) + url, model, headers = resolve_task_endpoint(owner=user or None) + if not url or not model: + # Fall back to default endpoint + url, model, headers = resolve_endpoint("default", owner=user or None) + if not url or not model: + raise HTTPException(500, "No endpoint configured for AI tidy") + + db = SessionLocal() + try: + q = ( + db.query(Document) + .outerjoin(DbSession, Document.session_id == DbSession.id) + .filter(Document.is_active == True) + .filter((Document.archived == False) | (Document.archived.is_(None))) + ) + q = _owner_session_filter(q, user) + docs = q.all() + + # Only review docs that haven't been reviewed yet + to_review = [d for d in docs if not d.tidy_verdict] + if not to_review: + return {"deleted": 0, "reviewed": 0, "message": "All documents already reviewed"} + + # Build a batch prompt — review up to 30 at a time + batch = to_review[:30] + doc_list = [] + for i, doc in enumerate(batch): + preview = (doc.current_content or "")[:300].strip() + doc_list.append(f"[{i}] title=\"{doc.title}\" lang={doc.language or 'text'} content_preview=\"{preview}\"") + + prompt = ( + "You are a document library cleaner. For each document below, decide if it is JUNK " + "(test, accidental, placeholder, empty-ish, tool-test, throwaway) or KEEP (real content worth saving).\n\n" + "Respond with ONLY a JSON array of verdicts, one per document, like: [\"junk\",\"keep\",\"junk\",...]\n" + "No explanation, no markdown, just the JSON array.\n\n" + + "\n".join(doc_list) + ) + + response = await llm_call_async( + url, model, + [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, + {"role": "user", "content": prompt}], + temperature=0.1, + max_tokens=200, + headers=headers, + timeout=30, + ) + + # Parse verdicts + import re + match = re.search(r'\[.*?\]', response, re.DOTALL) + if not match: + raise HTTPException(500, "AI returned invalid response") + + import json as _json + verdicts = _json.loads(match.group()) + + deleted = 0 + reviewed = 0 + for i, doc in enumerate(batch): + if i >= len(verdicts): + break + verdict = str(verdicts[i] or "").lower().strip() + if verdict == "junk": + doc.tidy_verdict = "junk" + db.delete(doc) + deleted += 1 + else: + doc.tidy_verdict = "keep" + reviewed += 1 + + db.commit() + return { + "deleted": deleted, + "reviewed": reviewed, + "remaining": len(to_review) - len(batch), + "message": f"Reviewed {reviewed}, removed {deleted} junk document{'s' if deleted != 1 else ''}", + } + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error(f"AI tidy failed: {e}") + raise HTTPException(500, f"AI tidy failed: {e}") + finally: + db.close() + + # ---- POST /api/document/{doc_id}/export-pdf/preview ---- + @router.post("/api/document/{doc_id}/export-pdf/preview") + async def export_pdf_preview(doc_id: str, request: Request) -> Dict[str, Any]: + """Return the field-value mapping that would be written to the PDF. + + Frontend shows this in a confirmation modal so the user can spot/fix + any wrong values before triggering the actual download. + """ + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") + + fields = load_field_sidecar(pdf_path) + if not fields: + raise HTTPException(404, "Field schema sidecar missing for source PDF") + + values = parse_markdown_to_values(doc.current_content or "") + field_meta = {f["name"]: f for f in fields} + + preview = [] + for name, current in values.items(): + meta = field_meta.get(name) + if not meta: + continue + preview.append({ + "name": name, + "label": meta.get("label") or name, + "type": meta.get("type"), + "options": meta.get("options") or [], + "page": meta.get("page"), + "value": current, + }) + + unknown = [ + name for name in values + if name not in field_meta + ] + return { + "doc_id": doc_id, + "upload_id": upload_id, + "fields": preview, + "unknown_fields": unknown, + "total": len(fields), + "filled": sum(1 for p in preview if p["value"] not in ("", False, None)), + } + finally: + db.close() + + # ---- GET /api/document/{doc_id}/render-pages ---- + @router.get("/api/document/{doc_id}/render-pages") + async def render_pages(doc_id: str, request: Request) -> Dict[str, Any]: + """Return per-page metadata for the interactive PDF view. + + Each page entry has its rendered-image dimensions (matching what + /page/{n}.png returns at the same DPI) plus the list of form fields + on that page with their rects translated to image-pixel coordinates. + Frontend overlays HTML form controls at those positions. + """ + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + fitz = _load_pdf_viewer_fitz() + schema = load_field_sidecar(pdf_path) or [] + values = parse_markdown_to_values(doc.current_content or "") + + # Group fields by page + by_page: Dict[int, list] = {} + for f in schema: + by_page.setdefault(f["page"], []).append(f) + + scale = _PDF_RENDER_SCALE + pdf_doc = fitz.open(pdf_path) + try: + pages_out = [] + for page_index in range(pdf_doc.page_count): + page = pdf_doc[page_index] + page_no = page_index + 1 + pw, ph = page.rect.width, page.rect.height + img_w = int(pw * scale) + img_h = int(ph * scale) + fields_out = [] + for f in by_page.get(page_no, []): + x0, y0, x1, y1 = f["rect"] + fields_out.append({ + "name": f["name"], + "type": f["type"], + "label": f.get("label") or "", + "options": f.get("options") or [], + "value": values.get(f["name"], f.get("value", "")), + "rect_px": [ + int(x0 * scale), int(y0 * scale), + int(x1 * scale), int(y1 * scale), + ], + }) + pages_out.append({ + "page": page_no, + "width": img_w, + "height": img_h, + "fields": fields_out, + }) + return {"doc_id": doc_id, "scale": scale, "pages": pages_out} + finally: + pdf_doc.close() + finally: + db.close() + + # ---- GET /api/document/{doc_id}/page/{n}.png ---- + @router.get("/api/document/{doc_id}/page/{page_no}.png") + async def render_page_png(doc_id: str, page_no: int, request: Request): + """Render one page of the source PDF as a PNG (no values stamped — the + frontend overlays HTML form inputs on top).""" + from fastapi.responses import Response + from src.pdf_form_doc import find_source_upload_id + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, "Source PDF not found") + finally: + db.close() + + fitz = _load_pdf_viewer_fitz() + pdf_doc = fitz.open(pdf_path) + try: + if page_no < 1 or page_no > pdf_doc.page_count: + raise HTTPException(404, "Page out of range") + page = pdf_doc[page_no - 1] + mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) + pix = page.get_pixmap(matrix=mat, alpha=False) + png_bytes = pix.tobytes("png") + return Response( + content=png_bytes, + media_type="image/png", + headers={"Cache-Control": "public, max-age=3600"}, + ) + finally: + pdf_doc.close() + + # ---- POST /api/document/{doc_id}/ai-fill-annotations ---- + @router.post("/api/document/{doc_id}/ai-fill-annotations") + async def ai_fill_annotations(doc_id: str, request: Request) -> Dict[str, Any]: + """Ask a vision-capable LLM to locate fillable areas on a flat PDF and + propose annotation values for each, given a free-form user instruction. + + Returns a list of annotations: [{page, x, y, w, h, value}] where x/y/w/h + are page-percentages (0–100) — same coordinate system as the freeform + annotations the frontend already renders. + """ + import base64 + import json + import fitz + from src.pdf_form_doc import find_source_upload_id + from src.document_processor import _resolve_vl_model, _load_vl_settings + from src.llm_core import llm_call_async + + body = await request.json() if request.headers.get("content-type", "").startswith("application/json") else {} + instruction = (body or {}).get("instruction", "").strip() + if not instruction: + raise HTTPException(400, "instruction is required") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, "Source PDF not found") + finally: + db.close() + + # Resolve VL model (admin-configured or auto-detected vision-capable) + settings = _load_vl_settings() + vl_model = settings.get("vision_model", "") + try: + url, model_id, headers = _resolve_vl_model(vl_model, owner=user) + except Exception as e: + raise HTTPException(503, f"No vision model available: {e}") + + system_prompt = ( + "You analyze rendered PDF page images and propose values to fill in. " + "For each blank line, box, underscore, or labeled space on the page that " + "should be filled given the user's instruction, output one annotation. " + "Coordinates are percentages (0-100) of the page width/height with the " + "origin at top-left. Width/height should match the visible blank box. " + "Return ONLY a JSON array, no prose, no markdown fences. Each entry: " + '{"x": number, "y": number, "w": number, "h": number, "value": string}. ' + "If a region should not be filled, omit it. If nothing should be filled, " + "return []." + ) + + all_annotations = [] + pdf_doc = fitz.open(pdf_path) + try: + for page_index in range(pdf_doc.page_count): + page = pdf_doc[page_index] + mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) + pix = page.get_pixmap(matrix=mat, alpha=False) + png_bytes = pix.tobytes("png") + b64 = base64.b64encode(png_bytes).decode("ascii") + + messages = [ + {"role": "system", "content": system_prompt}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + f"User instruction:\n{instruction}\n\n" + f"This is page {page_index + 1} of {pdf_doc.page_count}. " + "Return JSON array of annotations to add to this page." + ), + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + }, + ], + }, + ] + try: + raw = await llm_call_async( + url, model_id, messages, + temperature=0.1, max_tokens=2000, headers=headers, + ) + except Exception as e: + logger.error(f"VL call failed on page {page_index + 1}: {e}") + continue + + raw = (raw or "").strip() + if raw.startswith("```"): + raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip() + try: + parsed = json.loads(raw) + except Exception: + logger.warning(f"AI fill: page {page_index + 1} returned non-JSON: {raw[:200]}") + continue + if not isinstance(parsed, list): + continue + for item in parsed: + if not isinstance(item, dict): + continue + try: + x = float(item.get("x", 0)) + y = float(item.get("y", 0)) + w = float(item.get("w", 0)) + h = float(item.get("h", 0)) + value = str(item.get("value", "") or "") + except Exception: + continue + # Clamp + reject zero-size entries + if w <= 0.5 or h <= 0.3: + continue + x = max(0.0, min(99.0, x)) + y = max(0.0, min(99.0, y)) + w = max(0.5, min(100.0 - x, w)) + h = max(0.3, min(100.0 - y, h)) + if not value.strip(): + continue + all_annotations.append({ + "page": page_index + 1, + "x": round(x, 2), + "y": round(y, 2), + "w": round(w, 2), + "h": round(h, 2), + "value": value, + }) + finally: + pdf_doc.close() + + return {"annotations": all_annotations} + + # ---- GET /api/document/{doc_id}/render-pdf ---- + @router.get("/api/document/{doc_id}/render-pdf") + async def render_pdf(doc_id: str, request: Request): + """Inline PDF preview filled with the current markdown values. + + Same plumbing as the export route, but no signature stamping and + served inline (Content-Disposition: inline) so the browser can + embed it in an iframe. Cache-busted by the caller via query string. + """ + import base64 + import os + import tempfile + from fastapi.responses import FileResponse + from starlette.background import BackgroundTask + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, parse_markdown_annotations + from src.pdf_forms import fill_fields, stamp_annotations + from core.database import Signature + + # Track temp files for this request so they get unlinked AFTER + # the response is fully sent (BackgroundTask runs post-send). + _to_unlink: list[str] = [] + def _cleanup_temps(): + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + # Fail fast with a clear 503 if the optional PyMuPDF dependency + # is missing — fill_fields/stamp_annotations will otherwise + # raise RuntimeError deep inside and bubble out as a 500. + # Mirrors the convention in _load_pdf_viewer_fitz above. + _load_pdf_viewer_fitz() + + values = parse_markdown_to_values(doc.current_content or "") + out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(out_path) + try: + fill_fields(pdf_path, out_path, values) + except Exception as e: + logger.error(f"render_pdf fill_fields failed for {doc_id}: {e}") + _cleanup_temps() + raise HTTPException(500, f"PDF render failed: {e}") + + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad annotation signature data for {s.id}: {e}") + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.error(f"stamp_annotations (render) failed for {doc_id}: {e}") + + return FileResponse( + out_path, + media_type="application/pdf", + headers={"Content-Disposition": "inline"}, + background=BackgroundTask(_cleanup_temps), + ) + finally: + db.close() + + # ---- GET /api/document/{doc_id}/export-pdf ---- + @router.get("/api/document/{doc_id}/export-pdf") + async def export_pdf(doc_id: str, request: Request): + """Stream the filled PDF for download. + + Reads field values and signature selections from the markdown — there + is no separate confirmation step. Signature fields contain their + chosen signature ID encoded as `signature:` in the value. + """ + import base64 + import os + import tempfile + from fastapi.responses import FileResponse + from starlette.background import BackgroundTask + from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar, parse_markdown_annotations + from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations + from core.database import Signature + + _to_unlink: list[str] = [] + def _cleanup_temps(): + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") + + schema = load_field_sidecar(pdf_path) or [] + sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} + + all_values = parse_markdown_to_values(doc.current_content or "") + # Split: signature fields go to stamps, everything else to fill_fields + text_values: dict = {} + sig_ids: dict[str, str] = {} + for name, raw in all_values.items(): + if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): + sig_ids[name] = raw[len("signature:"):].strip() + elif name not in sig_field_names: + text_values[name] = raw + + stamps: dict = {} + if sig_ids: + # SECURITY: filter by owner — same reason as render_pdf. + _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) + if user: + _sig_q2 = _sig_q2.filter(Signature.owner == user) + rows = _sig_q2.all() + by_id = {s.id: s for s in rows} + for field_name, sid in sig_ids.items(): + s = by_id.get(sid) + if not s: + continue + try: + stamps[field_name] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad signature data for {sid}: {e}") + + filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(filled_path) + try: + fill_fields(pdf_path, filled_path, text_values) + except Exception as e: + logger.error(f"fill_fields failed for doc {doc_id}: {e}") + _cleanup_temps() + raise HTTPException(500, f"PDF fill failed: {e}") + + out_path = filled_path + if stamps: + stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(stamped_path) + try: + stamp_signatures(filled_path, stamped_path, stamps) + out_path = stamped_path + except Exception as e: + logger.error(f"stamp_signatures failed for doc {doc_id}: {e}") + + # Burn freeform annotations (Text/Check/Sign drops) on top. + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + # Resolve any signature annotations to their PNG bytes. + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception as e: + logger.warning(f"Bad annotation signature data for {s.id}: {e}") + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.error(f"stamp_annotations failed for doc {doc_id}: {e}") + + download_name = _slug(doc.title or "form") + "_annotated.pdf" + return FileResponse( + out_path, + media_type="application/pdf", + filename=download_name, + background=BackgroundTask(_cleanup_temps), + ) + finally: + db.close() + + # ---- POST /api/document/{doc_id}/prepare-signed-reply ---- + @router.post("/api/document/{doc_id}/prepare-signed-reply") + async def prepare_signed_reply(doc_id: str, request: Request): + """Bake the current PDF state (form fields + signature stamps + + annotations) into a flattened PDF, drop it in COMPOSE_UPLOADS_DIR + and return the reply context (To/Subject/threading headers) so the + frontend can open a reply draft with this attachment pre-loaded. + + Requires the document to have source_email_* metadata (set when the + doc was created via /api/email/attachment-as-doc). Otherwise 400. + """ + import base64 + import tempfile + import shutil + import uuid as _uuid + import email as _email_mod + from src.pdf_form_doc import ( + find_source_upload_id, parse_markdown_to_values, + load_field_sidecar, parse_markdown_annotations, + ) + from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations + from core.database import Signature + # COMPOSE_UPLOADS_DIR lives in email_routes — re-derive here so we + # don't import from a routes file (cycle-prone). Same env override + # as email_routes (ODYSSEUS_MAIL_ATTACHMENTS_DIR). + from pathlib import Path as _Path + _COMPOSE_DIR = _Path(MAIL_ATTACHMENTS_DIR) / "_compose" + _COMPOSE_DIR.mkdir(parents=True, exist_ok=True) + + user = get_current_user(request) + db = SessionLocal() + try: + doc = db.query(Document).filter(Document.id == doc_id).first() + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) + + if not (doc.source_email_uid and doc.source_email_folder): + raise HTTPException(400, "Document has no source email — cannot reply") + + # 1) Build the flattened PDF (same pipeline as export_pdf) + upload_id = find_source_upload_id(doc.current_content or "") + if not upload_id: + raise HTTPException(400, "Document is not linked to a source PDF") + pdf_path = _locate_current_user_upload(request, upload_id, user) + if not pdf_path: + raise HTTPException(404, f"Source PDF {upload_id} not found") + + schema = load_field_sidecar(pdf_path) or [] + sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} + all_values = parse_markdown_to_values(doc.current_content or "") + text_values: dict = {} + sig_ids: dict[str, str] = {} + for name, raw in all_values.items(): + if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): + sig_ids[name] = raw[len("signature:"):].strip() + elif name not in sig_field_names: + text_values[name] = raw + + stamps: dict = {} + if sig_ids: + # SECURITY: filter by owner — same reason as render_pdf. + _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) + if user: + _sig_q2 = _sig_q2.filter(Signature.owner == user) + rows = _sig_q2.all() + by_id = {s.id: s for s in rows} + for fname, sid in sig_ids.items(): + s = by_id.get(sid) + if not s: + continue + try: + stamps[fname] = base64.b64decode(s.data_png) + except Exception: + pass + + import os + _to_unlink: list[str] = [] + filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(filled_path) + fill_fields(pdf_path, filled_path, text_values) + out_path = filled_path + if stamps: + stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(stamped_path) + try: + stamp_signatures(filled_path, stamped_path, stamps) + out_path = stamped_path + except Exception as e: + logger.warning(f"stamp_signatures failed for {doc_id}: {e}") + + annotations = parse_markdown_annotations(doc.current_content or "") + if annotations: + ann_sig_ids = [ + a["value"][len("signature:"):].strip() + for a in annotations + if a.get("kind") == "signature" + and isinstance(a.get("value"), str) + and a["value"].startswith("signature:") + ] + ann_signature_pngs: dict[str, bytes] = {} + if ann_sig_ids: + # SECURITY: filter by owner so a caller can't reference + # someone else's signature ID from doc markdown and have + # it stamped/exported. + _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) + if user: + _sig_q = _sig_q.filter(Signature.owner == user) + sig_rows = _sig_q.all() + for s in sig_rows: + try: + ann_signature_pngs[s.id] = base64.b64decode(s.data_png) + except Exception: + pass + annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name + _to_unlink.append(annotated_path) + try: + stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) + out_path = annotated_path + except Exception as e: + logger.warning(f"stamp_annotations failed for {doc_id}: {e}") + + # 2) Move/copy into COMPOSE_UPLOADS_DIR with the token format + # `_` that /api/email/send expects. + filename = _slug(doc.title or "signed") + "_signed.pdf" + token = f"{_uuid.uuid4().hex}_{filename}" + dest = _COMPOSE_DIR / token + shutil.copyfile(out_path, str(dest)) + # Unlink the intermediate temp PDFs now that they've been + # copied into COMPOSE_UPLOADS_DIR. + for _p in _to_unlink: + try: + os.unlink(_p) + except FileNotFoundError: + pass + except Exception as _e: + logger.warning(f"Could not unlink temp PDF {_p}: {_e}") + + # 3) Fetch the source email's headers so we can build a clean reply + # context (To/Subject/In-Reply-To/References). + try: + from routes.email_routes import _imap, _decode_header + from routes.email_helpers import _q + except Exception: + _imap = None + _decode_header = lambda x: x or "" + _q = lambda x: x or "" + + to_addr = "" + from_name = "" + subject = "" + in_reply_to = doc.source_email_message_id or "" + references = in_reply_to + if _imap: + try: + with _imap(doc.source_email_account_id or None) as conn: + conn.select(_q(doc.source_email_folder), readonly=True) + status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)") + if status == "OK" and data and data[0]: + raw_hdr = data[0][1] + m = _email_mod.message_from_bytes(raw_hdr) + sender = _decode_header(m.get("From", "")) + from_name, to_addr = _email_mod.utils.parseaddr(sender) + if not to_addr: + to_addr = sender + subject = _decode_header(m.get("Subject", "") or "") + if subject and not subject.lower().startswith("re:"): + subject = "Re: " + subject + msg_refs = (m.get("References") or "").strip() + msg_in_reply = (m.get("Message-ID") or "").strip() or in_reply_to + in_reply_to = msg_in_reply + references = (msg_refs + " " + msg_in_reply).strip() if msg_refs else msg_in_reply + except Exception as e: + logger.warning(f"prepare-signed-reply header fetch failed: {e}") + + return { + "ok": True, + "attachment": { + "token": token, + "filename": filename, + "size": dest.stat().st_size, + }, + "reply": { + "to": to_addr, + "to_name": from_name, + "subject": subject, + "in_reply_to": in_reply_to, + "references": references, + "account_id": doc.source_email_account_id or None, + "source_uid": doc.source_email_uid, + "source_folder": doc.source_email_folder, + "source_message_id": doc.source_email_message_id, + }, + } + finally: + db.close() + + return router diff --git a/routes/document_helpers.py b/routes/document_helpers.py index a0c2d08eb..c1f68ca51 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -1,243 +1,14 @@ -"""document_helpers.py — Pydantic models, doc serializers, owner gating, file-locator helpers shared with document_routes.py.""" +"""Backward-compat shim — canonical location is routes/document/document_helpers.py. -"""Document routes — CRUD for living documents with version history.""" +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.document_helpers``, ``from routes.document_helpers import +X``, and the ``sys.modules.pop("routes.document_helpers")`` + re-import +pattern used by test_security_regressions.py all operate on the *same* object. +Keeps existing import paths working after slice 2m (#4082/#4071). +""" -import logging -import os -import re -from typing import Any, Dict, Optional +import sys as _sys -from fastapi import HTTPException, Request -from pydantic import BaseModel +from routes.document import document_helpers as _canonical # noqa: F401 -from core.database import Document, DocumentVersion -from core.database import Session as DbSession -from src.auth_helpers import _auth_disabled -from src.upload_handler import UploadHandler - -logger = logging.getLogger(__name__) - - -# ---- Request schemas ---- - -class DocumentCreate(BaseModel): - session_id: Optional[str] = None - title: str = "Untitled" - language: Optional[str] = None - content: str = "" - -class DocumentUpdate(BaseModel): - content: str - summary: Optional[str] = None - force_version: bool = False - -class DocumentPatch(BaseModel): - title: Optional[str] = None - language: Optional[str] = None - session_id: Optional[str] = None # link/unlink document to a session - - -# ---- Helpers ---- - -def _doc_to_dict(doc: Document) -> Dict[str, Any]: - return { - "id": doc.id, - "session_id": doc.session_id, - "title": doc.title, - "language": doc.language, - "current_content": doc.current_content, - "version_count": doc.version_count, - "is_active": doc.is_active, - "archived": bool(getattr(doc, "archived", False)), - "created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None, - "updated_at": (doc.updated_at.isoformat() + "Z") if doc.updated_at else None, - # Source-email provenance (set when doc was created from an email - # attachment) — drives the "Send signed reply" menu item. - "source_email_uid": getattr(doc, "source_email_uid", None), - "source_email_folder": getattr(doc, "source_email_folder", None), - "source_email_account_id": getattr(doc, "source_email_account_id", None), - "source_email_message_id": getattr(doc, "source_email_message_id", None), - } - -def _version_to_dict(v: DocumentVersion) -> Dict[str, Any]: - return { - "id": v.id, - "document_id": v.document_id, - "version_number": v.version_number, - "content": v.content, - "summary": v.summary, - "source": v.source, - "created_at": v.created_at.isoformat() if v.created_at else None, - } - - -def _verify_doc_owner(db, doc: Document, user: str): - """Verify `user` owns this document. Raise 404 if not. - - Documents now carry their own `owner` column, so a doc whose session - was deleted (session_id → NULL) can still prove ownership and stay - openable / cloneable. We trust that column first and only fall back to - the session join for any not-yet-backfilled legacy row. - """ - if user is None: - if _auth_disabled(): - return # Single-user / no-auth mode: allow access - raise HTTPException(403, "Authentication required") - if doc.owner is not None: - if doc.owner != user: - raise HTTPException(404, "Document not found") - return - # Legacy fallback: derive ownership from the linked session. - if not doc.session_id: - raise HTTPException(404, "Document not found") - session = db.query(DbSession).filter(DbSession.id == doc.session_id).first() - if not session or session.owner != user: - raise HTTPException(404, "Document not found") - - -def _owner_session_filter(q, user): - """Restrict a documents query to those owned by `user`. - - Documents now carry their own `owner` column (backfilled at boot from - the linked session, or assigned to the admin user for legacy/orphaned - docs). We filter on that directly rather than on a session join, so a - document whose session was deleted (session_id → NULL) still shows up - for its owner instead of silently vanishing from the Library + search. - - The owner backfill runs in init_db before the app serves requests, so - by the time this filter is live there are no NULL-owner rows to leak; - we therefore match the owner strictly for authenticated callers.""" - if not user: - if user == "" or _auth_disabled(): - return q - return q.filter(False) - return q.filter(Document.owner == user) - - - -def _slug(name: str) -> str: - """Filesystem-friendly version of a document title. - - Whitespace becomes underscores; other unsafe punctuation is dropped. - Preserves letters, digits, dot, hyphen, underscore. Idempotent. - """ - import re as _re - s = (name or "").strip() - # Drop the trailing extension if the title happens to include one - s = _re.sub(r'\.pdf$', '', s, flags=_re.IGNORECASE) - s = _re.sub(r'\s+', '_', s) - s = _re.sub(r'[^A-Za-z0-9._-]', '', s) - s = _re.sub(r'_+', '_', s).strip('_') - return s or "form" - - -# DPI scale for the interactive PDF view. ~150 DPI (2x of 72 PDF user-units). -_PDF_RENDER_SCALE = 2.0 - - -def _upload_path_inside(upload_dir: str, path: str) -> bool: - base = os.path.realpath(upload_dir) - p = os.path.realpath(path) - try: - return os.path.commonpath([base, p]) == base - except Exception: - return False - - -def _resolve_user_upload_path( - upload_handler: Any, - upload_id: str, - owner: Optional[str], - auth_manager=None, -) -> Optional[str]: - """Resolve an upload id to a filesystem path the caller may read.""" - if upload_handler is None: - return None - resolved = upload_handler.resolve_upload( - upload_id, - owner=owner, - auth_manager=auth_manager, - ) - if not isinstance(resolved, dict) or not resolved: - return None - path = resolved.get("path") - upload_dir = getattr(upload_handler, "upload_dir", None) - if path and upload_dir and not _upload_path_inside(upload_dir, path): - logger.warning("Upload path outside upload directory: %s", path) - return None - return path - - -def _locate_upload( - upload_dir: str, - file_id: str, - owner: Optional[str] = None, - auth_manager=None, - upload_handler: Any = None, -): - """Find an upload by its filename ID via UploadHandler.resolve_upload.""" - if upload_handler is None: - from src.upload_handler import UploadHandler - - base_dir = os.path.dirname(os.path.abspath(upload_dir)) - upload_handler = UploadHandler(base_dir, upload_dir) - return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager) - - -def _assert_pdf_marker_upload_owned( - request: Request, - content: str, - user: Optional[str], - upload_handler: Any, -) -> None: - """Reject document content whose pdf_source marker points at another user's upload.""" - if upload_handler is None: - return - from src.pdf_form_doc import find_source_upload_id - - upload_id = find_source_upload_id(content or "") - if not upload_id: - return - auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) - if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager): - raise HTTPException( - 400, - "Document PDF marker references an upload you do not own", - ) - - -def _derive_title(content: str) -> str: - """Derive a title from document content.""" - import re - if not isinstance(content, str): - return "Untitled" - text = content.strip() - if not text: - return "Untitled" - - # Markdown header - md = re.match(r'^#{1,3}\s+(.+)', text, re.MULTILINE) - if md: - title = md.group(1).strip() - if len(title) > 50: - title = title[:48] + "…" - return title - - # HTML heading - html = re.search(r']*>([^<]+)', text, re.IGNORECASE) - if html: - title = html.group(1).strip() - if len(title) > 50: - title = title[:48] + "…" - return title - - # First non-empty line (if short enough) - for line in text.split('\n'): - line = line.strip() - if line and 2 <= len(line) <= 60: - title = re.sub(r'[:#*`]+$', '', line).strip() - if title and len(title) > 50: - title = title[:48] + "…" - return title or "Untitled" - - return "Untitled" +_sys.modules[__name__] = _canonical diff --git a/routes/document_routes.py b/routes/document_routes.py index dae8b09fa..dd13e3c60 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -1,1810 +1,17 @@ -"""Document routes — CRUD for living documents with version history.""" +"""Backward-compat shim — canonical location is routes/document/document_routes.py. -import uuid -import logging -from datetime import datetime, timezone -from typing import Dict, Any, List, Optional +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.document_routes``, ``from routes.document_routes import +X``, ``importlib.import_module("routes.document_routes")``, and the +``import ... as droutes`` + ``droutes.SessionLocal = ...`` / +``monkeypatch.setattr(droutes, ...)`` pattern used by multiple tests all +operate on the *same* object the application actually uses. Keeps existing +import paths working after slice 2m (#4082/#4071). Source-introspection tests +read the canonical file by path. +""" -from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form +import sys as _sys -from sqlalchemy import case, func, or_ -from core.database import SessionLocal, Document, DocumentVersion -from core.database import Session as DbSession -from src.auth_helpers import get_current_user, _auth_disabled -from src.constants import MAIL_ATTACHMENTS_DIR -from src.upload_handler import reserve_upload_references +from routes.document import document_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - - -def _get_session_or_404(db, session_id: str, user: Optional[str]): - session = db.query(DbSession).filter(DbSession.id == session_id).first() - if not session: - raise HTTPException(404, "Session not found") - if user and session.owner != user: - raise HTTPException(404, "Session not found") - return session - - -def _aggregate_language_facets(lang_rows): - """Sum document counts per display language for the library facet. - - NULL-language and explicit "text" rows share the "text" bucket (the - language filter treats them as one), so they must be ADDED. The old dict - comprehension keyed both to "text", silently overwriting one group and - undercounting the facet versus what the filter actually returns. - """ - out = {} - for lang, cnt in lang_rows: - key = lang or "text" - out[key] = out.get(key, 0) + cnt - return out - - -def _library_language_for_document(doc: Document) -> str: - """Return the display language used by the document library. - - PDF documents are stored as markdown wrappers so the editor can preserve - extracted text, form fields, and annotations. The library should still - identify them as PDFs instead of exposing that internal wrapper format. - """ - from src.pdf_form_doc import find_source_upload_id - - if find_source_upload_id(doc.current_content or ""): - return "pdf" - return doc.language or "text" - - -def _email_source_key(content: str) -> tuple[str, str]: - """Return the source email identity embedded in an email draft document.""" - import re - - text = content or "" - uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text) - folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text) - uid = (uid_m.group(1).strip() if uid_m else "") - folder = (folder_m.group(1).strip() if folder_m else "INBOX") - return uid, folder - - -from routes.document_helpers import ( - DocumentCreate, DocumentUpdate, DocumentPatch, - _doc_to_dict, _version_to_dict, - _verify_doc_owner, _owner_session_filter, - _slug, _resolve_user_upload_path, _assert_pdf_marker_upload_owned, _derive_title, - _PDF_RENDER_SCALE, -) - - -def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: - router = APIRouter(tags=["documents"]) - - def _reserve_document_uploads(user: Optional[str], content: str) -> None: - missing_id = reserve_upload_references(upload_handler, user, content) - if missing_id: - raise HTTPException( - 409, - f"Referenced upload is no longer available: {missing_id}", - ) - - def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]): - if upload_handler is None: - return None - auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) - return _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager) - - def _load_pdf_viewer_fitz(): - from src.pdf_runtime import load_pymupdf_for_pdf_viewer - - try: - return load_pymupdf_for_pdf_viewer() - except RuntimeError as exc: - raise HTTPException(503, str(exc)) from exc - - # ---- POST /api/document ---- - @router.post("/api/document") - async def create_document(request: Request, req: DocumentCreate) -> Dict[str, Any]: - from src.auth_helpers import require_privilege - user = require_privilege(request, "can_use_documents") - db = SessionLocal() - try: - # session_id is optional: a doc can be a session-less "library" doc - # (e.g. files imported from the library) — session_id is nullable and - # the doc is owner-stamped, so it lives in the library on its own. - session = None - if req.session_id: - # Match the lenient ownership model the rest of the app uses - # (see _owner_filter): only block when an AUTHENTICATED user is - # writing into a DIFFERENT user's session. In single-user / - # unconfigured / localhost-bypass mode, falsey users preserve - # the existing lenient path. - session = _get_session_or_404(db, req.session_id, user) - - # If no language was supplied (e.g. cloning a doc whose language - # was never set), detect it from the content rather than storing - # NULL — which made the editor fall back to plain text. Defaults - # to markdown for prose. - language = req.language - if not language: - from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content - language = _sniff_doc_language(req.content) - else: - from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content - if _looks_like_email_document(req.content, req.title): - language = "email" - - _reserve_document_uploads(user, req.content) - _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) - - # Reply drafts are keyed to the source email. If a UI/tool path tries - # to create a second draft for the same email in the same chat, - # update the existing draft instead so quoted thread history stays - # attached to the visible document. - if language == "email" and req.session_id: - source_uid, source_folder = _email_source_key(req.content) - if source_uid: - candidates = ( - db.query(Document) - .filter(Document.session_id == req.session_id) - .filter(Document.is_active == True) - .filter(Document.language == "email") - .order_by(Document.updated_at.desc()) - .limit(25) - .all() - ) - for existing in candidates: - old_uid, old_folder = _email_source_key(existing.current_content or "") - if old_uid != source_uid or old_folder != source_folder: - continue - merged = _coerce_email_document_content(existing.current_content or "", req.content) - if existing.current_content != merged: - new_ver = (existing.version_count or 1) + 1 - existing.current_content = merged - existing.title = req.title or existing.title - existing.version_count = new_ver - db.add(DocumentVersion( - id=str(uuid.uuid4()), - document_id=existing.id, - version_number=new_ver, - content=merged, - summary="Updated existing email draft", - source="user", - )) - db.commit() - db.refresh(existing) - return _doc_to_dict(existing) - - doc_id = str(uuid.uuid4()) - ver_id = str(uuid.uuid4()) - - doc = Document( - id=doc_id, - session_id=req.session_id, - title=req.title, - language=language, - current_content=req.content, - version_count=1, - is_active=True, - # Stamp ownership directly so the doc survives its session - # being deleted. Fall back to the session's owner when the - # request is unauthenticated (single-user / localhost bypass). - owner=user or (session.owner if session else None), - ) - ver = DocumentVersion( - id=ver_id, - document_id=doc_id, - version_number=1, - content=req.content, - summary="Initial version", - source="user", - ) - db.add(doc) - db.add(ver) - db.commit() - db.refresh(doc) - try: - from src.event_bus import fire_event - fire_event("document_created", doc.owner) - except Exception: - logger.debug("document_created event dispatch failed", exc_info=True) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - logger.error(f"Failed to create document: {e}") - raise HTTPException(500, f"Failed to create document: {e}") - finally: - db.close() - - # ---- POST /api/documents/import-pdf ---- - @router.post("/api/documents/import-pdf") - async def import_pdf( - request: Request, - file: UploadFile = File(...), - session_id: Optional[str] = Form(None), - ) -> Dict[str, Any]: - """Upload a PDF and create the matching Document. - - Detects AcroForm fields — if any, creates a form-backed markdown doc - (clickable inputs in the PDF view). Otherwise creates a plain PDF doc - with a `pdf_source` marker so the viewer renders the pages without - overlays. - """ - from src.pdf_forms import has_form_fields, extract_fields - from src.pdf_form_doc import ( - save_field_sidecar, - create_form_markdown_document, - create_plain_pdf_document, - ) - from src.document_processor import _process_pdf, strip_pdf_content_marker - import os - - from src.auth_helpers import require_privilege - user = require_privilege(request, "can_use_documents") - - # session_id is optional — a library import isn't tied to a chat. When - # given, validate it; otherwise the PDF becomes a session-less library - # doc (the doc creators below already handle a missing session). - if session_id: - db = SessionLocal() - try: - _get_session_or_404(db, session_id, user) - finally: - db.close() - - if upload_handler is None: - raise HTTPException(500, "Upload handler not configured") - - client_ip = request.client.host if request.client else "unknown" - try: - meta = upload_handler.save_upload(file, client_ip, owner=user) - except HTTPException: - raise - except Exception as e: - logger.error(f"PDF import save_upload failed: {e}") - raise HTTPException(500, f"Upload failed: {e}") - - upload_id = meta["id"] - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(500, "Saved PDF could not be located") - - title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0] - try: - body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) - except Exception: - body_text = None - - is_form = False - try: - is_form = has_form_fields(pdf_path) - except Exception as e: - logger.warning(f"has_form_fields failed for {pdf_path}: {e}") - - if is_form: - fields = extract_fields(pdf_path) - save_field_sidecar(pdf_path, fields) - doc_id = create_form_markdown_document( - session_id=session_id, - fields=fields, - upload_id=upload_id, - title=title, - intro_text=body_text, - ) - else: - doc_id = create_plain_pdf_document( - session_id=session_id, - upload_id=upload_id, - title=title, - body_text=body_text, - ) - - if not doc_id: - raise HTTPException(500, "Failed to create document for PDF") - - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(500, "Created document not found") - # The PDF doc creators stamp owner from the session only; a - # session-less library import leaves owner NULL, which the Library's - # owner filter then hides. Stamp the requesting user so it shows. - if not doc.owner and user: - doc.owner = user - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - finally: - db.close() - - # ---- GET /api/documents/library ---- - @router.get("/api/documents/library") - async def documents_library( - request: Request, - search: Optional[str] = Query(None), - language: Optional[str] = Query(None), - sort: str = Query("recent"), - offset: int = Query(0, ge=0), - limit: int = Query(20, ge=1, le=50), - archived: bool = Query(False), - ) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - from sqlalchemy import or_ - pdf_marker_cond = or_( - Document.current_content.like('%\s*\n+#[^\n]*\n+)', re.MULTILINE) - head_match = head_re.match(content) - head = head_match.group(1) if head_match else (content.splitlines()[0] + "\n\n# " + (doc.title or "PDF") + "\n\n") - doc.current_content = head + body_text.strip() + "\n" - doc.version_count = (doc.version_count or 1) + 1 - db.add(DocumentVersion( - id=str(__import__("uuid").uuid4()), - document_id=doc_id, - version_number=doc.version_count, - content=doc.current_content, - summary="PDF text re-extracted (OCR)", - source="ocr", - )) - db.commit() - return {"ok": True, "id": doc_id, "extracted": True, "chars": len(body_text)} - finally: - db.close() - - # ---- POST /api/documents/export-zip — bundle selected docs into a .zip ---- - @router.post("/api/documents/export-zip") - async def documents_export_zip(request: Request): - """Zip the selected documents (each as a text file with the right - extension) — mirrors the gallery's bulk download-zip so multi-export - is one file instead of a blocked flood of individual downloads.""" - user = get_current_user(request) - try: - data = await request.json() - except Exception as e: - logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) - data = {} - ids = data.get("ids") or [] - if not ids: - raise HTTPException(400, "No documents specified") - _ext = { - "javascript": ".js", "python": ".py", "html": ".html", "css": ".css", - "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh", - "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c", - "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php", - "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini", - } - db = SessionLocal() - try: - import io - import re - import zipfile - from fastapi import Response - docs = db.query(Document).filter(Document.id.in_(ids)).all() - buf = io.BytesIO() - used = set() - wrote = 0 - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for doc in docs: - try: - _verify_doc_owner(db, doc, user) - except HTTPException: - continue # skip docs the user doesn't own - ext = _ext.get(doc.language or "text", ".txt") - base = (doc.title or "document").strip() or "document" - base = re.sub(r"[^\w\-. ]+", "", base)[:60].strip() or doc.id - name = base if "." in base else base + ext - i = 1 - while name in used: - name = f"{base}-{i}" + ("" if "." in base else ext) - i += 1 - used.add(name) - zf.writestr(name, doc.current_content or "") - wrote += 1 - if not wrote: - raise HTTPException(404, "No documents found") - return Response( - content=buf.getvalue(), - media_type="application/zip", - headers={"Content-Disposition": 'attachment; filename="documents.zip"'}, - ) - finally: - db.close() - - # ---- PUT /api/document/{doc_id} — user manual edit ---- - # Coalesce window: if the last user version was saved within this many - # seconds, update it in-place (user is still actively editing). - # Once the gap exceeds this, the next save creates a new version. - VERSION_COALESCE_SECONDS = 60 - - @router.put("/api/document/{doc_id}") - async def update_document(request: Request, doc_id: str, req: DocumentUpdate) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - incoming_content = req.content - from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document - is_email_doc = ( - (doc.language or "").lower() == "email" - or _looks_like_email_document(doc.current_content or "", doc.title or "") - or _looks_like_email_document(req.content or "", doc.title or "") - ) - if is_email_doc: - incoming_content = _coerce_email_document_content(doc.current_content or "", req.content) - doc.language = "email" - - # Skip if content is identical unless the caller explicitly wants - # a checkpoint version from the current editor state. - if doc.current_content == incoming_content and not req.force_version: - return _doc_to_dict(doc) - - _reserve_document_uploads(user, incoming_content) - _assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler) - - # Check if we can coalesce with the latest version - latest_ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - ).order_by(DocumentVersion.version_number.desc()).first() - - now = datetime.now(timezone.utc) - coalesced = False - if latest_ver and latest_ver.source == "user" and not req.force_version: - ver_time = latest_ver.created_at - if ver_time.tzinfo is None: - ver_time = ver_time.replace(tzinfo=timezone.utc) - age = (now - ver_time).total_seconds() - if age < VERSION_COALESCE_SECONDS: - # Update the existing version in-place - latest_ver.content = incoming_content - latest_ver.created_at = now - if req.summary: - latest_ver.summary = req.summary - coalesced = True - - if not coalesced: - new_ver = doc.version_count + 1 - ver = DocumentVersion( - id=str(uuid.uuid4()), - document_id=doc_id, - version_number=new_ver, - content=incoming_content, - summary=req.summary or "Manual edit", - source="user", - ) - doc.version_count = new_ver - db.add(ver) - - doc.current_content = incoming_content - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, f"Failed to update document: {e}") - finally: - db.close() - - # ---- PATCH /api/document/{doc_id} — metadata only ---- - @router.patch("/api/document/{doc_id}") - async def patch_document(request: Request, doc_id: str, req: DocumentPatch) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - if req.title is not None: - doc.title = req.title - if req.language is not None: - doc.language = req.language - if req.session_id is not None: - # Empty string = unlink from session - if req.session_id: - _get_session_or_404(db, req.session_id, user) - doc.session_id = req.session_id if req.session_id else None - if not req.session_id: - # Tab closed / doc detached from its session — drop the - # in-memory active-doc pointer so the last-resort injection - # path doesn't re-surface this doc in a later chat (#1160). - try: - from src.agent_tools.document_tools import clear_active_document - clear_active_document(doc_id) - except Exception as e: - logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e) - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- DELETE /api/document/{doc_id} — soft delete ---- - @router.delete("/api/document/{doc_id}") - async def delete_document(request: Request, doc_id: str) -> Dict[str, str]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - doc.is_active = False - # Closed/deleted — drop the in-memory active-doc pointer so it isn't - # re-injected into a later, unrelated chat (#1160). - try: - from src.agent_tools.document_tools import clear_active_document - clear_active_document(doc_id) - except Exception: - pass - db.commit() - return {"status": "deleted", "id": doc_id} - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- GET /api/document/{doc_id}/versions ---- - @router.get("/api/document/{doc_id}/versions") - async def list_versions(request: Request, doc_id: str) -> List[Dict[str, Any]]: - user = get_current_user(request) - db = SessionLocal() - try: - # Verify ownership before listing versions - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - versions = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id - ).order_by(DocumentVersion.version_number.desc()).all() - return [{ - "id": v.id, - "version_number": v.version_number, - "content": v.content, - "summary": v.summary, - "source": v.source, - "created_at": v.created_at.isoformat() if v.created_at else None, - } for v in versions] - finally: - db.close() - - # ---- GET /api/document/{doc_id}/version/{num} ---- - @router.get("/api/document/{doc_id}/version/{num}") - async def get_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - # Verify ownership - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - DocumentVersion.version_number == num, - ).first() - if not ver: - raise HTTPException(404, "Version not found") - return _version_to_dict(ver) - finally: - db.close() - - # ---- POST /api/document/{doc_id}/restore/{num} ---- - @router.post("/api/document/{doc_id}/restore/{num}") - async def restore_version(request: Request, doc_id: str, num: int) -> Dict[str, Any]: - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - old_ver = db.query(DocumentVersion).filter( - DocumentVersion.document_id == doc_id, - DocumentVersion.version_number == num, - ).first() - if not old_ver: - raise HTTPException(404, "Version not found") - - new_ver_num = doc.version_count + 1 - ver = DocumentVersion( - id=str(uuid.uuid4()), - document_id=doc_id, - version_number=new_ver_num, - content=old_ver.content, - summary=f"Restored from v{num}", - source="user", - ) - doc.current_content = old_ver.content - doc.version_count = new_ver_num - db.add(ver) - db.commit() - db.refresh(doc) - return _doc_to_dict(doc) - except HTTPException: - raise - except Exception as e: - db.rollback() - raise HTTPException(500, str(e)) - finally: - db.close() - - # ---- POST /api/documents/tidy — clean up broken/empty documents ---- - @router.post("/api/documents/tidy") - async def tidy_documents(request: Request) -> Dict[str, Any]: - """Fix empty titles and remove broken/empty documents (user's docs only).""" - user = get_current_user(request) - db = SessionLocal() - try: - q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True) - .filter((Document.archived == False) | (Document.archived.is_(None))) - ) - q = _owner_session_filter(q, user) - docs = q.all() - fixed_titles = 0 - deleted = 0 - - # Same junk-detection logic as the scheduled tidy_documents - # action (src/document_actions.py). Keep these two in sync. - import re as _re - from src.document_actions import _JUNK_TITLES - - to_delete = [] - now = datetime.now(timezone.utc) - for doc in docs: - created = doc.created_at - if created and created.tzinfo is None: - created = created.replace(tzinfo=timezone.utc) - - # Skip freshly created documents to avoid deleting them while the user is actively editing - if created and (now - created).total_seconds() < 900: # 15 minutes - continue - - content = (doc.current_content or "").strip() - title_raw = (doc.title or "").strip() - title = title_raw.lower() - is_fresh_empty = ( - not content - and created is not None - and (now - created).total_seconds() < 1800 - ) - if is_fresh_empty: - continue - - # Strip markdown noise to get a "real" character count - stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE) - stripped = _re.sub(r"[*_`>\-=]+", "", stripped) - stripped = _re.sub(r"\s+", " ", stripped).strip() - real_len = len(stripped) - - # Detect email-scaffold stubs: "To: \nSubject: \n---\n" style - # bodies with nothing typed in. Stub = every meaningful line - # is a header label (To:/From:/Subject:/...) with no real - # value (blank, "empty", "(empty)", "-", "none", "n/a"). - _is_email_stub = False - _HEADER_RE = _re.compile(r"^(to|from|cc|bcc|subject|reply-to):\s*(.*)$", _re.I) - _PLACEHOLDER_VALS = {"", "empty", "(empty)", "-", "—", "none", "n/a", "na", "tbd"} - if title in ("new email", "new mail", "new message") or doc.language == "email": - body_lines = [ln.strip() for ln in content.split("\n") - if ln.strip() and ln.strip() != "---"] - def _is_filler(ln): - m = _HEADER_RE.match(ln) - if not m: - return False - val = (m.group(2) or "").strip().lower() - return val in _PLACEHOLDER_VALS - has_real_body = any(not _is_filler(ln) for ln in body_lines) - if body_lines and not has_real_body: - _is_email_stub = True - - # Hard-delete obviously empty / junk documents - if not content or content in ("", "# Untitled"): - to_delete.append(doc); deleted += 1; continue - if _is_email_stub: - to_delete.append(doc); deleted += 1; continue - if title in _JUNK_TITLES: - to_delete.append(doc); deleted += 1; continue - - # Fix empty or placeholder titles on survivors - if not title_raw or title_raw == "Untitled": - new_title = _derive_title(content) - if new_title and new_title != "Untitled": - doc.title = new_title - fixed_titles += 1 - - for doc in to_delete: - db.delete(doc) - - # Also clean up inactive empty docs from previous soft-deletes - inactive_q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == False) - .filter((Document.current_content == None) | (Document.current_content == "")) - ) - inactive_q = _owner_session_filter(inactive_q, user) - inactive_docs = inactive_q.all() - for doc in inactive_docs: - db.delete(doc) - deleted += len(inactive_docs) - - db.commit() - return { - "fixed_titles": fixed_titles, - "deleted": deleted, - "message": f"Fixed {fixed_titles} title{'s' if fixed_titles != 1 else ''}, removed {deleted} empty document{'s' if deleted != 1 else ''}", - } - except Exception as e: - db.rollback() - logger.error(f"Document tidy failed: {e}") - raise HTTPException(500, f"Tidy failed: {e}") - finally: - db.close() - - # ---- POST /api/documents/ai-tidy — AI-powered cleanup of junk/test documents ---- - @router.post("/api/documents/ai-tidy") - async def ai_tidy_documents(request: Request) -> Dict[str, Any]: - """Use AI to judge if documents are junk/test/accidental, then delete them. - Caches verdicts so previously-reviewed docs are skipped.""" - from src.task_endpoint import resolve_task_endpoint - from src.endpoint_resolver import resolve_endpoint - from src.llm_core import llm_call_async - - user = get_current_user(request) - url, model, headers = resolve_task_endpoint(owner=user or None) - if not url or not model: - # Fall back to default endpoint - url, model, headers = resolve_endpoint("default", owner=user or None) - if not url or not model: - raise HTTPException(500, "No endpoint configured for AI tidy") - - db = SessionLocal() - try: - q = ( - db.query(Document) - .outerjoin(DbSession, Document.session_id == DbSession.id) - .filter(Document.is_active == True) - .filter((Document.archived == False) | (Document.archived.is_(None))) - ) - q = _owner_session_filter(q, user) - docs = q.all() - - # Only review docs that haven't been reviewed yet - to_review = [d for d in docs if not d.tidy_verdict] - if not to_review: - return {"deleted": 0, "reviewed": 0, "message": "All documents already reviewed"} - - # Build a batch prompt — review up to 30 at a time - batch = to_review[:30] - doc_list = [] - for i, doc in enumerate(batch): - preview = (doc.current_content or "")[:300].strip() - doc_list.append(f"[{i}] title=\"{doc.title}\" lang={doc.language or 'text'} content_preview=\"{preview}\"") - - prompt = ( - "You are a document library cleaner. For each document below, decide if it is JUNK " - "(test, accidental, placeholder, empty-ish, tool-test, throwaway) or KEEP (real content worth saving).\n\n" - "Respond with ONLY a JSON array of verdicts, one per document, like: [\"junk\",\"keep\",\"junk\",...]\n" - "No explanation, no markdown, just the JSON array.\n\n" - + "\n".join(doc_list) - ) - - response = await llm_call_async( - url, model, - [{"role": "system", "content": "You classify documents as junk or keep. Respond only with a JSON array."}, - {"role": "user", "content": prompt}], - temperature=0.1, - max_tokens=200, - headers=headers, - timeout=30, - ) - - # Parse verdicts - import re - match = re.search(r'\[.*?\]', response, re.DOTALL) - if not match: - raise HTTPException(500, "AI returned invalid response") - - import json as _json - verdicts = _json.loads(match.group()) - - deleted = 0 - reviewed = 0 - for i, doc in enumerate(batch): - if i >= len(verdicts): - break - verdict = str(verdicts[i] or "").lower().strip() - if verdict == "junk": - doc.tidy_verdict = "junk" - db.delete(doc) - deleted += 1 - else: - doc.tidy_verdict = "keep" - reviewed += 1 - - db.commit() - return { - "deleted": deleted, - "reviewed": reviewed, - "remaining": len(to_review) - len(batch), - "message": f"Reviewed {reviewed}, removed {deleted} junk document{'s' if deleted != 1 else ''}", - } - except HTTPException: - raise - except Exception as e: - db.rollback() - logger.error(f"AI tidy failed: {e}") - raise HTTPException(500, f"AI tidy failed: {e}") - finally: - db.close() - - # ---- POST /api/document/{doc_id}/export-pdf/preview ---- - @router.post("/api/document/{doc_id}/export-pdf/preview") - async def export_pdf_preview(doc_id: str, request: Request) -> Dict[str, Any]: - """Return the field-value mapping that would be written to the PDF. - - Frontend shows this in a confirmation modal so the user can spot/fix - any wrong values before triggering the actual download. - """ - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") - - fields = load_field_sidecar(pdf_path) - if not fields: - raise HTTPException(404, "Field schema sidecar missing for source PDF") - - values = parse_markdown_to_values(doc.current_content or "") - field_meta = {f["name"]: f for f in fields} - - preview = [] - for name, current in values.items(): - meta = field_meta.get(name) - if not meta: - continue - preview.append({ - "name": name, - "label": meta.get("label") or name, - "type": meta.get("type"), - "options": meta.get("options") or [], - "page": meta.get("page"), - "value": current, - }) - - unknown = [ - name for name in values - if name not in field_meta - ] - return { - "doc_id": doc_id, - "upload_id": upload_id, - "fields": preview, - "unknown_fields": unknown, - "total": len(fields), - "filled": sum(1 for p in preview if p["value"] not in ("", False, None)), - } - finally: - db.close() - - # ---- GET /api/document/{doc_id}/render-pages ---- - @router.get("/api/document/{doc_id}/render-pages") - async def render_pages(doc_id: str, request: Request) -> Dict[str, Any]: - """Return per-page metadata for the interactive PDF view. - - Each page entry has its rendered-image dimensions (matching what - /page/{n}.png returns at the same DPI) plus the list of form fields - on that page with their rects translated to image-pixel coordinates. - Frontend overlays HTML form controls at those positions. - """ - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - fitz = _load_pdf_viewer_fitz() - schema = load_field_sidecar(pdf_path) or [] - values = parse_markdown_to_values(doc.current_content or "") - - # Group fields by page - by_page: Dict[int, list] = {} - for f in schema: - by_page.setdefault(f["page"], []).append(f) - - scale = _PDF_RENDER_SCALE - pdf_doc = fitz.open(pdf_path) - try: - pages_out = [] - for page_index in range(pdf_doc.page_count): - page = pdf_doc[page_index] - page_no = page_index + 1 - pw, ph = page.rect.width, page.rect.height - img_w = int(pw * scale) - img_h = int(ph * scale) - fields_out = [] - for f in by_page.get(page_no, []): - x0, y0, x1, y1 = f["rect"] - fields_out.append({ - "name": f["name"], - "type": f["type"], - "label": f.get("label") or "", - "options": f.get("options") or [], - "value": values.get(f["name"], f.get("value", "")), - "rect_px": [ - int(x0 * scale), int(y0 * scale), - int(x1 * scale), int(y1 * scale), - ], - }) - pages_out.append({ - "page": page_no, - "width": img_w, - "height": img_h, - "fields": fields_out, - }) - return {"doc_id": doc_id, "scale": scale, "pages": pages_out} - finally: - pdf_doc.close() - finally: - db.close() - - # ---- GET /api/document/{doc_id}/page/{n}.png ---- - @router.get("/api/document/{doc_id}/page/{page_no}.png") - async def render_page_png(doc_id: str, page_no: int, request: Request): - """Render one page of the source PDF as a PNG (no values stamped — the - frontend overlays HTML form inputs on top).""" - from fastapi.responses import Response - from src.pdf_form_doc import find_source_upload_id - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, "Source PDF not found") - finally: - db.close() - - fitz = _load_pdf_viewer_fitz() - pdf_doc = fitz.open(pdf_path) - try: - if page_no < 1 or page_no > pdf_doc.page_count: - raise HTTPException(404, "Page out of range") - page = pdf_doc[page_no - 1] - mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) - pix = page.get_pixmap(matrix=mat, alpha=False) - png_bytes = pix.tobytes("png") - return Response( - content=png_bytes, - media_type="image/png", - headers={"Cache-Control": "public, max-age=3600"}, - ) - finally: - pdf_doc.close() - - # ---- POST /api/document/{doc_id}/ai-fill-annotations ---- - @router.post("/api/document/{doc_id}/ai-fill-annotations") - async def ai_fill_annotations(doc_id: str, request: Request) -> Dict[str, Any]: - """Ask a vision-capable LLM to locate fillable areas on a flat PDF and - propose annotation values for each, given a free-form user instruction. - - Returns a list of annotations: [{page, x, y, w, h, value}] where x/y/w/h - are page-percentages (0–100) — same coordinate system as the freeform - annotations the frontend already renders. - """ - import base64 - import json - import fitz - from src.pdf_form_doc import find_source_upload_id - from src.document_processor import _resolve_vl_model, _load_vl_settings - from src.llm_core import llm_call_async - - body = await request.json() if request.headers.get("content-type", "").startswith("application/json") else {} - instruction = (body or {}).get("instruction", "").strip() - if not instruction: - raise HTTPException(400, "instruction is required") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, "Source PDF not found") - finally: - db.close() - - # Resolve VL model (admin-configured or auto-detected vision-capable) - settings = _load_vl_settings() - vl_model = settings.get("vision_model", "") - try: - url, model_id, headers = _resolve_vl_model(vl_model, owner=user) - except Exception as e: - raise HTTPException(503, f"No vision model available: {e}") - - system_prompt = ( - "You analyze rendered PDF page images and propose values to fill in. " - "For each blank line, box, underscore, or labeled space on the page that " - "should be filled given the user's instruction, output one annotation. " - "Coordinates are percentages (0-100) of the page width/height with the " - "origin at top-left. Width/height should match the visible blank box. " - "Return ONLY a JSON array, no prose, no markdown fences. Each entry: " - '{"x": number, "y": number, "w": number, "h": number, "value": string}. ' - "If a region should not be filled, omit it. If nothing should be filled, " - "return []." - ) - - all_annotations = [] - pdf_doc = fitz.open(pdf_path) - try: - for page_index in range(pdf_doc.page_count): - page = pdf_doc[page_index] - mat = fitz.Matrix(_PDF_RENDER_SCALE, _PDF_RENDER_SCALE) - pix = page.get_pixmap(matrix=mat, alpha=False) - png_bytes = pix.tobytes("png") - b64 = base64.b64encode(png_bytes).decode("ascii") - - messages = [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": [ - { - "type": "text", - "text": ( - f"User instruction:\n{instruction}\n\n" - f"This is page {page_index + 1} of {pdf_doc.page_count}. " - "Return JSON array of annotations to add to this page." - ), - }, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64}"}, - }, - ], - }, - ] - try: - raw = await llm_call_async( - url, model_id, messages, - temperature=0.1, max_tokens=2000, headers=headers, - ) - except Exception as e: - logger.error(f"VL call failed on page {page_index + 1}: {e}") - continue - - raw = (raw or "").strip() - if raw.startswith("```"): - raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip() - try: - parsed = json.loads(raw) - except Exception: - logger.warning(f"AI fill: page {page_index + 1} returned non-JSON: {raw[:200]}") - continue - if not isinstance(parsed, list): - continue - for item in parsed: - if not isinstance(item, dict): - continue - try: - x = float(item.get("x", 0)) - y = float(item.get("y", 0)) - w = float(item.get("w", 0)) - h = float(item.get("h", 0)) - value = str(item.get("value", "") or "") - except Exception: - continue - # Clamp + reject zero-size entries - if w <= 0.5 or h <= 0.3: - continue - x = max(0.0, min(99.0, x)) - y = max(0.0, min(99.0, y)) - w = max(0.5, min(100.0 - x, w)) - h = max(0.3, min(100.0 - y, h)) - if not value.strip(): - continue - all_annotations.append({ - "page": page_index + 1, - "x": round(x, 2), - "y": round(y, 2), - "w": round(w, 2), - "h": round(h, 2), - "value": value, - }) - finally: - pdf_doc.close() - - return {"annotations": all_annotations} - - # ---- GET /api/document/{doc_id}/render-pdf ---- - @router.get("/api/document/{doc_id}/render-pdf") - async def render_pdf(doc_id: str, request: Request): - """Inline PDF preview filled with the current markdown values. - - Same plumbing as the export route, but no signature stamping and - served inline (Content-Disposition: inline) so the browser can - embed it in an iframe. Cache-busted by the caller via query string. - """ - import base64 - import os - import tempfile - from fastapi.responses import FileResponse - from starlette.background import BackgroundTask - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, parse_markdown_annotations - from src.pdf_forms import fill_fields, stamp_annotations - from core.database import Signature - - # Track temp files for this request so they get unlinked AFTER - # the response is fully sent (BackgroundTask runs post-send). - _to_unlink: list[str] = [] - def _cleanup_temps(): - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - # Fail fast with a clear 503 if the optional PyMuPDF dependency - # is missing — fill_fields/stamp_annotations will otherwise - # raise RuntimeError deep inside and bubble out as a 500. - # Mirrors the convention in _load_pdf_viewer_fitz above. - _load_pdf_viewer_fitz() - - values = parse_markdown_to_values(doc.current_content or "") - out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(out_path) - try: - fill_fields(pdf_path, out_path, values) - except Exception as e: - logger.error(f"render_pdf fill_fields failed for {doc_id}: {e}") - _cleanup_temps() - raise HTTPException(500, f"PDF render failed: {e}") - - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad annotation signature data for {s.id}: {e}") - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.error(f"stamp_annotations (render) failed for {doc_id}: {e}") - - return FileResponse( - out_path, - media_type="application/pdf", - headers={"Content-Disposition": "inline"}, - background=BackgroundTask(_cleanup_temps), - ) - finally: - db.close() - - # ---- GET /api/document/{doc_id}/export-pdf ---- - @router.get("/api/document/{doc_id}/export-pdf") - async def export_pdf(doc_id: str, request: Request): - """Stream the filled PDF for download. - - Reads field values and signature selections from the markdown — there - is no separate confirmation step. Signature fields contain their - chosen signature ID encoded as `signature:` in the value. - """ - import base64 - import os - import tempfile - from fastapi.responses import FileResponse - from starlette.background import BackgroundTask - from src.pdf_form_doc import find_source_upload_id, parse_markdown_to_values, load_field_sidecar, parse_markdown_annotations - from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations - from core.database import Signature - - _to_unlink: list[str] = [] - def _cleanup_temps(): - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found in uploads") - - schema = load_field_sidecar(pdf_path) or [] - sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} - - all_values = parse_markdown_to_values(doc.current_content or "") - # Split: signature fields go to stamps, everything else to fill_fields - text_values: dict = {} - sig_ids: dict[str, str] = {} - for name, raw in all_values.items(): - if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): - sig_ids[name] = raw[len("signature:"):].strip() - elif name not in sig_field_names: - text_values[name] = raw - - stamps: dict = {} - if sig_ids: - # SECURITY: filter by owner — same reason as render_pdf. - _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) - if user: - _sig_q2 = _sig_q2.filter(Signature.owner == user) - rows = _sig_q2.all() - by_id = {s.id: s for s in rows} - for field_name, sid in sig_ids.items(): - s = by_id.get(sid) - if not s: - continue - try: - stamps[field_name] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad signature data for {sid}: {e}") - - filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(filled_path) - try: - fill_fields(pdf_path, filled_path, text_values) - except Exception as e: - logger.error(f"fill_fields failed for doc {doc_id}: {e}") - _cleanup_temps() - raise HTTPException(500, f"PDF fill failed: {e}") - - out_path = filled_path - if stamps: - stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(stamped_path) - try: - stamp_signatures(filled_path, stamped_path, stamps) - out_path = stamped_path - except Exception as e: - logger.error(f"stamp_signatures failed for doc {doc_id}: {e}") - - # Burn freeform annotations (Text/Check/Sign drops) on top. - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - # Resolve any signature annotations to their PNG bytes. - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception as e: - logger.warning(f"Bad annotation signature data for {s.id}: {e}") - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.error(f"stamp_annotations failed for doc {doc_id}: {e}") - - download_name = _slug(doc.title or "form") + "_annotated.pdf" - return FileResponse( - out_path, - media_type="application/pdf", - filename=download_name, - background=BackgroundTask(_cleanup_temps), - ) - finally: - db.close() - - # ---- POST /api/document/{doc_id}/prepare-signed-reply ---- - @router.post("/api/document/{doc_id}/prepare-signed-reply") - async def prepare_signed_reply(doc_id: str, request: Request): - """Bake the current PDF state (form fields + signature stamps + - annotations) into a flattened PDF, drop it in COMPOSE_UPLOADS_DIR - and return the reply context (To/Subject/threading headers) so the - frontend can open a reply draft with this attachment pre-loaded. - - Requires the document to have source_email_* metadata (set when the - doc was created via /api/email/attachment-as-doc). Otherwise 400. - """ - import base64 - import tempfile - import shutil - import uuid as _uuid - import email as _email_mod - from src.pdf_form_doc import ( - find_source_upload_id, parse_markdown_to_values, - load_field_sidecar, parse_markdown_annotations, - ) - from src.pdf_forms import fill_fields, stamp_signatures, stamp_annotations - from core.database import Signature - # COMPOSE_UPLOADS_DIR lives in email_routes — re-derive here so we - # don't import from a routes file (cycle-prone). Same env override - # as email_routes (ODYSSEUS_MAIL_ATTACHMENTS_DIR). - from pathlib import Path as _Path - _COMPOSE_DIR = _Path(MAIL_ATTACHMENTS_DIR) / "_compose" - _COMPOSE_DIR.mkdir(parents=True, exist_ok=True) - - user = get_current_user(request) - db = SessionLocal() - try: - doc = db.query(Document).filter(Document.id == doc_id).first() - if not doc: - raise HTTPException(404, "Document not found") - _verify_doc_owner(db, doc, user) - - if not (doc.source_email_uid and doc.source_email_folder): - raise HTTPException(400, "Document has no source email — cannot reply") - - # 1) Build the flattened PDF (same pipeline as export_pdf) - upload_id = find_source_upload_id(doc.current_content or "") - if not upload_id: - raise HTTPException(400, "Document is not linked to a source PDF") - pdf_path = _locate_current_user_upload(request, upload_id, user) - if not pdf_path: - raise HTTPException(404, f"Source PDF {upload_id} not found") - - schema = load_field_sidecar(pdf_path) or [] - sig_field_names = {f["name"] for f in schema if f.get("type") == "signature"} - all_values = parse_markdown_to_values(doc.current_content or "") - text_values: dict = {} - sig_ids: dict[str, str] = {} - for name, raw in all_values.items(): - if name in sig_field_names and isinstance(raw, str) and raw.startswith("signature:"): - sig_ids[name] = raw[len("signature:"):].strip() - elif name not in sig_field_names: - text_values[name] = raw - - stamps: dict = {} - if sig_ids: - # SECURITY: filter by owner — same reason as render_pdf. - _sig_q2 = db.query(Signature).filter(Signature.id.in_(list(sig_ids.values()))) - if user: - _sig_q2 = _sig_q2.filter(Signature.owner == user) - rows = _sig_q2.all() - by_id = {s.id: s for s in rows} - for fname, sid in sig_ids.items(): - s = by_id.get(sid) - if not s: - continue - try: - stamps[fname] = base64.b64decode(s.data_png) - except Exception: - pass - - import os - _to_unlink: list[str] = [] - filled_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(filled_path) - fill_fields(pdf_path, filled_path, text_values) - out_path = filled_path - if stamps: - stamped_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(stamped_path) - try: - stamp_signatures(filled_path, stamped_path, stamps) - out_path = stamped_path - except Exception as e: - logger.warning(f"stamp_signatures failed for {doc_id}: {e}") - - annotations = parse_markdown_annotations(doc.current_content or "") - if annotations: - ann_sig_ids = [ - a["value"][len("signature:"):].strip() - for a in annotations - if a.get("kind") == "signature" - and isinstance(a.get("value"), str) - and a["value"].startswith("signature:") - ] - ann_signature_pngs: dict[str, bytes] = {} - if ann_sig_ids: - # SECURITY: filter by owner so a caller can't reference - # someone else's signature ID from doc markdown and have - # it stamped/exported. - _sig_q = db.query(Signature).filter(Signature.id.in_(ann_sig_ids)) - if user: - _sig_q = _sig_q.filter(Signature.owner == user) - sig_rows = _sig_q.all() - for s in sig_rows: - try: - ann_signature_pngs[s.id] = base64.b64decode(s.data_png) - except Exception: - pass - annotated_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name - _to_unlink.append(annotated_path) - try: - stamp_annotations(out_path, annotated_path, annotations, ann_signature_pngs) - out_path = annotated_path - except Exception as e: - logger.warning(f"stamp_annotations failed for {doc_id}: {e}") - - # 2) Move/copy into COMPOSE_UPLOADS_DIR with the token format - # `_` that /api/email/send expects. - filename = _slug(doc.title or "signed") + "_signed.pdf" - token = f"{_uuid.uuid4().hex}_{filename}" - dest = _COMPOSE_DIR / token - shutil.copyfile(out_path, str(dest)) - # Unlink the intermediate temp PDFs now that they've been - # copied into COMPOSE_UPLOADS_DIR. - for _p in _to_unlink: - try: - os.unlink(_p) - except FileNotFoundError: - pass - except Exception as _e: - logger.warning(f"Could not unlink temp PDF {_p}: {_e}") - - # 3) Fetch the source email's headers so we can build a clean reply - # context (To/Subject/In-Reply-To/References). - try: - from routes.email_routes import _imap, _decode_header - from routes.email_helpers import _q - except Exception: - _imap = None - _decode_header = lambda x: x or "" - _q = lambda x: x or "" - - to_addr = "" - from_name = "" - subject = "" - in_reply_to = doc.source_email_message_id or "" - references = in_reply_to - if _imap: - try: - with _imap(doc.source_email_account_id or None) as conn: - conn.select(_q(doc.source_email_folder), readonly=True) - status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)") - if status == "OK" and data and data[0]: - raw_hdr = data[0][1] - m = _email_mod.message_from_bytes(raw_hdr) - sender = _decode_header(m.get("From", "")) - from_name, to_addr = _email_mod.utils.parseaddr(sender) - if not to_addr: - to_addr = sender - subject = _decode_header(m.get("Subject", "") or "") - if subject and not subject.lower().startswith("re:"): - subject = "Re: " + subject - msg_refs = (m.get("References") or "").strip() - msg_in_reply = (m.get("Message-ID") or "").strip() or in_reply_to - in_reply_to = msg_in_reply - references = (msg_refs + " " + msg_in_reply).strip() if msg_refs else msg_in_reply - except Exception as e: - logger.warning(f"prepare-signed-reply header fetch failed: {e}") - - return { - "ok": True, - "attachment": { - "token": token, - "filename": filename, - "size": dest.stat().st_size, - }, - "reply": { - "to": to_addr, - "to_name": from_name, - "subject": subject, - "in_reply_to": in_reply_to, - "references": references, - "account_id": doc.source_email_account_id or None, - "source_uid": doc.source_email_uid, - "source_folder": doc.source_email_folder, - "source_message_id": doc.source_email_message_id, - }, - } - finally: - db.close() - - return router +_sys.modules[__name__] = _canonical diff --git a/routes/email_helpers.py b/routes/email_helpers.py index c8639e1c7..257f5f921 100644 --- a/routes/email_helpers.py +++ b/routes/email_helpers.py @@ -247,6 +247,7 @@ import re as _re_reply _REPLY_OPEN_RE = _re_reply.compile(r"<<<\s*(?:REPLY|SUMMARY|OUTPUT)\s*>>+", _re_reply.I) _REPLY_CLOSE_RE = _re_reply.compile(r"<<<\s*END\s*>>+", _re_reply.I) _REPLY_ROLE_MARKER_RE = _re_reply.compile(r"?|?", _re_reply.I) +_SUMMARY_BULLET_RE = _re_reply.compile(r"^(?:[-*\u2022]\s+|\d+[.)]\s+)") def _extract_reply(text: str) -> str: @@ -277,6 +278,125 @@ def _extract_reply(text: str) -> str: return _strip_think(t).strip() +def _build_email_summary_messages(sender: str, subject: str, body_for_llm: str) -> list[dict[str, str]]: + return [ + { + "role": "system", + "content": ( + "You are an email summarizer. Format: 1-3 short bullet points " + "(use '- '). Cover: main point, action items, deadlines. If the " + "email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR " + "CONTENTS - pull invoice totals, deadlines, key clauses, concrete " + "numbers/dates from PDFs/docs into the bullets. Be terse.\n\n" + "OUTPUT FORMAT: Put ONLY the bullet points between these exact " + "markers, each on its own line:\n" + "<<>>\n" + "- ...\n" + "<<>>\n" + "Any reasoning must come BEFORE <<>> (ideally inside " + "...). Only the text between the markers is kept." + ), + }, + { + "role": "user", + "content": ( + f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}" + "\n\n---\n\nSummarize the email. Output the bullets between " + "<<>> and <<>>." + ), + }, + ] + + +async def _generate_email_summary( + url: str, + model: str, + sender: str, + subject: str, + body_for_llm: str, + *, + headers: dict | None = None, + max_tokens: int = 8192, + timeout: int = 180, +) -> str: + """Generate an interactive email summary through the shared LLM adapter.""" + from src.llm_core import llm_call_async + + raw = await llm_call_async( + url=url, + model=model, + messages=_build_email_summary_messages(sender, subject, body_for_llm), + temperature=0.3, + max_tokens=max_tokens, + headers=headers, + timeout=timeout, + workload="foreground", + ) + return _normalize_email_summary(raw) + + +async def _generate_scheduled_email_summary( + url: str, + model: str, + sender: str, + subject: str, + body_for_llm: str, + *, + headers: dict | None = None, + owner: str | None = None, + max_tokens: int = 8192, + timeout: int = 180, +) -> str: + """Generate a scheduled summary through the background task candidate chain.""" + from src.task_endpoint import task_llm_call_async + + raw = await task_llm_call_async( + messages=_build_email_summary_messages(sender, subject, body_for_llm), + fallback_url=url, + fallback_model=model, + fallback_headers=headers, + owner=owner, + temperature=0.3, + max_tokens=max_tokens, + timeout=timeout, + ) + return _normalize_email_summary(raw) + + +def _normalize_email_summary(raw) -> str: + """Extract a stable cache/UI summary from provider output.""" + raw_text = raw or "" + if _REPLY_OPEN_RE.search(raw_text): + summary = _extract_reply(raw_text) + if summary: + return summary + + cleaned = _strip_think(raw_text).strip() + bullets = [ + line.strip() + for line in cleaned.splitlines() + if _SUMMARY_BULLET_RE.match(line.strip()) + ] + if bullets: + return "\n".join(bullets) + return cleaned.strip() + + +EMAIL_SUMMARY_ERROR_CODE = "email_summary_unavailable" +EMAIL_SUMMARY_ERROR_MESSAGE = "Failed to summarize" + + +def _email_summary_failure_log_detail(exc: BaseException) -> str: + """Return useful provider-failure metadata without echoing exception text.""" + detail = f"type={type(exc).__name__}" + status = getattr(exc, "status_code", None) + if status is None: + status = getattr(getattr(exc, "response", None), "status_code", None) + if isinstance(status, int): + detail += f" status={status}" + return detail + + def _apply_email_style_mechanics(text: str) -> str: """Enforce deterministic writing-style mechanics that models often miss.""" if not text: diff --git a/routes/email_pollers.py b/routes/email_pollers.py index 5d96bd0f9..a2507989d 100644 --- a/routes/email_pollers.py +++ b/routes/email_pollers.py @@ -40,6 +40,7 @@ from routes.email_helpers import ( _pre_retrieve_context, _attach_compose_uploads, _cleanup_compose_uploads, _q, SCHEDULED_DB, _EMAIL_REPLY_SYS_PROMPT_BASE, _email_cache_owner_clause, + _generate_scheduled_email_summary, _email_summary_failure_log_detail, ) logger = logging.getLogger(__name__) @@ -653,6 +654,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None no_msgid = 0 examined = 0 _summaries_created = 0 + _summary_failed = 0 _events_created = 0 _replies_drafted = 0 _reply_failed = 0 @@ -785,16 +787,17 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if need_sum: try: - summary = await task_llm_call_async( - messages=[ - {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<>>\n- ...\n<<>>\nAny reasoning or planning must come BEFORE <<>> (ideally inside ...). Only the text between the markers is kept."}, - {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<>> and <<>>."}, - ], - fallback_url=url, fallback_model=model, fallback_headers=headers, + summary = await _generate_scheduled_email_summary( + url=url, + model=model, + sender=sender, + subject=subject, + body_for_llm=body_for_llm, + headers=req_headers, owner=account_owner or None, - temperature=0.3, max_tokens=16384, timeout=240, + max_tokens=16384, + timeout=240, ) - summary = _extract_reply((summary or "").strip()) if summary: _c = _sql3.connect(SCHEDULED_DB) _c.execute(""" @@ -808,10 +811,19 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None _summaries_created += 1 _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) _detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") + else: + _summary_failed += 1 + _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) + _detail_lines.append(f"summary empty · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") except Exception as e: + _summary_failed += 1 _uid_text = uid.decode() if isinstance(uid, bytes) else str(uid) _detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}") - logger.warning(f"Auto-summary {uid} failed: {e}") + logger.warning( + "Auto-summary uid=%s failed %s", + _uid_text, + _email_summary_failure_log_detail(e), + ) if need_reply: await _emit_progress(progress_cb, f"Drafting reply {processed + 1}/{_max_process} · checked {examined}/{len(uid_list)}") @@ -1320,6 +1332,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None parts.append(f"processed {processed} new") if auto_sum: parts.append(f"summarized {_summaries_created}") + if _summary_failed: + parts.append(f"{_summary_failed} summary failed") if auto_reply_draft: parts.append(f"drafted {_replies_drafted} repl" + ("y" if _replies_drafted == 1 else "ies")) if _reply_failed: diff --git a/routes/email_routes.py b/routes/email_routes.py index 3c8e407bd..a28a68118 100644 --- a/routes/email_routes.py +++ b/routes/email_routes.py @@ -45,6 +45,7 @@ from src.upload_limits import read_upload_limited, EMAIL_COMPOSE_UPLOAD_MAX_BYTE from routes.email_helpers import ( _strip_think, _extract_reply, _apply_email_style_mechanics, require_owner, require_user, _assert_owns_account, + _account_visible_to_owner, _q, _attach_compose_uploads, _cleanup_compose_uploads, _load_settings, _save_settings, _get_email_config, _send_smtp_message, _smtp_security_mode, @@ -57,7 +58,8 @@ from routes.email_helpers import ( _extract_attachment_to_disk, _extract_html, _extract_text, _fetch_sender_thread_context, _pre_retrieve_context, _EMAIL_REPLY_SYS_PROMPT_BASE, _POOL_HOOKS, - _friendly_email_auth_error, + _friendly_email_auth_error, _email_summary_failure_log_detail, + _generate_email_summary, EMAIL_SUMMARY_ERROR_CODE, EMAIL_SUMMARY_ERROR_MESSAGE, SendEmailRequest, ExtractStyleRequest, ATTACHMENTS_DIR, COMPOSE_UPLOADS_DIR, SCHEDULED_DB, attachment_extract_dir, _email_cache_owner_clause, email_translation_body_hash, @@ -194,6 +196,64 @@ def _coerce_port(value, default): return None, f"Invalid port {value!r}; must be a whole number" +def _lock_email_account_owner_mutation(db, *owners: str) -> None: + """Delegate account/default serialization to the shared DB primitive.""" + from core.database import lock_email_account_owner_mutations + + lock_email_account_owner_mutations(db, *owners) + + +def _email_account_owner_scope(query, owner: str): + """Restrict a query to one normalized EmailAccount owner partition.""" + from core.database import EmailAccount + from sqlalchemy import or_ + + if owner: + return query.filter(EmailAccount.owner == owner) + return query.filter(or_(EmailAccount.owner == None, EmailAccount.owner == "")) # noqa: E711 + + +def _discover_email_account_mutation_scope(account_id: str, owner: str) -> str: + """Read the initial lock key and fail closed before a mutation session.""" + from core.database import EmailAccount, SessionLocal + + db = SessionLocal() + try: + row = db.get(EmailAccount, account_id) + if row is None or (owner and not _account_visible_to_owner(row, owner)): + raise HTTPException(404, "Account not found") + return row.owner or "" + except HTTPException: + raise + except Exception as exc: + logger.error("Account-owner mutation check failed: %s", exc) + raise HTTPException(503, "Account check failed") + finally: + db.close() + + +def _lock_and_reload_email_account(db, account_id: str, owner: str, scope: str): + """Lock, reload, and revalidate an account, retrying if its owner moved.""" + from core.database import EmailAccount + + owner_scopes = {scope or ""} + while True: + _lock_email_account_owner_mutation(db, *owner_scopes) + row = db.get(EmailAccount, account_id, populate_existing=True) + if row is None or (owner and not _account_visible_to_owner(row, owner)): + raise HTTPException(404, "Account not found") + + current_scope = row.owner or "" + if current_scope in owner_scopes or db.get_bind().dialect.name == "sqlite": + return row + + # The account changed owner after discovery but before lock acquisition. + # Release the partial lock set and reacquire all observed scopes in the + # shared helper's canonical order, then validate from the database again. + db.rollback() + owner_scopes.add(current_scope) + + def _email_tag_owner_aliases(account_id: str | None, owner: str = "") -> list[str]: aliases = [owner or ""] try: @@ -2860,13 +2920,22 @@ def setup_email_routes(): return indexed_response return {"emails": [], "total": 0, "error": "Mail operation failed"} - def _read_email_sync(uid, folder, account_id, owner, mark_seen=True, full=False): + def _read_email_sync(uid, folder, account_id, owner, mark_seen=False, full=False): """Sync IMAP read — wrapped in to_thread by the async handler. The normal reader path fetches the headers plus a bounded body prefix. That avoids downloading multi-megabyte attachments just to open a message. Full-message fetch remains available for flows that need attachment metadata immediately, such as forwarding. + + `mark_seen` defaults to False because it mutates provider state: it + selects the mailbox read-write and issues a STORE. Only a foreground + open should ask for it, and it has to ask explicitly. + + A failed \\Seen transition is reported as `mark_seen_failed` on an + otherwise normal response, never as an error. The body has already been + fetched at that point, so refusing to return it would turn a cosmetic + flag failure into an unreadable message. """ import time as _t _t0 = _t.monotonic() @@ -2874,9 +2943,28 @@ def setup_email_routes(): preview_bytes = 384 * 1024 _t_select = 0.0 _t_fetch = 0.0 + mark_seen_failed = False try: with _imap(account_id, owner=owner) as conn: - conn.select(_q(folder), readonly=True) + # A foreground open owns both the body fetch and the \Seen + # transition. Keep them on one read-write IMAP selection so the + # route never schedules a second connection that can race the + # response. Prefetch/read-only callers retain BODY.PEEK and a + # read-only mailbox selection. + try: + conn.select(_q(folder), readonly=not mark_seen) + except Exception as select_exc: + if not mark_seen: + raise + # Read-only mailboxes (shared archives, some provider + # folders) reject a read-write SELECT. Serve the message + # read-only and report the flag failure. + logger.warning( + f"read-write SELECT rejected for {folder!r}; " + f"serving read-only without \\Seen: {select_exc}" + ) + conn.select(_q(folder), readonly=True) + mark_seen_failed = True _t_select = _t.monotonic() - _t0 fetch_query = "(BODY.PEEK[])" if full else f"(BODY.PEEK[HEADER] BODY.PEEK[TEXT]<0.{preview_bytes}>)" status, msg_data = _imap_uid_fetch(conn, uid, fetch_query) @@ -2902,22 +2990,44 @@ def setup_email_routes(): header_part = msg_data[0][1] or b"" raw = header_part + b"\r\n" + text_part - msg = email_mod.message_from_bytes(raw) + # Parse the fetched payload before mutating provider state. If + # the message is malformed enough that the reader cannot build + # a response, the caller gets an error while the message stays + # unread instead of receiving a false optimistic rollback. + msg = email_mod.message_from_bytes(raw) - subject = _decode_header(msg.get("Subject", "(no subject)")) - sender = _decode_header(msg.get("From", "unknown")) - to = _decode_header(msg.get("To", "")) - cc = _decode_header(msg.get("Cc", "")) - date_str = msg.get("Date", "") - message_id = msg.get("Message-ID", "") - in_reply_to = msg.get("In-Reply-To", "") - references = msg.get("References", "") - body = _extract_text(msg) - body_html = _extract_html(msg) + subject = _decode_header(msg.get("Subject", "(no subject)")) + sender = _decode_header(msg.get("From", "unknown")) + to = _decode_header(msg.get("To", "")) + cc = _decode_header(msg.get("Cc", "")) + date_str = msg.get("Date", "") + message_id = msg.get("Message-ID", "") + in_reply_to = msg.get("In-Reply-To", "") + references = msg.get("References", "") + body = _extract_text(msg) + body_html = _extract_html(msg) + + sender_name, sender_addr = email.utils.parseaddr(sender) + parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None + attachments = _list_attachments_from_msg(msg) if full else (_email_attachment_meta_cache_get(owner, account_id, folder, uid) or []) + + if mark_seen and not mark_seen_failed: + seen_status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)") + if seen_status != "OK": + # Report, don't raise. The parsed body below is still a + # valid response; only the flag claim is untrue. + logger.warning( + f"IMAP STORE \\Seen failed for UID {uid} in {folder!r}: {seen_status}" + ) + mark_seen_failed = True + + # Only record the local flag transition when the provider actually + # accepted it, so the index and list cache cannot drift ahead of + # the mailbox. + if mark_seen and not mark_seen_failed: + _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) + _update_list_cache_seen(account_id, folder, uid, True) - sender_name, sender_addr = email.utils.parseaddr(sender) - parsed_date = email.utils.parsedate_to_datetime(date_str) if date_str else None - attachments = _list_attachments_from_msg(msg) if full else (_email_attachment_meta_cache_get(owner, account_id, folder, uid) or []) related_attachments = [] if full and not _has_visible_attachments(msg): related_attachments = _related_thread_attachments_sync( @@ -3038,20 +3148,29 @@ def setup_email_routes(): "boundaries": cached_boundaries, "thread_turns": cached_turns, "sender_signature": cached_sender_sig, + # Per-request, not part of the message: the route strips this + # before caching so a one-off flag failure is never replayed to + # later readers. + "mark_seen_failed": mark_seen_failed, } except Exception as e: logger.error(f"Failed to read email {uid}: {e}") return {"error": "Mail operation failed"} def _mark_email_seen_sync(uid, folder, account_id, owner): + """Synchronously mark a cached email seen and report success.""" try: with _imap(account_id, owner=owner) as conn: - conn.select(_q(folder)) - conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "\\Seen") + conn.select(_q(folder), readonly=False) + status, _ = conn.uid("STORE", _uid_bytes(uid), "+FLAGS", "(\\Seen)") + if status != "OK": + return False _email_index_update_flags(owner, account_id, folder, uid, "\\Seen", True) _update_list_cache_seen(account_id, folder, uid, True) + return True except Exception as e: - logger.debug(f"mark-seen after cached read failed uid={uid}: {e}") + logger.warning(f"mark-seen after cached read failed uid={uid}: {e}") + return False @router.get("/read/{uid}") async def read_email_by_uid( @@ -3077,32 +3196,32 @@ def setup_email_routes(): if cached.get("attachment_version") != EMAIL_READ_ATTACHMENT_VERSION: cached = None if cached is not None: - if mark_seen: - try: - _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner)) - except RuntimeError: - pass + # A cache hit already holds a complete, valid message. Await the + # STORE so the response reports the real flag state, but never let + # a failed STORE withhold a body we are holding in memory. + if mark_seen and not await _asyncio.to_thread( + _mark_email_seen_sync, uid, folder, account_id, owner + ): + return {**cached, "mark_seen_failed": True} return cached if not full: persisted = _email_preview_cache_get(owner, account_id, folder, uid) if persisted and persisted.get("attachment_version") == EMAIL_READ_ATTACHMENT_VERSION: _read_cache_put(ck, persisted) - if mark_seen: - try: - _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner)) - except RuntimeError: - pass + if mark_seen and not await _asyncio.to_thread( + _mark_email_seen_sync, uid, folder, account_id, owner + ): + return {**persisted, "mark_seen_failed": True} return persisted result = await _asyncio.to_thread(_read_email_sync, uid, folder, account_id, owner, mark_seen, full) if result and not result.get("error"): - _read_cache_put(ck, result) + # `mark_seen_failed` describes this request, not the message, so it + # must not enter either cache — a later reader would otherwise be + # told a STORE failed that it never issued. + cacheable = {k: v for k, v in result.items() if k != "mark_seen_failed"} + _read_cache_put(ck, cacheable) if not full: - _email_preview_cache_put(owner, account_id, folder, uid, result) - if mark_seen: - try: - _asyncio.create_task(_asyncio.to_thread(_mark_email_seen_sync, uid, folder, account_id, owner)) - except RuntimeError: - pass + _email_preview_cache_put(owner, account_id, folder, uid, cacheable) return result def _schedule_recent_email_warm(emails: list, folder: str, account_id: str | None, owner: str): @@ -4766,8 +4885,6 @@ def setup_email_routes(): """Generate a quick AI summary of an email body.""" try: from src.endpoint_resolver import resolve_endpoint - from src.llm_core import _uses_max_completion_tokens, _restricts_temperature - import requests as _req body = data.get("body", "") subject = data.get("subject", "") @@ -4778,7 +4895,11 @@ def setup_email_routes(): if account_id: _assert_owns_account(account_id, owner) if not body: - return {"success": False, "error": "No body provided"} + return { + "success": False, + "error": "No body provided", + "error_code": "email_summary_missing_body", + } # If we know which UID this is, fetch the raw message and pull # attachment text so the summary can reference invoice totals, @@ -4807,53 +4928,43 @@ def setup_email_routes(): if not url: url, model, headers = resolve_endpoint("default", owner=owner) if not url or not model: - return {"success": False, "error": "No LLM endpoint configured"} + return { + "success": False, + "error": "No model configured for email summaries", + "error_code": "email_summary_not_configured", + } req_headers = {"Content-Type": "application/json"} if headers: req_headers.update(headers) - tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" - payload = { - "model": model, - "messages": [ - {"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull invoice totals, deadlines, key clauses, concrete numbers/dates from PDFs/docs into the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<>>\n- ...\n<<>>\nAny reasoning must come BEFORE <<>> (ideally inside ...). Only the text between the markers is kept."}, - {"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<>> and <<>>."}, - ], - tok_key: 8192, - "temperature": 0.3, - "stream": False, - } - # Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature. - if _restricts_temperature(model): - payload.pop("temperature", None) - resp = await asyncio.to_thread( - _req.post, url, json=payload, headers=req_headers, timeout=180 - ) - if not resp.ok: - return {"success": False, "error": f"LLM HTTP {resp.status_code}"} - rdata = resp.json() - msg = (rdata.get("choices") or [{}])[0].get("message", {}) - content = (msg.get("content") or "").strip() - content = _extract_reply(content) + try: + content = await _generate_email_summary( + url=url, + model=model, + sender=sender, + subject=subject, + body_for_llm=body_for_llm, + headers=req_headers, + max_tokens=8192, + timeout=180, + ) + except Exception as e: + logger.warning( + "Email summary LLM call failed %s", + _email_summary_failure_log_detail(e), + ) + return { + "success": False, + "error": EMAIL_SUMMARY_ERROR_MESSAGE, + "error_code": EMAIL_SUMMARY_ERROR_CODE, + } if not content: - # Model put everything in reasoning_content — extract bullet points - rc = (msg.get("reasoning_content") or "").strip() - # Find bullet-point style output (lines starting with -, •, *, or numbered) - bullet_lines = [] - for line in rc.split("\n"): - stripped = line.strip() - if re.match(r"^[-•*]\s+|^\d+[.)]\s+", stripped): - bullet_lines.append(stripped) - if bullet_lines: - content = "\n".join(bullet_lines) - else: - # Last resort: take the last paragraph - paragraphs = [p.strip() for p in rc.split("\n\n") if p.strip()] - content = paragraphs[-1] if paragraphs else rc[:500] - - if not content: - return {"success": False, "error": "Empty response from model"} + return { + "success": False, + "error": "The model returned an empty summary", + "error_code": "email_summary_empty", + } # Cache the summary if we have a message_id mid = data.get("message_id", "") @@ -4876,8 +4987,15 @@ def setup_email_routes(): return {"success": True, "summary": content, "model_used": model} except Exception as e: - logger.error(f"Failed to summarize: {e}") - return {"success": False, "error": "Mail operation failed"} + logger.error( + "Email summary route failed %s", + _email_summary_failure_log_detail(e), + ) + return { + "success": False, + "error": EMAIL_SUMMARY_ERROR_MESSAGE, + "error_code": EMAIL_SUMMARY_ERROR_CODE, + } @router.post("/translate") async def translate_email(data: dict, owner: str = Depends(require_owner)): @@ -5209,9 +5327,9 @@ def setup_email_routes(): # Build a candidate chain so a stale session-stored API key # (the most common cause of "authentication failed" here) # doesn't kill AI Reply outright — fall through to the - # user's Utility / Default endpoints AND their configured - # fallback chains. Dedupe by url+model so we don't retry - # the same broken endpoint. + # user's Utility / Default endpoints and the active Utility + # fallback chain. The retired default-fallback hook stays empty. + # Dedupe by url+model so we don't retry the same broken endpoint. from src.llm_core import llm_call_async_with_fallback from src.endpoint_resolver import ( resolve_utility_fallback_candidates, @@ -5240,7 +5358,7 @@ def setup_email_routes(): _add(_d_url, _d_model, _d_headers) except Exception: pass - # Configured fallback chains last. + # Active Utility fallbacks, then the retired default hook. for cand in resolve_utility_fallback_candidates(owner=owner) or []: _add(*cand) for cand in resolve_chat_fallback_candidates(owner=owner) or []: @@ -5428,9 +5546,9 @@ def setup_email_routes(): import uuid as _uuid db = SessionLocal() try: + _lock_email_account_owner_mutation(db, owner) q = db.query(EmailAccount).filter(EmailAccount.is_default == True) # noqa: E712 - if owner: - q = q.filter(EmailAccount.owner == owner) + q = _email_account_owner_scope(q, owner) row = q.first() if row is None: row = EmailAccount(id=_uuid.uuid4().hex, owner=owner, name="Default", is_default=True, enabled=True) @@ -5456,8 +5574,7 @@ def setup_email_routes(): if data.get("smtp_password"): row.smtp_password = _enc(data["smtp_password"]) clear_q = db.query(EmailAccount).filter(EmailAccount.id != row.id) - if owner: - clear_q = clear_q.filter(EmailAccount.owner == owner) + clear_q = _email_account_owner_scope(clear_q, owner) clear_q.update({EmailAccount.is_default: False}) db.commit() finally: @@ -5552,6 +5669,7 @@ def setup_email_routes(): return {"ok": False, "error": port_err} db = SessionLocal() try: + _lock_email_account_owner_mutation(db, owner) row = EmailAccount( id=_uuid.uuid4().hex, name=name, @@ -5578,9 +5696,7 @@ def setup_email_routes(): # the one-default invariant — but scope it to THIS user's accounts, # otherwise creating a default would clear every other user's # default flag too. - scope_q = db.query(EmailAccount) - if owner: - scope_q = scope_q.filter(EmailAccount.owner == owner) + scope_q = _email_account_owner_scope(db.query(EmailAccount), owner) existing_count = scope_q.count() if row.is_default or existing_count == 0: scope_q.update({EmailAccount.is_default: False}) @@ -5631,28 +5747,39 @@ def setup_email_routes(): @router.delete("/accounts/{account_id}") async def delete_email_account(account_id: str, owner: str = Depends(require_user)): - _assert_owns_account(account_id, owner) + initial_scope = _discover_email_account_mutation_scope(account_id, owner) from core.database import SessionLocal, EmailAccount db = SessionLocal() try: - row = db.get(EmailAccount, account_id) - if not row: - return {"ok": False, "error": "Account not found"} + row = _lock_and_reload_email_account( + db, account_id, owner, initial_scope + ) + row_scope = row.owner or "" was_default = bool(row.is_default) db.delete(row) - db.commit() + # Flush the removal before staging a replacement default. The + # partial unique index is checked statement-by-statement, and the + # ORM is otherwise free to UPDATE the promoted row before DELETE. + db.flush() # If the deleted row was default, promote the next-oldest enabled # row owned by THIS user. Without the owner filter we'd promote # another user's account and the deleter would silently inherit # it as their default. if was_default: - promote_q = db.query(EmailAccount).filter(EmailAccount.enabled == True) # noqa: E712 - if owner: - promote_q = promote_q.filter(EmailAccount.owner == owner) - promote = promote_q.order_by(EmailAccount.created_at.asc()).first() + promote_q = db.query(EmailAccount).filter( + EmailAccount.id != account_id, + EmailAccount.enabled == True, # noqa: E712 + ) + promote_q = _email_account_owner_scope(promote_q, row_scope) + promote = promote_q.order_by( + EmailAccount.created_at.asc(), EmailAccount.id.asc() + ).first() if promote: promote.is_default = True - db.commit() + # Deletion and any replacement promotion are one durable state + # transition, so another worker can never observe or race the old + # split-commit gap. + db.commit() return {"ok": True} finally: db.close() @@ -5865,18 +5992,18 @@ def setup_email_routes(): @router.post("/accounts/{account_id}/set-default") async def set_default_account(account_id: str, owner: str = Depends(require_user)): - _assert_owns_account(account_id, owner) + initial_scope = _discover_email_account_mutation_scope(account_id, owner) from core.database import SessionLocal, EmailAccount db = SessionLocal() try: - row = db.get(EmailAccount, account_id) - if not row: - return {"ok": False, "error": "Account not found"} - # SECURITY: scope the "clear other defaults" sweep to this user's - # accounts so we don't unset another user's default flag. - clear_q = db.query(EmailAccount) - if owner: - clear_q = clear_q.filter(EmailAccount.owner == owner) + row = _lock_and_reload_email_account( + db, account_id, owner, initial_scope + ) + # Scope the sweep to the target row's normalized owner partition; + # this also handles visible legacy NULL/empty-owner accounts. + clear_q = _email_account_owner_scope( + db.query(EmailAccount), row.owner or "" + ) clear_q.update({EmailAccount.is_default: False}) row.is_default = True db.commit() @@ -5895,7 +6022,7 @@ def setup_email_routes(): raise HTTPException(400, "GOOGLE_OAUTH_CLIENT_ID not set — add it to .env") redirect_uri = ( os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") - or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + or f"{request.url.scheme}://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" ) state = make_oauth_state(account_id, owner) params = urllib.parse.urlencode({ @@ -5932,7 +6059,7 @@ def setup_email_routes(): client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "") redirect_uri = ( os.environ.get("GOOGLE_OAUTH_REDIRECT_URI") - or f"http://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" + or f"{request.url.scheme}://{request.headers.get('host', 'localhost:7000')}/api/email/oauth/google/callback" ) import httpx as _httpx try: diff --git a/routes/gallery/gallery_routes.py b/routes/gallery/gallery_routes.py index 457df210d..e6b5e0713 100644 --- a/routes/gallery/gallery_routes.py +++ b/routes/gallery/gallery_routes.py @@ -127,6 +127,25 @@ def _load_grounding_backend(): return cached +def _model_input_to_device(value, device: str, torch): + if not hasattr(value, "to"): + return value + if ( + device == "mps" + and hasattr(torch, "float64") + and getattr(value, "dtype", None) == torch.float64 + ): + return value.to(device=device, dtype=torch.float32) + return value.to(device) + + +def _model_inputs_to_device(inputs, device: str, torch) -> Dict[str, Any]: + return { + key: _model_input_to_device(value, device, torch) + for key, value in inputs.items() + } + + def _ground_text_to_box(image, text: str, *, threshold: float = 0.05): query = (text or "").strip() if not query: @@ -142,10 +161,7 @@ def _ground_text_to_box(image, text: str, *, threshold: float = 0.05): labels.append(f"a photo of {query}") try: inputs = processor(text=[labels], images=image, return_tensors="pt") - model_inputs = { - k: (v.to(device) if hasattr(v, "to") else v) - for k, v in inputs.items() - } + model_inputs = _model_inputs_to_device(inputs, device, torch) with torch.no_grad(): outputs = model(**model_inputs) target_sizes = torch.tensor([[image.height, image.width]]) @@ -1869,10 +1885,7 @@ def setup_gallery_routes() -> APIRouter: try: inputs = processor(image, **kwargs) - model_inputs = { - k: (v.to(device) if hasattr(v, "to") else v) - for k, v in inputs.items() - } + model_inputs = _model_inputs_to_device(inputs, device, torch) with torch.no_grad(): outputs = model(**model_inputs) masks = processor.image_processor.post_process_masks( diff --git a/routes/history/history_routes.py b/routes/history/history_routes.py index f9fa3bd5a..4a6208e33 100644 --- a/routes/history/history_routes.py +++ b/routes/history/history_routes.py @@ -137,44 +137,6 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: entry["metadata"] = meta return entry - def _db_message_metadata(m: DbChatMessage) -> Dict[str, Any]: - meta = {} - if m.meta_data: - try: - meta = json.loads(m.meta_data) or {} - except (json.JSONDecodeError, ValueError): - meta = {} - if m.timestamp and "timestamp" not in meta: - meta["timestamp"] = m.timestamp.isoformat() + "Z" - return meta - - def _hydrate_session_history_from_db(session_id: str, rows: list[DbChatMessage]) -> None: - """Rebuild in-memory context from raw DB rows after a history load. - - The browser history endpoint can return paged/display-trimmed messages, - but the next model call reads ``session.history``. After a restart or a - stale in-memory session, selecting an old chat through the paged endpoint - used to show the transcript while the model only saw fresh context. - """ - if not rows: - return - try: - session = session_manager.get_session(session_id) - except KeyError: - return - session.history = [ - ChatMessage(role=m.role, content=m.content, metadata=_db_message_metadata(m) or None) - for m in rows - ] - session.message_count = len(session.history) - - def _session_needs_db_history_hydration(session_id: str, total: int) -> bool: - try: - session = session_manager.get_session(session_id) - except KeyError: - return False - return len(session.history or []) < int(total or 0) - @router.get("/api/history/{session_id}") async def get_session_history( request: Request, @@ -198,6 +160,8 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: ) page_offset = int(offset) if offset is not None else max(total - page_limit, 0) page_offset = max(0, min(page_offset, total)) + # Keep display pagination page-scoped. ``get_session`` is the + # full model-context hydration seam and must not be entered here. rows = ( db.query(DbChatMessage) .filter(DbChatMessage.session_id == session_id) @@ -206,14 +170,6 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: .limit(page_limit) .all() ) - if _session_needs_db_history_hydration(session_id, total): - full_rows = ( - db.query(DbChatMessage) - .filter(DbChatMessage.session_id == session_id) - .order_by(DbChatMessage.timestamp) - .all() - ) - _hydrate_session_history_from_db(session_id, full_rows) history_dict = [ entry for entry in (_db_history_entry(m) for m in rows) if not (entry.get("metadata") or {}).get("hidden") @@ -258,7 +214,10 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: entry["metadata"] = msg["metadata"] history_dict.append(entry) - # Fallback: load from DB if in-memory is empty + # Fallback: load from DB if in-memory renders empty. Display only — + # get_session above is the hydration seam, so nothing here writes back + # into session.history — rebuilding it from raw rows would overwrite + # parsed multimodal content and the _db_id edit/delete keys it just set. if not history_dict: db = SessionLocal() try: @@ -268,17 +227,10 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: .order_by(DbChatMessage.timestamp) .all() ) - db_history = [] - for m in db_messages: - db_history.append(_db_history_entry(m)) - if db_history: - # Rebuild in-memory history from the full set so hidden - # messages (e.g. compaction summaries) are kept for AI context. - _hydrate_session_history_from_db(session_id, db_messages) # Response excludes hidden messages, matching the in-memory path. history_dict = [ - m for m in db_history - if not (m.get("metadata") or {}).get("hidden") + entry for entry in (_db_history_entry(m) for m in db_messages) + if not (entry.get("metadata") or {}).get("hidden") ] except Exception as e: logger.error(f"DB fallback failed for {session_id}: {e}") @@ -645,8 +597,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter: body = await request.json() keep_count = body.get("keep_count", 0) - # Get the source session - source = session_manager.sessions.get(session_id) + # Get the source session. keep_count indexes into source.history, + # so this must go through get_session — reading the cache directly + # forks an empty transcript out of a metadata-only session after a + # restart (display pagination no longer hydrates it). + try: + source = session_manager.get_session(session_id) + except KeyError: + raise HTTPException(404, "Session not found") if not source: raise HTTPException(404, "Session not found") diff --git a/routes/mcp/__init__.py b/routes/mcp/__init__.py new file mode 100644 index 000000000..bb445ddcc --- /dev/null +++ b/routes/mcp/__init__.py @@ -0,0 +1,5 @@ +"""MCP route domain package (slice 2o, #4082/#4071). + +Contains mcp_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/mcp_routes.py re-exports from here. +""" diff --git a/routes/mcp/mcp_routes.py b/routes/mcp/mcp_routes.py new file mode 100644 index 000000000..a0ade88b6 --- /dev/null +++ b/routes/mcp/mcp_routes.py @@ -0,0 +1,697 @@ +# routes/mcp_routes.py +"""MCP (Model Context Protocol) server management routes.""" +import json +import os +import uuid +import urllib.parse +import html +from pathlib import Path +from fastapi import APIRouter, Form, HTTPException, Request +from fastapi.responses import RedirectResponse, HTMLResponse +import logging +import httpx + +from core.database import McpServer, SessionLocal +from core.middleware import require_admin +from src.constants import DATA_DIR, MCP_OAUTH_DIR +from src.mcp_manager import McpManager + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/mcp", tags=["mcp"]) + + +def _mcp_oauth_base_dir() -> Path: + """Directory that may contain OAuth files managed by Odysseus.""" + return Path(MCP_OAUTH_DIR).resolve(strict=False) + + +def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str: + """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth.""" + raw = str(raw_path or "").strip() + if not raw: + return "" + + base = _mcp_oauth_base_dir() + path = Path(os.path.expanduser(raw)) + if not path.is_absolute(): + path = base / path + resolved = path.resolve(strict=False) + + try: + resolved.relative_to(base) + except ValueError as exc: + raise HTTPException( + 400, + f"Invalid OAuth {field_name}: path must stay under {base}", + ) from exc + return str(resolved) + + +def _sanitize_mcp_oauth_config(oauth_cfg): + """Return an OAuth config copy with file paths confined to mcp_oauth.""" + if not oauth_cfg: + return oauth_cfg + if not isinstance(oauth_cfg, dict): + return {} + sanitized = dict(oauth_cfg) + for field_name in ("keys_file", "token_file"): + if sanitized.get(field_name): + sanitized[field_name] = _resolve_mcp_oauth_path( + sanitized[field_name], + field_name, + ) + return sanitized + + +def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool: + """Check token existence without letting legacy bad paths break listing.""" + if not isinstance(oauth_cfg, dict): + return False + try: + token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file") + except HTTPException: + if strict: + raise + logger.warning("Ignoring MCP OAuth config with unsafe token_file") + return True + return bool(token_file and not os.path.exists(token_file)) + + +def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None: + """Pass sanitized Gmail package paths to MCP servers that honor them.""" + if not oauth_cfg or not isinstance(env, dict): + return + keys_file = oauth_cfg.get("keys_file") + token_file = oauth_cfg.get("token_file") + if keys_file: + env["GMAIL_OAUTH_PATH"] = keys_file + if token_file: + env["GMAIL_CREDENTIALS_PATH"] = token_file + + +def _load_disabled_map(): + """Load per-server disabled tool sets from DB.""" + db = SessionLocal() + try: + disabled_map = {} + for srv in db.query(McpServer).all(): + if srv.disabled_tools: + try: + names = json.loads(srv.disabled_tools) + if names: + disabled_map[srv.id] = set(names) + except (json.JSONDecodeError, TypeError): + pass + return disabled_map + finally: + db.close() + + +def _mcp_oauth_redirect_uri() -> str: + """Shared callback URL for legacy Google and generic MCP OAuth flows.""" + from src.mcp_oauth import REDIRECT_URI + return REDIRECT_URI + + +def setup_mcp_routes(mcp_manager: McpManager): + """Setup MCP routes with the provided manager.""" + + @router.get("/servers") + def list_servers(request: Request): + """List all configured MCP servers with connection status.""" + require_admin(request) + db = SessionLocal() + try: + servers = db.query(McpServer).all() + result = [] + for srv in servers: + status = mcp_manager.get_server_status(srv.id) + oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None + needs_oauth = False + if oauth_cfg: + needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False) + disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else [] + total_tools = status.get("tool_count", 0) + result.append({ + "id": srv.id, + "name": srv.name, + "transport": srv.transport, + "command": srv.command, + "args": json.loads(srv.args) if srv.args else [], + "env": json.loads(srv.env) if srv.env else {}, + "url": srv.url, + "is_enabled": srv.is_enabled, + "status": status.get("status", "disconnected"), + "tool_count": total_tools, + "disabled_tool_count": len(disabled_list), + "enabled_tool_count": max(0, total_tools - len(disabled_list)), + "error": status.get("error"), + "auth_url": status.get("auth_url"), + "has_oauth": oauth_cfg is not None, + "needs_oauth": needs_oauth, + }) + return result + finally: + db.close() + + @router.post("/servers") + async def add_server( + request: Request, + name: str = Form(...), + transport: str = Form("stdio"), + command: str = Form(None), + args: str = Form("[]"), + env: str = Form("{}"), + url: str = Form(None), + oauth_file: str = Form(None), + oauth_config: str = Form(None), + ): + """Add a new MCP server config and attempt connection. Admin-only: + registering a stdio server is equivalent to executing arbitrary + binaries on the host.""" + require_admin(request) + server_id = str(uuid.uuid4())[:8] + + # Validate + if transport == "stdio" and not command: + raise HTTPException(400, "command is required for stdio transport") + if transport == "sse" and not url: + raise HTTPException(400, "url is required for SSE transport") + if transport == "http" and not url: + raise HTTPException(400, "url is required for HTTP transport") + + # Parse JSON fields + try: + parsed_args = json.loads(args) if args else [] + except json.JSONDecodeError: + parsed_args = [] + try: + parsed_env = json.loads(env) if env else {} + except json.JSONDecodeError: + parsed_env = {} + if not isinstance(parsed_env, dict): + parsed_env = {} + + # Parse OAuth config + parsed_oauth_config = None + if oauth_config: + try: + parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config)) + except json.JSONDecodeError: + pass + _apply_mcp_oauth_env(parsed_env, parsed_oauth_config) + + # Write OAuth credentials file if provided (for Google MCP servers) + logger.info(f"MCP add_server: oauth_file={oauth_file!r}") + if oauth_file: + try: + oauth_data = json.loads(oauth_file) + oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir") + oauth_filename = oauth_data.get("filename", "") + client_id = oauth_data.get("client_id", "") + client_secret = oauth_data.get("client_secret", "") + if oauth_dir and oauth_filename and client_id and client_secret: + filepath = _resolve_mcp_oauth_path( + Path(oauth_dir) / str(oauth_filename), + "filename", + ) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + creds = { + "installed": { + "client_id": client_id, + "client_secret": client_secret, + "redirect_uris": ["http://localhost"], + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + } + } + with open(filepath, "w", encoding="utf-8") as f: + json.dump(creds, f, indent=2) + logger.info(f"Wrote OAuth credentials to {filepath}") + parsed_env.pop("GOOGLE_CLIENT_ID", None) + parsed_env.pop("GOOGLE_CLIENT_SECRET", None) + except (json.JSONDecodeError, OSError) as e: + logger.warning(f"Failed to write OAuth file: {e}") + + # Save to DB + db = SessionLocal() + try: + srv = McpServer( + id=server_id, + name=name, + transport=transport, + command=command, + args=json.dumps(parsed_args), + env=json.dumps(parsed_env), + url=url, + is_enabled=True, + oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None, + ) + db.add(srv) + db.commit() + finally: + db.close() + + # Check if OAuth token already exists — skip connection attempt if not + needs_oauth = False + if parsed_oauth_config: + needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config) + + connected = False + if not needs_oauth: + connected = await mcp_manager.connect_server( + server_id=server_id, + name=name, + transport=transport, + command=command, + args=parsed_args, + env=parsed_env, + url=url, + ) + + status = mcp_manager.get_server_status(server_id) + needs_auth = status.get("status") == "needs_auth" + return { + "id": server_id, + "name": name, + "connected": connected, + "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"), + "tool_count": status.get("tool_count", 0), + "error": "OAuth authorization required" if needs_oauth else status.get("error"), + "needs_oauth": needs_oauth, + "needs_auth": needs_auth, + "auth_url": status.get("auth_url"), + } + + @router.post("/servers/{server_id}/reconnect") + async def reconnect_server(server_id: str, request: Request): + """Reconnect to an MCP server.""" + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + + await mcp_manager.disconnect_server(server_id) + + args = json.loads(srv.args) if srv.args else [] + env = json.loads(srv.env) if srv.env else {} + connected = await mcp_manager.connect_server( + server_id=server_id, + name=srv.name, + transport=srv.transport, + command=srv.command, + args=args, + env=env, + url=srv.url, + ) + + status = mcp_manager.get_server_status(server_id) + return { + "connected": connected, + "status": status.get("status", "disconnected"), + "tool_count": status.get("tool_count", 0), + "error": status.get("error"), + "auth_url": status.get("auth_url"), + "needs_auth": status.get("status") == "needs_auth", + } + finally: + db.close() + + @router.patch("/servers/{server_id}") + async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)): + """Enable or disable an MCP server.""" + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + + enabled = str(is_enabled).lower() == "true" + srv.is_enabled = enabled + db.commit() + + if enabled: + args = json.loads(srv.args) if srv.args else [] + env = json.loads(srv.env) if srv.env else {} + await mcp_manager.connect_server( + server_id=server_id, + name=srv.name, + transport=srv.transport, + command=srv.command, + args=args, + env=env, + url=srv.url, + ) + else: + await mcp_manager.disconnect_server(server_id) + + return {"id": server_id, "is_enabled": enabled} + finally: + db.close() + + @router.delete("/servers/{server_id}") + async def delete_server(server_id: str, request: Request): + """Remove an MCP server.""" + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + + await mcp_manager.disconnect_server(server_id) + + db.delete(srv) + db.commit() + return {"status": "deleted"} + finally: + db.close() + + @router.get("/tools") + def list_tools(request: Request): + """List all discovered MCP tools across all connected servers.""" + require_admin(request) + disabled_map = _load_disabled_map() + return mcp_manager.get_all_tools(disabled_map) + + @router.get("/servers/{server_id}/tools") + def list_server_tools(server_id: str, request: Request): + """List all tools for a specific MCP server with enabled/disabled state.""" + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else [] + disabled_set = set(disabled_list) + finally: + db.close() + + all_tools = mcp_manager.get_all_tools() + server_tools = [t for t in all_tools if t["server_id"] == server_id] + for t in server_tools: + t["is_disabled"] = t["name"] in disabled_set + return server_tools + + @router.patch("/servers/{server_id}/tools") + async def update_disabled_tools(server_id: str, request: Request): + """Bulk update disabled tools list for a server. + + Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]} + """ + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + + body = await request.json() + disabled = body.get("disabled", []) + if not isinstance(disabled, list): + raise HTTPException(400, "disabled must be a list of tool names") + + srv.disabled_tools = json.dumps(disabled) if disabled else None + db.commit() + + return {"id": server_id, "disabled_count": len(disabled)} + finally: + db.close() + + # ── OAuth flow for Google MCP servers ────────────────────────── + + @router.get("/oauth/authorize/{server_id}") + def oauth_authorize(server_id: str, request: Request): + """Show OAuth authorization page with Google sign-in link.""" + require_admin(request) + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + raise HTTPException(404, "Server not found") + if not srv.oauth_config: + raise HTTPException(400, "Server has no OAuth config") + + oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config)) + keys_file = oauth_cfg.get("keys_file", "") + if not keys_file or not os.path.exists(keys_file): + raise HTTPException(400, "OAuth keys file not found") + + with open(keys_file, encoding="utf-8") as f: + keys_data = json.load(f) + keys = keys_data.get("installed") or keys_data.get("web") + if not keys: + raise HTTPException(400, "Invalid OAuth keys file format") + + client_id = keys["client_id"] + scopes = oauth_cfg.get("scopes", []) + + # For Desktop App creds, default to localhost — the user will + # paste the resulting URL back if they're on a different device. + redirect_uri = _mcp_oauth_redirect_uri() + + params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": " ".join(scopes), + "access_type": "offline", + "prompt": "consent", + "state": server_id, + } + auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params) + + # Determine if user is accessing from the same machine + host = request.headers.get("host", "") + is_local = host.startswith("localhost") or host.startswith("127.0.0.1") + + if is_local: + # Same machine — just redirect, callback will work directly + return RedirectResponse(auth_url) + else: + # Remote device — show paste-back page + return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri)) + finally: + db.close() + + @router.get("/oauth/callback") + async def oauth_callback(code: str, state: str, request: Request): + """Handle OAuth callback. Generic MCP OAuth flows resolve via the + pending-state registry; Google flows fall through to the legacy path.""" + require_admin(request) + from src.mcp_oauth import resolve_pending + if resolve_pending(state, code): + return HTMLResponse(_oauth_result_page( + "Authorization Successful", + "The MCP server is connecting. You can close this window and return to Odysseus.", + success=True, + )) + # Legacy Google path: state is the server_id + return await _exchange_and_connect(state, code, request) + + @router.post("/oauth/exchange/{server_id}") + async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)): + """Manual code exchange — user pastes the callback URL from their browser.""" + require_admin(request) + try: + parsed = urllib.parse.urlparse(callback_url) + params = urllib.parse.parse_qs(parsed.query) + code = params.get("code", [None])[0] + if not code: + return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400) + except Exception: + return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400) + + # Generic MCP OAuth: if the pasted URL carries a state we are waiting on, + # resolve it directly (the background connect finishes the handshake). + state = params.get("state", [None])[0] + from src.mcp_oauth import resolve_pending + if state and resolve_pending(state, code): + return HTMLResponse(_oauth_result_page( + "Authorization Successful", + "The MCP server is connecting. You can close this window and return to Odysseus.", + success=True, + )) + + return await _exchange_and_connect(server_id, code, request) + + async def _exchange_and_connect(server_id: str, code: str, request: Request): + """Exchange auth code for tokens and connect the MCP server.""" + db = SessionLocal() + try: + srv = db.query(McpServer).filter(McpServer.id == server_id).first() + if not srv: + return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404) + if not srv.oauth_config: + return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400) + + oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config)) + keys_file = oauth_cfg.get("keys_file", "") + token_file = oauth_cfg.get("token_file", "") + if not keys_file or not token_file: + raise HTTPException(400, "OAuth keys/token file not configured") + + with open(keys_file, encoding="utf-8") as f: + keys_data = json.load(f) + keys = keys_data.get("installed") or keys_data.get("web") + client_id = keys["client_id"] + client_secret = keys["client_secret"] + + redirect_uri = _mcp_oauth_redirect_uri() + + async with httpx.AsyncClient() as client: + resp = await client.post( + "https://oauth2.googleapis.com/token", + data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + }, + ) + + if resp.status_code != 200: + err = resp.text + logger.error(f"OAuth token exchange failed: {err}") + return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400) + + tokens = resp.json() + logger.info(f"OAuth tokens received for server {server_id}") + + # Save tokens to the file the MCP package expects + os.makedirs(os.path.dirname(token_file), exist_ok=True) + with open(token_file, "w", encoding="utf-8") as f: + json.dump(tokens, f, indent=2) + logger.info(f"Saved OAuth tokens to {token_file}") + + # Attempt to connect the MCP server now + args = json.loads(srv.args) if srv.args else [] + env = json.loads(srv.env) if srv.env else {} + connected = await mcp_manager.connect_server( + server_id=server_id, + name=srv.name, + transport=srv.transport, + command=srv.command, + args=args, + env=env, + url=srv.url, + ) + + if connected: + status = mcp_manager.get_server_status(server_id) + tool_count = status.get("tool_count", 0) + return HTMLResponse(_oauth_result_page( + "Authorization Successful", + f"{srv.name} connected with {tool_count} tools. You can close this window.", + success=True, + )) + else: + status = mcp_manager.get_server_status(server_id) + return HTMLResponse(_oauth_result_page( + "Authorized but Connection Failed", + f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.", + )) + except HTTPException as e: + logger.warning(f"OAuth callback rejected: {e.detail}") + return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code) + except Exception as e: + logger.exception(f"OAuth callback error: {e}") + return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500) + finally: + db.close() + + return router + + +def _oauth_authorize_page( + auth_url: str, + server_id: str, + host: str, + redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback", +) -> str: + """Page with Google sign-in link and URL paste-back form for remote access.""" + # Escape values interpolated into the page: `host` comes from the request + # Host header and `server_id` from the OAuth state — neither is trusted. + auth_url = html.escape(auth_url, quote=True) + server_id = html.escape(server_id, quote=True) + host = html.escape(host, quote=True) + redirect_uri = html.escape(redirect_uri, quote=True) + return f""" + +Authorize — Odysseus + +
+

Authorize Google Account

+
+ 1. Click the button below to sign in with Google
+ 2. After approving, your browser will show an error page — that's normal
+ 3. Copy the full URL from your browser's address bar
+ 4. Paste it below and click Connect +
+ Sign in with Google +
+
+

Paste the URL from your browser after signing in:

+ +
+
+
""" + + +def _oauth_result_page(title: str, message: str, success: bool = False) -> str: + """Generate a simple HTML page for the OAuth result.""" + safe_title = html.escape(title) + safe_message = html.escape(message) + color = "#00661a" if success else "#e06c75" + icon = "✓" if success else "✗" + return f""" + +{safe_title} + +
+
{icon}
+

{safe_title}

+

{safe_message}

+
""" diff --git a/routes/mcp_routes.py b/routes/mcp_routes.py index a0ade88b6..8304dc1d4 100644 --- a/routes/mcp_routes.py +++ b/routes/mcp_routes.py @@ -1,697 +1,18 @@ -# routes/mcp_routes.py -"""MCP (Model Context Protocol) server management routes.""" -import json -import os -import uuid -import urllib.parse -import html -from pathlib import Path -from fastapi import APIRouter, Form, HTTPException, Request -from fastapi.responses import RedirectResponse, HTMLResponse -import logging -import httpx +"""Backward-compat shim — canonical location is routes/mcp/mcp_routes.py. -from core.database import McpServer, SessionLocal -from core.middleware import require_admin -from src.constants import DATA_DIR, MCP_OAUTH_DIR -from src.mcp_manager import McpManager +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.mcp_routes``, ``from routes.mcp_routes import X``, +``importlib.import_module("routes.mcp_routes")``, the +``sys.modules.pop("routes.mcp_routes")`` + re-import pattern in +test_security_regressions.py, and the ``monkeypatch.setattr(mcp_routes, +"MCP_OAUTH_DIR", ...)`` pattern all operate on the *same* object. This also +makes ``mcp_routes.__file__`` resolve to the canonical file (which the +source-introspection at line 839 reads). Keeps existing import paths working +after slice 2o (#4082/#4071). +""" -logger = logging.getLogger(__name__) +import sys as _sys -router = APIRouter(prefix="/api/mcp", tags=["mcp"]) +from routes.mcp import mcp_routes as _canonical # noqa: F401 - -def _mcp_oauth_base_dir() -> Path: - """Directory that may contain OAuth files managed by Odysseus.""" - return Path(MCP_OAUTH_DIR).resolve(strict=False) - - -def _resolve_mcp_oauth_path(raw_path, field_name: str) -> str: - """Resolve an MCP OAuth path and keep it under DATA_DIR/mcp_oauth.""" - raw = str(raw_path or "").strip() - if not raw: - return "" - - base = _mcp_oauth_base_dir() - path = Path(os.path.expanduser(raw)) - if not path.is_absolute(): - path = base / path - resolved = path.resolve(strict=False) - - try: - resolved.relative_to(base) - except ValueError as exc: - raise HTTPException( - 400, - f"Invalid OAuth {field_name}: path must stay under {base}", - ) from exc - return str(resolved) - - -def _sanitize_mcp_oauth_config(oauth_cfg): - """Return an OAuth config copy with file paths confined to mcp_oauth.""" - if not oauth_cfg: - return oauth_cfg - if not isinstance(oauth_cfg, dict): - return {} - sanitized = dict(oauth_cfg) - for field_name in ("keys_file", "token_file"): - if sanitized.get(field_name): - sanitized[field_name] = _resolve_mcp_oauth_path( - sanitized[field_name], - field_name, - ) - return sanitized - - -def _mcp_oauth_token_missing(oauth_cfg, *, strict: bool = True) -> bool: - """Check token existence without letting legacy bad paths break listing.""" - if not isinstance(oauth_cfg, dict): - return False - try: - token_file = _resolve_mcp_oauth_path(oauth_cfg.get("token_file", ""), "token_file") - except HTTPException: - if strict: - raise - logger.warning("Ignoring MCP OAuth config with unsafe token_file") - return True - return bool(token_file and not os.path.exists(token_file)) - - -def _apply_mcp_oauth_env(env: dict, oauth_cfg) -> None: - """Pass sanitized Gmail package paths to MCP servers that honor them.""" - if not oauth_cfg or not isinstance(env, dict): - return - keys_file = oauth_cfg.get("keys_file") - token_file = oauth_cfg.get("token_file") - if keys_file: - env["GMAIL_OAUTH_PATH"] = keys_file - if token_file: - env["GMAIL_CREDENTIALS_PATH"] = token_file - - -def _load_disabled_map(): - """Load per-server disabled tool sets from DB.""" - db = SessionLocal() - try: - disabled_map = {} - for srv in db.query(McpServer).all(): - if srv.disabled_tools: - try: - names = json.loads(srv.disabled_tools) - if names: - disabled_map[srv.id] = set(names) - except (json.JSONDecodeError, TypeError): - pass - return disabled_map - finally: - db.close() - - -def _mcp_oauth_redirect_uri() -> str: - """Shared callback URL for legacy Google and generic MCP OAuth flows.""" - from src.mcp_oauth import REDIRECT_URI - return REDIRECT_URI - - -def setup_mcp_routes(mcp_manager: McpManager): - """Setup MCP routes with the provided manager.""" - - @router.get("/servers") - def list_servers(request: Request): - """List all configured MCP servers with connection status.""" - require_admin(request) - db = SessionLocal() - try: - servers = db.query(McpServer).all() - result = [] - for srv in servers: - status = mcp_manager.get_server_status(srv.id) - oauth_cfg = json.loads(srv.oauth_config) if srv.oauth_config else None - needs_oauth = False - if oauth_cfg: - needs_oauth = _mcp_oauth_token_missing(oauth_cfg, strict=False) - disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else [] - total_tools = status.get("tool_count", 0) - result.append({ - "id": srv.id, - "name": srv.name, - "transport": srv.transport, - "command": srv.command, - "args": json.loads(srv.args) if srv.args else [], - "env": json.loads(srv.env) if srv.env else {}, - "url": srv.url, - "is_enabled": srv.is_enabled, - "status": status.get("status", "disconnected"), - "tool_count": total_tools, - "disabled_tool_count": len(disabled_list), - "enabled_tool_count": max(0, total_tools - len(disabled_list)), - "error": status.get("error"), - "auth_url": status.get("auth_url"), - "has_oauth": oauth_cfg is not None, - "needs_oauth": needs_oauth, - }) - return result - finally: - db.close() - - @router.post("/servers") - async def add_server( - request: Request, - name: str = Form(...), - transport: str = Form("stdio"), - command: str = Form(None), - args: str = Form("[]"), - env: str = Form("{}"), - url: str = Form(None), - oauth_file: str = Form(None), - oauth_config: str = Form(None), - ): - """Add a new MCP server config and attempt connection. Admin-only: - registering a stdio server is equivalent to executing arbitrary - binaries on the host.""" - require_admin(request) - server_id = str(uuid.uuid4())[:8] - - # Validate - if transport == "stdio" and not command: - raise HTTPException(400, "command is required for stdio transport") - if transport == "sse" and not url: - raise HTTPException(400, "url is required for SSE transport") - if transport == "http" and not url: - raise HTTPException(400, "url is required for HTTP transport") - - # Parse JSON fields - try: - parsed_args = json.loads(args) if args else [] - except json.JSONDecodeError: - parsed_args = [] - try: - parsed_env = json.loads(env) if env else {} - except json.JSONDecodeError: - parsed_env = {} - if not isinstance(parsed_env, dict): - parsed_env = {} - - # Parse OAuth config - parsed_oauth_config = None - if oauth_config: - try: - parsed_oauth_config = _sanitize_mcp_oauth_config(json.loads(oauth_config)) - except json.JSONDecodeError: - pass - _apply_mcp_oauth_env(parsed_env, parsed_oauth_config) - - # Write OAuth credentials file if provided (for Google MCP servers) - logger.info(f"MCP add_server: oauth_file={oauth_file!r}") - if oauth_file: - try: - oauth_data = json.loads(oauth_file) - oauth_dir = _resolve_mcp_oauth_path(oauth_data.get("dir", ""), "dir") - oauth_filename = oauth_data.get("filename", "") - client_id = oauth_data.get("client_id", "") - client_secret = oauth_data.get("client_secret", "") - if oauth_dir and oauth_filename and client_id and client_secret: - filepath = _resolve_mcp_oauth_path( - Path(oauth_dir) / str(oauth_filename), - "filename", - ) - os.makedirs(os.path.dirname(filepath), exist_ok=True) - creds = { - "installed": { - "client_id": client_id, - "client_secret": client_secret, - "redirect_uris": ["http://localhost"], - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://accounts.google.com/o/oauth2/token", - } - } - with open(filepath, "w", encoding="utf-8") as f: - json.dump(creds, f, indent=2) - logger.info(f"Wrote OAuth credentials to {filepath}") - parsed_env.pop("GOOGLE_CLIENT_ID", None) - parsed_env.pop("GOOGLE_CLIENT_SECRET", None) - except (json.JSONDecodeError, OSError) as e: - logger.warning(f"Failed to write OAuth file: {e}") - - # Save to DB - db = SessionLocal() - try: - srv = McpServer( - id=server_id, - name=name, - transport=transport, - command=command, - args=json.dumps(parsed_args), - env=json.dumps(parsed_env), - url=url, - is_enabled=True, - oauth_config=json.dumps(parsed_oauth_config) if parsed_oauth_config else None, - ) - db.add(srv) - db.commit() - finally: - db.close() - - # Check if OAuth token already exists — skip connection attempt if not - needs_oauth = False - if parsed_oauth_config: - needs_oauth = _mcp_oauth_token_missing(parsed_oauth_config) - - connected = False - if not needs_oauth: - connected = await mcp_manager.connect_server( - server_id=server_id, - name=name, - transport=transport, - command=command, - args=parsed_args, - env=parsed_env, - url=url, - ) - - status = mcp_manager.get_server_status(server_id) - needs_auth = status.get("status") == "needs_auth" - return { - "id": server_id, - "name": name, - "connected": connected, - "status": "needs_oauth" if needs_oauth else status.get("status", "disconnected"), - "tool_count": status.get("tool_count", 0), - "error": "OAuth authorization required" if needs_oauth else status.get("error"), - "needs_oauth": needs_oauth, - "needs_auth": needs_auth, - "auth_url": status.get("auth_url"), - } - - @router.post("/servers/{server_id}/reconnect") - async def reconnect_server(server_id: str, request: Request): - """Reconnect to an MCP server.""" - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - - await mcp_manager.disconnect_server(server_id) - - args = json.loads(srv.args) if srv.args else [] - env = json.loads(srv.env) if srv.env else {} - connected = await mcp_manager.connect_server( - server_id=server_id, - name=srv.name, - transport=srv.transport, - command=srv.command, - args=args, - env=env, - url=srv.url, - ) - - status = mcp_manager.get_server_status(server_id) - return { - "connected": connected, - "status": status.get("status", "disconnected"), - "tool_count": status.get("tool_count", 0), - "error": status.get("error"), - "auth_url": status.get("auth_url"), - "needs_auth": status.get("status") == "needs_auth", - } - finally: - db.close() - - @router.patch("/servers/{server_id}") - async def toggle_server(server_id: str, request: Request, is_enabled: str = Form(...)): - """Enable or disable an MCP server.""" - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - - enabled = str(is_enabled).lower() == "true" - srv.is_enabled = enabled - db.commit() - - if enabled: - args = json.loads(srv.args) if srv.args else [] - env = json.loads(srv.env) if srv.env else {} - await mcp_manager.connect_server( - server_id=server_id, - name=srv.name, - transport=srv.transport, - command=srv.command, - args=args, - env=env, - url=srv.url, - ) - else: - await mcp_manager.disconnect_server(server_id) - - return {"id": server_id, "is_enabled": enabled} - finally: - db.close() - - @router.delete("/servers/{server_id}") - async def delete_server(server_id: str, request: Request): - """Remove an MCP server.""" - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - - await mcp_manager.disconnect_server(server_id) - - db.delete(srv) - db.commit() - return {"status": "deleted"} - finally: - db.close() - - @router.get("/tools") - def list_tools(request: Request): - """List all discovered MCP tools across all connected servers.""" - require_admin(request) - disabled_map = _load_disabled_map() - return mcp_manager.get_all_tools(disabled_map) - - @router.get("/servers/{server_id}/tools") - def list_server_tools(server_id: str, request: Request): - """List all tools for a specific MCP server with enabled/disabled state.""" - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - disabled_list = json.loads(srv.disabled_tools) if srv.disabled_tools else [] - disabled_set = set(disabled_list) - finally: - db.close() - - all_tools = mcp_manager.get_all_tools() - server_tools = [t for t in all_tools if t["server_id"] == server_id] - for t in server_tools: - t["is_disabled"] = t["name"] in disabled_set - return server_tools - - @router.patch("/servers/{server_id}/tools") - async def update_disabled_tools(server_id: str, request: Request): - """Bulk update disabled tools list for a server. - - Expects JSON body: {"disabled": ["tool_name_1", "tool_name_2"]} - """ - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - - body = await request.json() - disabled = body.get("disabled", []) - if not isinstance(disabled, list): - raise HTTPException(400, "disabled must be a list of tool names") - - srv.disabled_tools = json.dumps(disabled) if disabled else None - db.commit() - - return {"id": server_id, "disabled_count": len(disabled)} - finally: - db.close() - - # ── OAuth flow for Google MCP servers ────────────────────────── - - @router.get("/oauth/authorize/{server_id}") - def oauth_authorize(server_id: str, request: Request): - """Show OAuth authorization page with Google sign-in link.""" - require_admin(request) - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - raise HTTPException(404, "Server not found") - if not srv.oauth_config: - raise HTTPException(400, "Server has no OAuth config") - - oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config)) - keys_file = oauth_cfg.get("keys_file", "") - if not keys_file or not os.path.exists(keys_file): - raise HTTPException(400, "OAuth keys file not found") - - with open(keys_file, encoding="utf-8") as f: - keys_data = json.load(f) - keys = keys_data.get("installed") or keys_data.get("web") - if not keys: - raise HTTPException(400, "Invalid OAuth keys file format") - - client_id = keys["client_id"] - scopes = oauth_cfg.get("scopes", []) - - # For Desktop App creds, default to localhost — the user will - # paste the resulting URL back if they're on a different device. - redirect_uri = _mcp_oauth_redirect_uri() - - params = { - "client_id": client_id, - "redirect_uri": redirect_uri, - "response_type": "code", - "scope": " ".join(scopes), - "access_type": "offline", - "prompt": "consent", - "state": server_id, - } - auth_url = "https://accounts.google.com/o/oauth2/v2/auth?" + urllib.parse.urlencode(params) - - # Determine if user is accessing from the same machine - host = request.headers.get("host", "") - is_local = host.startswith("localhost") or host.startswith("127.0.0.1") - - if is_local: - # Same machine — just redirect, callback will work directly - return RedirectResponse(auth_url) - else: - # Remote device — show paste-back page - return HTMLResponse(_oauth_authorize_page(auth_url, server_id, host, redirect_uri)) - finally: - db.close() - - @router.get("/oauth/callback") - async def oauth_callback(code: str, state: str, request: Request): - """Handle OAuth callback. Generic MCP OAuth flows resolve via the - pending-state registry; Google flows fall through to the legacy path.""" - require_admin(request) - from src.mcp_oauth import resolve_pending - if resolve_pending(state, code): - return HTMLResponse(_oauth_result_page( - "Authorization Successful", - "The MCP server is connecting. You can close this window and return to Odysseus.", - success=True, - )) - # Legacy Google path: state is the server_id - return await _exchange_and_connect(state, code, request) - - @router.post("/oauth/exchange/{server_id}") - async def oauth_exchange(server_id: str, request: Request, callback_url: str = Form(...)): - """Manual code exchange — user pastes the callback URL from their browser.""" - require_admin(request) - try: - parsed = urllib.parse.urlparse(callback_url) - params = urllib.parse.parse_qs(parsed.query) - code = params.get("code", [None])[0] - if not code: - return HTMLResponse(_oauth_result_page("Error", "No authorization code found in the URL. Make sure you copied the full URL from your browser."), status_code=400) - except Exception: - return HTMLResponse(_oauth_result_page("Error", "Invalid URL format."), status_code=400) - - # Generic MCP OAuth: if the pasted URL carries a state we are waiting on, - # resolve it directly (the background connect finishes the handshake). - state = params.get("state", [None])[0] - from src.mcp_oauth import resolve_pending - if state and resolve_pending(state, code): - return HTMLResponse(_oauth_result_page( - "Authorization Successful", - "The MCP server is connecting. You can close this window and return to Odysseus.", - success=True, - )) - - return await _exchange_and_connect(server_id, code, request) - - async def _exchange_and_connect(server_id: str, code: str, request: Request): - """Exchange auth code for tokens and connect the MCP server.""" - db = SessionLocal() - try: - srv = db.query(McpServer).filter(McpServer.id == server_id).first() - if not srv: - return HTMLResponse(_oauth_result_page("Error", "Server not found."), status_code=404) - if not srv.oauth_config: - return HTMLResponse(_oauth_result_page("Error", "No OAuth config."), status_code=400) - - oauth_cfg = _sanitize_mcp_oauth_config(json.loads(srv.oauth_config)) - keys_file = oauth_cfg.get("keys_file", "") - token_file = oauth_cfg.get("token_file", "") - if not keys_file or not token_file: - raise HTTPException(400, "OAuth keys/token file not configured") - - with open(keys_file, encoding="utf-8") as f: - keys_data = json.load(f) - keys = keys_data.get("installed") or keys_data.get("web") - client_id = keys["client_id"] - client_secret = keys["client_secret"] - - redirect_uri = _mcp_oauth_redirect_uri() - - async with httpx.AsyncClient() as client: - resp = await client.post( - "https://oauth2.googleapis.com/token", - data={ - "code": code, - "client_id": client_id, - "client_secret": client_secret, - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - }, - ) - - if resp.status_code != 200: - err = resp.text - logger.error(f"OAuth token exchange failed: {err}") - return HTMLResponse(_oauth_result_page("Authorization Failed", f"Google returned an error: {err}"), status_code=400) - - tokens = resp.json() - logger.info(f"OAuth tokens received for server {server_id}") - - # Save tokens to the file the MCP package expects - os.makedirs(os.path.dirname(token_file), exist_ok=True) - with open(token_file, "w", encoding="utf-8") as f: - json.dump(tokens, f, indent=2) - logger.info(f"Saved OAuth tokens to {token_file}") - - # Attempt to connect the MCP server now - args = json.loads(srv.args) if srv.args else [] - env = json.loads(srv.env) if srv.env else {} - connected = await mcp_manager.connect_server( - server_id=server_id, - name=srv.name, - transport=srv.transport, - command=srv.command, - args=args, - env=env, - url=srv.url, - ) - - if connected: - status = mcp_manager.get_server_status(server_id) - tool_count = status.get("tool_count", 0) - return HTMLResponse(_oauth_result_page( - "Authorization Successful", - f"{srv.name} connected with {tool_count} tools. You can close this window.", - success=True, - )) - else: - status = mcp_manager.get_server_status(server_id) - return HTMLResponse(_oauth_result_page( - "Authorized but Connection Failed", - f"Tokens saved, but the server failed to connect: {status.get('error', 'unknown error')}. Try reconnecting from Settings.", - )) - except HTTPException as e: - logger.warning(f"OAuth callback rejected: {e.detail}") - return HTMLResponse(_oauth_result_page("Error", str(e.detail)), status_code=e.status_code) - except Exception as e: - logger.exception(f"OAuth callback error: {e}") - return HTMLResponse(_oauth_result_page("Error", str(e)), status_code=500) - finally: - db.close() - - return router - - -def _oauth_authorize_page( - auth_url: str, - server_id: str, - host: str, - redirect_uri: str = "http://localhost:7000/api/mcp/oauth/callback", -) -> str: - """Page with Google sign-in link and URL paste-back form for remote access.""" - # Escape values interpolated into the page: `host` comes from the request - # Host header and `server_id` from the OAuth state — neither is trusted. - auth_url = html.escape(auth_url, quote=True) - server_id = html.escape(server_id, quote=True) - host = html.escape(host, quote=True) - redirect_uri = html.escape(redirect_uri, quote=True) - return f""" - -Authorize — Odysseus - -
-

Authorize Google Account

-
- 1. Click the button below to sign in with Google
- 2. After approving, your browser will show an error page — that's normal
- 3. Copy the full URL from your browser's address bar
- 4. Paste it below and click Connect -
- Sign in with Google -
-
-

Paste the URL from your browser after signing in:

- -
-
-
""" - - -def _oauth_result_page(title: str, message: str, success: bool = False) -> str: - """Generate a simple HTML page for the OAuth result.""" - safe_title = html.escape(title) - safe_message = html.escape(message) - color = "#00661a" if success else "#e06c75" - icon = "✓" if success else "✗" - return f""" - -{safe_title} - -
-
{icon}
-

{safe_title}

-

{safe_message}

-
""" +_sys.modules[__name__] = _canonical diff --git a/routes/memory/memory_routes.py b/routes/memory/memory_routes.py index d290046ec..c4232bec4 100644 --- a/routes/memory/memory_routes.py +++ b/routes/memory/memory_routes.py @@ -21,7 +21,7 @@ def _strip_list_prefix(text: str) -> str: return text return _LIST_PREFIX_RE.sub("", text, count=1).strip() -from services.memory import MemoryManager +from services.memory import MemoryManager, MemoryStoreUnreadable from core.session_manager import SessionManager from src.request_models import MemoryAddRequest from core.database import SessionLocal @@ -35,6 +35,22 @@ from src.upload_limits import read_upload_limited, MEMORY_IMPORT_MAX_BYTES logger = logging.getLogger(__name__) +def _load_for_update(memory_manager) -> List[Dict[str, Any]]: + """Load the whole store for a read-modify-write cycle. + + A transient read failure must not look like an empty store: the caller + would append to ``[]`` and save that back, atomically destroying every + existing memory (issue #5673). Surface it as a 503 and change nothing. + """ + try: + return memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to rewrite the memory store: %s", e) + raise HTTPException( + 503, "Memory store is temporarily unreadable — no changes were made." + ) + + def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionManager, memory_vector=None): """Set up memory-related routes.""" router = APIRouter(prefix="/api/memory", tags=["memory"]) @@ -116,7 +132,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM new_entry = memory_manager.add_entry(text, memory_data.source, memory_data.category, owner=user) if memory_data.session_id: new_entry["session_id"] = memory_data.session_id - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) all_mem.append(new_entry) memory_manager.save(all_mem) # Sync vector index @@ -487,7 +503,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def pin_memory(request: Request, memory_id: str, pinned: bool = Form(True)): """Pin or unpin a memory. Pinned memories are always included in context.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) for i, memory in enumerate(all_mem): if memory["id"] == memory_id: _verify_memory_owner(memory, user) @@ -512,7 +528,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def update_memory(request: Request, memory_id: str, text: str = Form(...), category: str = Form(None)): """Update an existing memory item with new text and optional category.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) for i, memory in enumerate(all_mem): if memory["id"] == memory_id: _verify_memory_owner(memory, user) @@ -534,7 +550,7 @@ def setup_memory_routes(memory_manager: MemoryManager, session_manager: SessionM def delete_memory(request: Request, memory_id: str): """Delete a memory item by its ID.""" user = _owner(request) - all_mem = memory_manager.load_all() + all_mem = _load_for_update(memory_manager) # Find and verify ownership before deleting target = next((m for m in all_mem if m["id"] == memory_id), None) diff --git a/routes/model_routes.py b/routes/model_routes.py index 600150a66..de8f884cb 100644 --- a/routes/model_routes.py +++ b/routes/model_routes.py @@ -46,10 +46,11 @@ _ENDPOINT_SETTING_FIELDS = { } _ENDPOINT_FALLBACK_FIELDS = { - "default_model_fallbacks": "Default Model Fallbacks", "utility_model_fallbacks": "Utility Model Fallbacks", "vision_model_fallbacks": "Vision Model Fallbacks", } +# `default_model_fallbacks` is intentionally absent. The legacy data remains +# stored as-is even when an endpoint is removed, but no longer affects routing. def _speech_settings_using_endpoint(settings: dict, ep_id: str) -> list: @@ -2437,7 +2438,6 @@ def setup_model_routes(model_discovery): _user_prefs = _load_for_user(_user) or {} ep_id = (_user_prefs.get("default_endpoint_id") or "").strip() model = (_user_prefs.get("default_model") or "").strip() - _fallbacks = _user_prefs.get("default_model_fallbacks") or [] # If user has no personal default, fall back to global default # But only based on the "share_defaults_with_users" flag # (only if share_defaults_with_users is enabled) @@ -2446,12 +2446,9 @@ def setup_model_routes(model_discovery): ep_id = settings.get("default_endpoint_id", "") if not model: model = settings.get("default_model", "") - if not _fallbacks: - _fallbacks = settings.get("default_model_fallbacks") or [] else: ep_id = settings.get("default_endpoint_id", "") model = settings.get("default_model", "") - _fallbacks = settings.get("default_model_fallbacks") or [] db = SessionLocal() try: ep = None @@ -2466,33 +2463,6 @@ def setup_model_routes(model_discovery): if _user and not _is_admin: ep_q = owner_filter(ep_q, ModelEndpoint, _user) ep = ep_q.first() - # Configured fallback chain — when the chosen default endpoint is - # gone/disabled, honor the user's configured `default_model_fallbacks` - # in order BEFORE arbitrarily grabbing the first enabled endpoint. - # (Previously this jumped straight to "first enabled", which is why - # deleting/changing the main endpoint silently reassigned the default - # chat to some unrelated endpoint instead of the fallback.) - if not ep: - for entry in _fallbacks: - if not isinstance(entry, dict): - continue - fid = (entry.get("endpoint_id") or "").strip() - if not fid: - continue - cand_q = db.query(ModelEndpoint).filter( - ModelEndpoint.id == fid, ModelEndpoint.is_enabled == True - ) - if _user and not _is_admin: - cand_q = owner_filter(cand_q, ModelEndpoint, _user) - cand = cand_q.first() - if cand: - ep = cand - # Use the fallback entry's model. Reset even when empty - # so we don't carry the prior endpoint's stale model onto - # this fallback — the cached-models lookup below then - # fills it from the fallback endpoint. - model = (entry.get("model") or "").strip() - break # Last resort: first enabled endpoint owned by THIS user. Do not # include null-owner/shared endpoints here: a brand-new user with # no explicit default should not auto-open a pending chat using an diff --git a/routes/prefs_routes.py b/routes/prefs_routes.py index f2a778c2d..91a17419e 100644 --- a/routes/prefs_routes.py +++ b/routes/prefs_routes.py @@ -1,8 +1,8 @@ """User preferences API — per-user key/value store backed by a JSON file.""" import json -import os from typing import Optional from fastapi import APIRouter, Request +from core.atomic_io import atomic_write_json from src.auth_helpers import get_current_user from src.constants import USER_PREFS_FILE @@ -20,13 +20,7 @@ def _load(): def _save(prefs): - os.makedirs(os.path.dirname(PREFS_FILE) or ".", exist_ok=True) - tmp = f"{PREFS_FILE}.tmp.{os.getpid()}" - with open(tmp, "w", encoding="utf-8") as f: - json.dump(prefs, f, indent=2) - f.flush() - os.fsync(f.fileno()) - os.replace(tmp, PREFS_FILE) + atomic_write_json(PREFS_FILE, prefs, indent=2) def _load_for_user(user: Optional[str] = None) -> dict: diff --git a/routes/search/__init__.py b/routes/search/__init__.py new file mode 100644 index 000000000..ea051bbe0 --- /dev/null +++ b/routes/search/__init__.py @@ -0,0 +1,5 @@ +"""Search route domain package (slice 2j, #4082/#4071). + +Contains search_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/search_routes.py re-exports from here. +""" diff --git a/routes/search/search_routes.py b/routes/search/search_routes.py new file mode 100644 index 000000000..1effb7b8f --- /dev/null +++ b/routes/search/search_routes.py @@ -0,0 +1,111 @@ +"""Search routes — /api/search/config GET, /api/search POST.""" + +import logging +from typing import Dict, Any + +from fastapi import APIRouter, Request + +import time + +from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO +from services.search.core import _call_provider +from services.search.providers import _get_provider_key, _get_search_instance + +logger = logging.getLogger(__name__) + + +async def _request_values(request: Request) -> Dict[str, Any]: + """Accept JSON, form data, or query params for search endpoints. + + The browser UI posts FormData, while the agent's generic app_api tool + posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler + runs, which made the model think SearXNG was broken. + """ + values: Dict[str, Any] = dict(request.query_params) + content_type = (request.headers.get("content-type") or "").lower() + try: + if "application/json" in content_type: + body = await request.json() + if isinstance(body, dict): + values.update(body) + else: + form = await request.form() + values.update(dict(form)) + except Exception: + pass + return values + + +def setup_search_routes(config) -> APIRouter: + router = APIRouter(tags=["search"]) + + @router.get("/api/search/config") + async def get_search_settings() -> Dict[str, Any]: + return get_search_config() + + @router.post("/api/search") + async def do_web_search(request: Request) -> Dict[str, Any]: + """Standalone web search — returns context string + source list. + + Used by Compare mode to pre-search once and share results across panes. + """ + values = await _request_values(request) + query = str(values.get("query") or values.get("q") or "").strip() + if not query: + return {"context": "", "sources": [], "error": "query is required"} + time_filter = values.get("time_filter") or values.get("freshness") + if time_filter is not None: + time_filter = str(time_filter).strip() or None + try: + context, sources = comprehensive_web_search( + query, return_sources=True, time_filter=time_filter, + ) + return {"context": context, "sources": sources} + except Exception as e: + logger.error(f"Standalone web search failed: {e}") + return {"context": "", "sources": [], "error": str(e)} + + @router.get("/api/search/providers") + async def list_search_providers(): + """Return available search providers with config status.""" + providers = [] + for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items(): + if pid == "disabled": + continue + available = True + if needs_key and not _get_provider_key(pid): + available = False + if needs_url and pid == "searxng" and not _get_search_instance(): + available = False + providers.append({ + "id": pid, + "label": label, + "available": available, + }) + return providers + + @router.post("/api/search/query") + async def search_with_provider(request: Request) -> Dict[str, Any]: + """Search using a specific provider. Used by compare search mode.""" + values = await _request_values(request) + query = str(values.get("query") or values.get("q") or "").strip() + provider = str(values.get("provider") or "").strip() + try: + count = int(values.get("count") or values.get("limit") or 10) + except Exception: + count = 10 + if not query: + return {"results": [], "provider": provider, "error": "query is required"} + if provider not in PROVIDER_INFO or provider == "disabled": + return {"results": [], "provider": provider, "error": "Unknown provider"} + t0 = time.time() + try: + results = _call_provider(provider, query, min(count, 20)) + elapsed = round(time.time() - t0, 2) + return {"results": results, "provider": provider, "time": elapsed} + except Exception as e: + elapsed = round(time.time() - t0, 2) + logger.error(f"Search provider {provider} failed: {e}") + return {"results": [], "provider": provider, "time": elapsed, "error": str(e)} + + return router diff --git a/routes/search_routes.py b/routes/search_routes.py index 1effb7b8f..03b94438b 100644 --- a/routes/search_routes.py +++ b/routes/search_routes.py @@ -1,111 +1,13 @@ -"""Search routes — /api/search/config GET, /api/search POST.""" +"""Backward-compat shim — canonical location is routes/search/search_routes.py. -import logging -from typing import Dict, Any +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.search_routes`` and ``from routes.search_routes import X`` +keep resolving to the canonical module. Keeps existing import paths working +after slice 2j (#4082/#4071). +""" -from fastapi import APIRouter, Request +import sys as _sys -import time +from routes.search import search_routes as _canonical # noqa: F401 -from services.search import get_search_config, comprehensive_web_search, PROVIDER_INFO -from services.search.core import _call_provider -from services.search.providers import _get_provider_key, _get_search_instance - -logger = logging.getLogger(__name__) - - -async def _request_values(request: Request) -> Dict[str, Any]: - """Accept JSON, form data, or query params for search endpoints. - - The browser UI posts FormData, while the agent's generic app_api tool - posts JSON. FastAPI Form(...) rejects JSON with a 422 before our handler - runs, which made the model think SearXNG was broken. - """ - values: Dict[str, Any] = dict(request.query_params) - content_type = (request.headers.get("content-type") or "").lower() - try: - if "application/json" in content_type: - body = await request.json() - if isinstance(body, dict): - values.update(body) - else: - form = await request.form() - values.update(dict(form)) - except Exception: - pass - return values - - -def setup_search_routes(config) -> APIRouter: - router = APIRouter(tags=["search"]) - - @router.get("/api/search/config") - async def get_search_settings() -> Dict[str, Any]: - return get_search_config() - - @router.post("/api/search") - async def do_web_search(request: Request) -> Dict[str, Any]: - """Standalone web search — returns context string + source list. - - Used by Compare mode to pre-search once and share results across panes. - """ - values = await _request_values(request) - query = str(values.get("query") or values.get("q") or "").strip() - if not query: - return {"context": "", "sources": [], "error": "query is required"} - time_filter = values.get("time_filter") or values.get("freshness") - if time_filter is not None: - time_filter = str(time_filter).strip() or None - try: - context, sources = comprehensive_web_search( - query, return_sources=True, time_filter=time_filter, - ) - return {"context": context, "sources": sources} - except Exception as e: - logger.error(f"Standalone web search failed: {e}") - return {"context": "", "sources": [], "error": str(e)} - - @router.get("/api/search/providers") - async def list_search_providers(): - """Return available search providers with config status.""" - providers = [] - for pid, (label, needs_key, needs_url) in PROVIDER_INFO.items(): - if pid == "disabled": - continue - available = True - if needs_key and not _get_provider_key(pid): - available = False - if needs_url and pid == "searxng" and not _get_search_instance(): - available = False - providers.append({ - "id": pid, - "label": label, - "available": available, - }) - return providers - - @router.post("/api/search/query") - async def search_with_provider(request: Request) -> Dict[str, Any]: - """Search using a specific provider. Used by compare search mode.""" - values = await _request_values(request) - query = str(values.get("query") or values.get("q") or "").strip() - provider = str(values.get("provider") or "").strip() - try: - count = int(values.get("count") or values.get("limit") or 10) - except Exception: - count = 10 - if not query: - return {"results": [], "provider": provider, "error": "query is required"} - if provider not in PROVIDER_INFO or provider == "disabled": - return {"results": [], "provider": provider, "error": "Unknown provider"} - t0 = time.time() - try: - results = _call_provider(provider, query, min(count, 20)) - elapsed = round(time.time() - t0, 2) - return {"results": results, "provider": provider, "time": elapsed} - except Exception as e: - elapsed = round(time.time() - t0, 2) - logger.error(f"Search provider {provider} failed: {e}") - return {"results": [], "provider": provider, "time": elapsed, "error": str(e)} - - return router +_sys.modules[__name__] = _canonical diff --git a/routes/session_routes.py b/routes/session_routes.py index dc29a64e4..b1d79f7fe 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -801,15 +801,6 @@ def setup_session_routes( finally: db.close() - @router.get("/history/{sid}") - def get_history(request: Request, sid: str): - _verify_session_owner(request, sid) - try: - session = session_manager.get_session(sid) - except KeyError: - raise HTTPException(404, f"Session {sid} not found") - return {"history": [msg.to_dict() for msg in session.history]} - @router.get("/session/{sid}/export") def export_session(request: Request, sid: str, fmt: str = "md", filename: str = ""): """Export conversation history as a downloadable file. diff --git a/routes/skills_routes.py b/routes/skills_routes.py index 711baa2e5..00bef589f 100644 --- a/routes/skills_routes.py +++ b/routes/skills_routes.py @@ -1409,7 +1409,7 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: # Prefer the configured DEFAULT (→ Utility) model — not the current chat # session's model. Fall back to the caller's session model only if unset. - url, model, headers = resolve_endpoint("default", owner=user) + url, model, headers = resolve_endpoint("utility", owner=user) if not url or not model: url = url or ((body.get("endpoint_url") or "").strip() or None) model = model or ((body.get("model") or "").strip() or None) diff --git a/routes/vault/__init__.py b/routes/vault/__init__.py new file mode 100644 index 000000000..8aa82701d --- /dev/null +++ b/routes/vault/__init__.py @@ -0,0 +1,5 @@ +"""Vault route domain package (slice 2k, #4082/#4071). + +Contains vault_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/vault_routes.py re-exports from here. +""" diff --git a/routes/vault/vault_routes.py b/routes/vault/vault_routes.py new file mode 100644 index 000000000..7e97500f0 --- /dev/null +++ b/routes/vault/vault_routes.py @@ -0,0 +1,242 @@ +""" +vault_routes.py + +Vaultwarden / Bitwarden CLI integration — config and unlock endpoints. +Stores the BW_SESSION key in data/vault.json with restrictive permissions. +""" + +import json +import logging +import os +import shutil +import asyncio +from pathlib import Path +from datetime import datetime +from fastapi import APIRouter, Request +from pydantic import BaseModel + +from core.middleware import require_admin +from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool +from src.constants import VAULT_FILE as _VAULT_FILE + +logger = logging.getLogger(__name__) + +VAULT_FILE = Path(_VAULT_FILE) + + +def _find_bw() -> str: + """Locate the bw binary, checking PATH and common npm-global locations. + + On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by + which_tool via PATHEXT. + """ + p = which_tool("bw") + if p: + return p + if IS_WINDOWS: + appdata = os.environ.get("APPDATA", os.path.expanduser("~")) + for candidate in ( + os.path.join(appdata, "npm", "bw.cmd"), + os.path.join(appdata, "npm", "bw.exe"), + ): + if os.path.isfile(candidate): + return candidate + return "bw" + home = os.path.expanduser("~") + for candidate in ( + f"{home}/.npm-global/bin/bw", + f"{home}/.nvm/versions/node/*/bin/bw", + "/usr/local/bin/bw", + "/opt/homebrew/bin/bw", + ): + if "*" in candidate: + import glob + for m in glob.glob(candidate): + if os.path.isfile(m) and os.access(m, os.X_OK): + return m + elif os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below) + + +def _load_config() -> dict: + if VAULT_FILE.exists(): + try: + data = json.loads(VAULT_FILE.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + pass + return {} + + +def _save_config(cfg: dict): + VAULT_FILE.parent.mkdir(parents=True, exist_ok=True) + VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir + # is ACL-restricted already). + safe_chmod(str(VAULT_FILE), 0o600) + + +async def _run_bw(args: list, session: str = None, input_text: str = None, + bw_password: str = None) -> tuple: + env = {} + env.update(os.environ) + if session: + env["BW_SESSION"] = session + # Secrets must never be passed as argv — process arguments are world-readable + # via `ps` / `/proc//cmdline` to any local user. Keep --passwordenv + # support for bw commands that need it; unlock/login callers should prefer + # stdin so the master password is not left in the child environment either. + if bw_password is not None: + env["BW_PASSWORD"] = bw_password + bw_path = _find_bw() + try: + proc = await asyncio.create_subprocess_exec( + bw_path, *args, + stdin=asyncio.subprocess.PIPE if input_text else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + except FileNotFoundError: + return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127 + except Exception as e: + return "", f"Failed to launch bw: {e}", 1 + try: + stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None) + except Exception as e: + return "", f"bw subprocess error: {e}", 1 + return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode + + +class VaultConfig(BaseModel): + server_url: str = "" + email: str = "" + + +class VaultUnlockRequest(BaseModel): + master_password: str + + +class VaultLoginRequest(BaseModel): + email: str + master_password: str + + +def setup_vault_routes(): + router = APIRouter(prefix="/api/vault", tags=["vault"]) + + @router.get("/config") + async def get_config(request: Request): + """Return vault config (no sensitive fields).""" + require_admin(request) + cfg = _load_config() + return { + "server_url": cfg.get("server_url", ""), + "email": cfg.get("email", ""), + "unlocked": bool(cfg.get("session")), + "unlocked_at": cfg.get("unlocked_at", ""), + "bw_installed": await _check_bw_installed(), + } + + @router.post("/config") + async def save_config(req: VaultConfig, request: Request): + """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden.""" + require_admin(request) + cfg = _load_config() + cfg["server_url"] = req.server_url.strip().rstrip("/") + cfg["email"] = req.email.strip() + + if cfg["server_url"]: + _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]]) + if rc != 0: + return {"ok": False, "error": f"bw config failed: {stderr[:300]}"} + + _save_config(cfg) + return {"ok": True} + + @router.post("/login") + async def login(req: VaultLoginRequest, request: Request): + """Log in to Vaultwarden (required once per account).""" + require_admin(request) + cfg = _load_config() + # Update email + cfg["email"] = req.email + _save_config(cfg) + + stdout, stderr, rc = await _run_bw( + ["login", req.email, "--raw"], + input_text=req.master_password + "\n", + ) + if rc != 0: + # Already logged in is OK + if "already logged in" in stderr.lower(): + return {"ok": True, "already": True} + return {"ok": False, "error": f"Login failed: {stderr[:300]}"} + # bw login --raw prints session key on success (when 2FA disabled) + if stdout: + cfg["session"] = stdout + cfg["unlocked_at"] = datetime.utcnow().isoformat() + _save_config(cfg) + return {"ok": True} + + @router.post("/unlock") + async def unlock(req: VaultUnlockRequest, request: Request): + """Unlock the vault and save the session key.""" + require_admin(request) + # Pass the master password on stdin, not argv. argv is visible through + # `ps` / /proc//cmdline; stdin also avoids leaving the secret in + # the child process environment. + stdout, stderr, rc = await _run_bw( + ["unlock", "--raw"], + input_text=req.master_password + "\n", + ) + if rc != 0: + return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"} + session = stdout.strip() + if not session: + return {"ok": False, "error": "bw returned empty session"} + cfg = _load_config() + cfg["session"] = session + cfg["unlocked_at"] = datetime.utcnow().isoformat() + _save_config(cfg) + return {"ok": True, "message": "Vault unlocked"} + + @router.post("/lock") + async def lock(request: Request): + """Lock the vault (clear session from config).""" + require_admin(request) + cfg = _load_config() + cfg.pop("session", None) + cfg.pop("unlocked_at", None) + _save_config(cfg) + # Also tell bw to lock + await _run_bw(["lock"]) + return {"ok": True, "message": "Vault locked"} + + @router.post("/logout") + async def logout(request: Request): + """Log out of the Bitwarden CLI completely.""" + require_admin(request) + await _run_bw(["logout"]) + cfg = _load_config() + cfg.pop("session", None) + cfg.pop("email", None) + cfg.pop("unlocked_at", None) + _save_config(cfg) + return {"ok": True} + + return router + + +async def _check_bw_installed() -> bool: + try: + proc = await asyncio.create_subprocess_exec( + _find_bw(), "--version", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.communicate() + return proc.returncode == 0 + except Exception: + return False diff --git a/routes/vault_routes.py b/routes/vault_routes.py index 7e97500f0..cfed2ba39 100644 --- a/routes/vault_routes.py +++ b/routes/vault_routes.py @@ -1,242 +1,14 @@ -""" -vault_routes.py +"""Backward-compat shim — canonical location is routes/vault/vault_routes.py. -Vaultwarden / Bitwarden CLI integration — config and unlock endpoints. -Stores the BW_SESSION key in data/vault.json with restrictive permissions. +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.vault_routes``, ``from routes.vault_routes import X``, +and the ``import ... as vr`` + ``monkeypatch.setattr(vr, ...)`` pattern used +by test_vault_password_not_in_argv.py all operate on the *same* object. +Keeps existing import paths working after slice 2k (#4082/#4071). """ -import json -import logging -import os -import shutil -import asyncio -from pathlib import Path -from datetime import datetime -from fastapi import APIRouter, Request -from pydantic import BaseModel +import sys as _sys -from core.middleware import require_admin -from core.platform_compat import IS_WINDOWS, safe_chmod, which_tool -from src.constants import VAULT_FILE as _VAULT_FILE +from routes.vault import vault_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - -VAULT_FILE = Path(_VAULT_FILE) - - -def _find_bw() -> str: - """Locate the bw binary, checking PATH and common npm-global locations. - - On Windows the Bitwarden CLI shim is `bw.cmd`/`bw.exe`, resolved by - which_tool via PATHEXT. - """ - p = which_tool("bw") - if p: - return p - if IS_WINDOWS: - appdata = os.environ.get("APPDATA", os.path.expanduser("~")) - for candidate in ( - os.path.join(appdata, "npm", "bw.cmd"), - os.path.join(appdata, "npm", "bw.exe"), - ): - if os.path.isfile(candidate): - return candidate - return "bw" - home = os.path.expanduser("~") - for candidate in ( - f"{home}/.npm-global/bin/bw", - f"{home}/.nvm/versions/node/*/bin/bw", - "/usr/local/bin/bw", - "/opt/homebrew/bin/bw", - ): - if "*" in candidate: - import glob - for m in glob.glob(candidate): - if os.path.isfile(m) and os.access(m, os.X_OK): - return m - elif os.path.isfile(candidate) and os.access(candidate, os.X_OK): - return candidate - return "bw" # fall back to PATH lookup (will FileNotFoundError, handled below) - - -def _load_config() -> dict: - if VAULT_FILE.exists(): - try: - data = json.loads(VAULT_FILE.read_text(encoding="utf-8")) - return data if isinstance(data, dict) else {} - except Exception: - pass - return {} - - -def _save_config(cfg: dict): - VAULT_FILE.parent.mkdir(parents=True, exist_ok=True) - VAULT_FILE.write_text(json.dumps(cfg, indent=2), encoding="utf-8") - # POSIX: restrict the BW_SESSION store to 0o600. Windows: no-op (profile dir - # is ACL-restricted already). - safe_chmod(str(VAULT_FILE), 0o600) - - -async def _run_bw(args: list, session: str = None, input_text: str = None, - bw_password: str = None) -> tuple: - env = {} - env.update(os.environ) - if session: - env["BW_SESSION"] = session - # Secrets must never be passed as argv — process arguments are world-readable - # via `ps` / `/proc//cmdline` to any local user. Keep --passwordenv - # support for bw commands that need it; unlock/login callers should prefer - # stdin so the master password is not left in the child environment either. - if bw_password is not None: - env["BW_PASSWORD"] = bw_password - bw_path = _find_bw() - try: - proc = await asyncio.create_subprocess_exec( - bw_path, *args, - stdin=asyncio.subprocess.PIPE if input_text else None, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=env, - ) - except FileNotFoundError: - return "", "bw CLI not installed (install `nodejs-bitwarden-cli` or `bitwarden-cli`)", 127 - except Exception as e: - return "", f"Failed to launch bw: {e}", 1 - try: - stdout, stderr = await proc.communicate(input=input_text.encode() if input_text else None) - except Exception as e: - return "", f"bw subprocess error: {e}", 1 - return stdout.decode(errors="replace").strip(), stderr.decode(errors="replace").strip(), proc.returncode - - -class VaultConfig(BaseModel): - server_url: str = "" - email: str = "" - - -class VaultUnlockRequest(BaseModel): - master_password: str - - -class VaultLoginRequest(BaseModel): - email: str - master_password: str - - -def setup_vault_routes(): - router = APIRouter(prefix="/api/vault", tags=["vault"]) - - @router.get("/config") - async def get_config(request: Request): - """Return vault config (no sensitive fields).""" - require_admin(request) - cfg = _load_config() - return { - "server_url": cfg.get("server_url", ""), - "email": cfg.get("email", ""), - "unlocked": bool(cfg.get("session")), - "unlocked_at": cfg.get("unlocked_at", ""), - "bw_installed": await _check_bw_installed(), - } - - @router.post("/config") - async def save_config(req: VaultConfig, request: Request): - """Save vault URL + email. Runs 'bw config server' to point at Vaultwarden.""" - require_admin(request) - cfg = _load_config() - cfg["server_url"] = req.server_url.strip().rstrip("/") - cfg["email"] = req.email.strip() - - if cfg["server_url"]: - _, stderr, rc = await _run_bw(["config", "server", cfg["server_url"]]) - if rc != 0: - return {"ok": False, "error": f"bw config failed: {stderr[:300]}"} - - _save_config(cfg) - return {"ok": True} - - @router.post("/login") - async def login(req: VaultLoginRequest, request: Request): - """Log in to Vaultwarden (required once per account).""" - require_admin(request) - cfg = _load_config() - # Update email - cfg["email"] = req.email - _save_config(cfg) - - stdout, stderr, rc = await _run_bw( - ["login", req.email, "--raw"], - input_text=req.master_password + "\n", - ) - if rc != 0: - # Already logged in is OK - if "already logged in" in stderr.lower(): - return {"ok": True, "already": True} - return {"ok": False, "error": f"Login failed: {stderr[:300]}"} - # bw login --raw prints session key on success (when 2FA disabled) - if stdout: - cfg["session"] = stdout - cfg["unlocked_at"] = datetime.utcnow().isoformat() - _save_config(cfg) - return {"ok": True} - - @router.post("/unlock") - async def unlock(req: VaultUnlockRequest, request: Request): - """Unlock the vault and save the session key.""" - require_admin(request) - # Pass the master password on stdin, not argv. argv is visible through - # `ps` / /proc//cmdline; stdin also avoids leaving the secret in - # the child process environment. - stdout, stderr, rc = await _run_bw( - ["unlock", "--raw"], - input_text=req.master_password + "\n", - ) - if rc != 0: - return {"ok": False, "error": f"Unlock failed: {stderr[:300]}"} - session = stdout.strip() - if not session: - return {"ok": False, "error": "bw returned empty session"} - cfg = _load_config() - cfg["session"] = session - cfg["unlocked_at"] = datetime.utcnow().isoformat() - _save_config(cfg) - return {"ok": True, "message": "Vault unlocked"} - - @router.post("/lock") - async def lock(request: Request): - """Lock the vault (clear session from config).""" - require_admin(request) - cfg = _load_config() - cfg.pop("session", None) - cfg.pop("unlocked_at", None) - _save_config(cfg) - # Also tell bw to lock - await _run_bw(["lock"]) - return {"ok": True, "message": "Vault locked"} - - @router.post("/logout") - async def logout(request: Request): - """Log out of the Bitwarden CLI completely.""" - require_admin(request) - await _run_bw(["logout"]) - cfg = _load_config() - cfg.pop("session", None) - cfg.pop("email", None) - cfg.pop("unlocked_at", None) - _save_config(cfg) - return {"ok": True} - - return router - - -async def _check_bw_installed() -> bool: - try: - proc = await asyncio.create_subprocess_exec( - _find_bw(), "--version", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await proc.communicate() - return proc.returncode == 0 - except Exception: - return False +_sys.modules[__name__] = _canonical diff --git a/routes/webhook/__init__.py b/routes/webhook/__init__.py new file mode 100644 index 000000000..e51389e3a --- /dev/null +++ b/routes/webhook/__init__.py @@ -0,0 +1,5 @@ +"""Webhook route domain package (slice 2l, #4082/#4071). + +Contains webhook_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/webhook_routes.py re-exports from here. +""" diff --git a/routes/webhook/webhook_routes.py b/routes/webhook/webhook_routes.py new file mode 100644 index 000000000..8d3a704c6 --- /dev/null +++ b/routes/webhook/webhook_routes.py @@ -0,0 +1,395 @@ +"""Webhook, API Token, and sync chat routes.""" + +import uuid +import logging +from typing import Optional + +import httpx +from fastapi import APIRouter, HTTPException, Request, Form +from pydantic import BaseModel, Field + +from core.database import SessionLocal, Webhook, ModelEndpoint +from src.auth_helpers import owner_filter +from src.url_security import validate_public_http_url +from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api", tags=["webhooks"]) + +# Input limits +MAX_NAME_LEN = 100 +MAX_URL_LEN = 2048 +MAX_SECRET_LEN = 256 +MAX_MESSAGE_LEN = 32_000 + + +from core.middleware import require_admin as _require_admin + + +def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]): + """First enabled ModelEndpoint visible to token_owner — their own rows plus + legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would + let a chat-scoped token fall back onto another user's private endpoint and + silently spend that owner's API key/quota. Prefer owner rows before shared + rows. Fails closed to null-owner rows only when token_owner is absent. + Does not validate base_url — admin-configured local/LAN endpoints remain allowed. + """ + query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712 + if token_owner: + query = owner_filter(query, ModelEndpoint, token_owner) + return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first() + return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711 + + +def _caller_owns_session(sess_owner, caller) -> bool: + """Strict session-ownership gate for the token-authenticated sync-chat + endpoint (`POST /api/v1/chat`). + + Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner + gates in notes/calendar/gallery: a caller may resume a session ONLY when + its owner matches them exactly. A null/empty session owner (legacy or + migrated rows) is deliberately NOT resumable by an arbitrary token — the + old ``sess_owner and sess_owner != caller`` form skipped the check whenever + ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile + device) could resume such a session, inject a message, and read back its + history and reuse the owner's endpoint credentials. Fail closed: an + unresolvable caller also returns False. + """ + if not caller: + return False + return sess_owner == caller + + +def setup_webhook_routes( + webhook_manager: WebhookManager, + auth_manager, + session_manager=None, + api_key_manager=None, +) -> APIRouter: + + @router.get("/webhooks") + def list_webhooks(request: Request): + _require_admin(request) + db = SessionLocal() + try: + hooks = db.query(Webhook).all() + return [ + { + "id": w.id, + "name": w.name, + "url": w.url, + "has_secret": bool(w.secret), + "events": w.events.split(",") if w.events else [], + "is_active": w.is_active, + "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None, + "last_status_code": w.last_status_code, + "last_error": w.last_error, + "created_at": w.created_at.isoformat() if w.created_at else None, + } + for w in hooks + ] + finally: + db.close() + + @router.post("/webhooks") + def create_webhook( + request: Request, + name: str = Form(""), + url: str = Form(""), + secret: str = Form(""), + events: str = Form(""), + ): + _require_admin(request) + name = name.strip()[:MAX_NAME_LEN] + if not name: + raise HTTPException(400, "Webhook name is required") + try: + url = validate_webhook_url(url) + except ValueError as e: + raise HTTPException(400, str(e)) + try: + events = validate_events(events) + except ValueError as e: + raise HTTPException(400, str(e)) + + secret_val = secret.strip()[:MAX_SECRET_LEN] or None + # Encrypt the secret at rest using the same Fernet key as API keys + encrypted_secret = None + if secret_val and api_key_manager: + encrypted_secret = api_key_manager.encrypt_api_key(secret_val) + elif secret_val: + encrypted_secret = secret_val # Fallback if no encryption available + + webhook_id = str(uuid.uuid4())[:8] + db = SessionLocal() + try: + db.add(Webhook( + id=webhook_id, + name=name, + url=url, + secret=encrypted_secret, + events=events, + is_active=True, + )) + db.commit() + finally: + db.close() + + return {"id": webhook_id, "name": name} + + @router.post("/webhooks/{webhook_id}/test") + async def test_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() + if not wh: + raise HTTPException(404, "Webhook not found") + url, secret = wh.url, wh.secret + finally: + db.close() + + await webhook_manager.deliver_test(webhook_id, url, secret) + return {"status": "sent"} + + @router.patch("/webhooks/{webhook_id}") + def toggle_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() + if not wh: + raise HTTPException(404, "Webhook not found") + wh.is_active = not wh.is_active + db.commit() + return {"id": webhook_id, "is_active": wh.is_active} + finally: + db.close() + + @router.delete("/webhooks/{webhook_id}") + def delete_webhook(request: Request, webhook_id: str): + _require_admin(request) + db = SessionLocal() + try: + deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete() + db.commit() + if not deleted: + raise HTTPException(404, "Webhook not found") + finally: + db.close() + return {"status": "deleted"} + + # ================================================================ + # Sync Chat Endpoint (for n8n / Make / Activepieces) + # ================================================================ + + # Known provider base URLs — auto-resolved from api_key prefix or model name + KNOWN_PROVIDERS = { + "deepseek": "https://api.deepseek.com/v1", + "openai": "https://api.openai.com/v1", + "mistral": "https://api.mistral.ai/v1", + "groq": "https://api.groq.com/openai/v1", + "together": "https://api.together.xyz/v1", + "openrouter": "https://openrouter.ai/api/v1", + "ollama": "https://ollama.com/api", + "opencode-zen": "https://opencode.ai/zen/v1", + "opencode-go": "https://opencode.ai/zen/go/v1", + "fireworks": "https://api.fireworks.ai/inference/v1", + "venice": "https://api.venice.ai/api/v1", + "kimi-code": "https://api.kimi.com/coding/v1", + "kimicode": "https://api.kimi.com/coding/v1", + } + + # Model prefix → provider mapping for auto-detection + MODEL_PROVIDER_MAP = { + "deepseek": "deepseek", + "gpt-": "openai", + "o1": "openai", + "o3": "openai", + "o4": "openai", + "mistral": "mistral", + "llama": "groq", + "mixtral": "groq", + "kimi-for-coding": "kimi-code", + "kimi": "kimi-code", + } + + def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]: + """Try to auto-resolve a base URL from provider name or model prefix.""" + if provider and provider.lower() in KNOWN_PROVIDERS: + return KNOWN_PROVIDERS[provider.lower()] + if model: + model_lower = model.lower() + for prefix, prov in MODEL_PROVIDER_MAP.items(): + if model_lower.startswith(prefix): + return KNOWN_PROVIDERS[prov] + return None + + class SyncChatRequest(BaseModel): + message: str = Field(..., max_length=MAX_MESSAGE_LEN) + model: Optional[str] = Field(None, max_length=200) + session: Optional[str] = Field(None, max_length=100) + api_key: Optional[str] = Field(None, max_length=256) + base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN) + provider: Optional[str] = Field(None, max_length=50) + + @router.post("/v1/chat") + async def sync_chat(request: Request, body: SyncChatRequest): + if not getattr(request.state, "api_token", False): + raise HTTPException(403, "This endpoint requires an API token") + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + if "chat" not in scopes: + raise HTTPException(403, "API token is not scoped for chat") + token_owner = getattr(request.state, "api_token_owner", None) + + from core.models import ChatMessage + from src.llm_core import llm_call_async + from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base + + message = body.message.strip() + if not message: + raise HTTPException(400, "Message is required") + + session_id = body.session + sess = None + + # --- Case 1: Resume an existing session --- + if session_id and session_manager: + try: + sess = session_manager.get_session(session_id) + except (KeyError, Exception): + raise HTTPException(404, "Session not found") + # SECURITY: verify the API-token's user owns this session — without + # this any token holder could resume any user's chat by passing its + # ID. The token's user is on request.state.user (set by API-token + # middleware); fall back to require_user if not present. + try: + from src.auth_helpers import get_current_user as _gcu + _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request) + except Exception: + _tok_user = None + # Strict ownership (see _caller_owns_session): fail closed so a + # null-owner / cross-owner session can't be resumed by an arbitrary + # chat-scoped token. + _sess_owner = getattr(sess, "owner", None) + if not _caller_owns_session(_sess_owner, _tok_user): + raise HTTPException(404, "Session not found") + + # --- Case 2: Direct API key + model (no pre-configured endpoint needed) --- + if not sess and body.api_key: + api_key = body.api_key.strip() + model = body.model or "deepseek-chat" + + # Validate only token-supplied direct base_url; auto-resolved known-provider + # URLs are not subject to extra local/LAN blocking beyond existing provider logic. + direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None + if direct_base_url: + try: + base_url = validate_public_http_url(direct_base_url) + except ValueError as e: + detail = str(e).replace("URL", "base_url", 1) + raise HTTPException(400, detail) + else: + base_url = _resolve_base_url(model, body.provider) + if not base_url: + raise HTTPException(400, + "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') " + "or provider ('deepseek', 'openai', 'groq', etc.)") + base_url = normalize_base(base_url) + endpoint_url = build_chat_url(base_url) + + if not session_manager: + raise HTTPException(500, "Session manager not available") + + sid = str(uuid.uuid4()) + sess = session_manager.create_session( + session_id=sid, name="API Chat", endpoint_url=endpoint_url, + model=model, owner=token_owner, + ) + sess.headers = build_headers(api_key, base_url) + session_manager.save_sessions() + session_id = sid + + # --- Case 3: Fall back to first configured ModelEndpoint --- + if not sess: + db = SessionLocal() + try: + ep = _select_api_chat_fallback_endpoint(db, token_owner) + finally: + db.close() + + if not ep: + raise HTTPException(400, + "No session, api_key, or configured endpoints. " + "Pass api_key + model, or configure an endpoint in Admin.") + + base_url = normalize_base(ep.base_url) + endpoint_url = build_chat_url(base_url) + model = body.model or "auto" + api_key = ep.api_key + if getattr(ep, "provider_auth_id", None): + try: + from src.endpoint_resolver import resolve_endpoint_runtime + base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner) + endpoint_url = build_chat_url(base_url) + except Exception: + raise HTTPException(500, "Could not resolve endpoint credentials") + + if model == "auto": + try: + async with httpx.AsyncClient(timeout=5) as client: + models_url = build_models_url(base_url) + hdrs = build_headers(api_key, base_url) + if models_url: + resp = await client.get(models_url, headers=hdrs) + resp.raise_for_status() + data = resp.json() + items = data if isinstance(data, list) else (data.get("data") or []) + ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] + if not ids and isinstance(data, dict): + ids = [ + m.get("name") or m.get("model") + for m in (data.get("models") or []) + if m.get("name") or m.get("model") + ] + else: + import json as _json + ids = _json.loads(ep.cached_models or "[]") + model = ids[0] if ids else "auto" + except Exception: + raise HTTPException(500, "Could not discover models from endpoint") + + if not session_manager: + raise HTTPException(500, "Session manager not available") + + sid = str(uuid.uuid4()) + sess = session_manager.create_session( + session_id=sid, name="API Chat", endpoint_url=endpoint_url, + model=model, owner=token_owner, + ) + if api_key: + sess.headers = build_headers(api_key, base_url) + session_manager.save_sessions() + session_id = sid + + # --- Send message and get response --- + sess.add_message(ChatMessage("user", message)) + + messages = [{"role": m.role, "content": m.content} for m in sess.history] + + reply = await llm_call_async( + sess.endpoint_url, sess.model, messages, + headers=sess.headers, timeout=120, + ) + sess.add_message(ChatMessage("assistant", reply)) + session_manager.save_sessions() + + webhook_manager.fire_and_forget("chat.completed", { + "session_id": session_id, "model": sess.model, + "user_message": message[:2000], "response": reply[:2000], + }) + + return {"response": reply, "session_id": session_id, "model": sess.model} + + return router diff --git a/routes/webhook_routes.py b/routes/webhook_routes.py index 8d3a704c6..7c5e0453e 100644 --- a/routes/webhook_routes.py +++ b/routes/webhook_routes.py @@ -1,395 +1,16 @@ -"""Webhook, API Token, and sync chat routes.""" +"""Backward-compat shim — canonical location is routes/webhook/webhook_routes.py. -import uuid -import logging -from typing import Optional +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.webhook_routes``, ``from routes.webhook_routes import X``, +``importlib.import_module("routes.webhook_routes")``, and the +``__import__("routes.webhook_routes", fromlist=[...])`` + ``setattr(wh_mod, +...)`` pattern used by test_null_owner_gates.py all operate on the *same* +object. Keeps existing import paths working after slice 2l (#4082/#4071). +Source-introspection tests read the canonical file by path. +""" -import httpx -from fastapi import APIRouter, HTTPException, Request, Form -from pydantic import BaseModel, Field +import sys as _sys -from core.database import SessionLocal, Webhook, ModelEndpoint -from src.auth_helpers import owner_filter -from src.url_security import validate_public_http_url -from src.webhook_manager import WebhookManager, validate_webhook_url, validate_events +from routes.webhook import webhook_routes as _canonical # noqa: F401 -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/api", tags=["webhooks"]) - -# Input limits -MAX_NAME_LEN = 100 -MAX_URL_LEN = 2048 -MAX_SECRET_LEN = 256 -MAX_MESSAGE_LEN = 32_000 - - -from core.middleware import require_admin as _require_admin - - -def _select_api_chat_fallback_endpoint(db, token_owner: Optional[str]): - """First enabled ModelEndpoint visible to token_owner — their own rows plus - legacy null-owner ("shared") rows. Owner-scoped: an unscoped .first() would - let a chat-scoped token fall back onto another user's private endpoint and - silently spend that owner's API key/quota. Prefer owner rows before shared - rows. Fails closed to null-owner rows only when token_owner is absent. - Does not validate base_url — admin-configured local/LAN endpoints remain allowed. - """ - query = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) # noqa: E712 - if token_owner: - query = owner_filter(query, ModelEndpoint, token_owner) - return query.order_by(ModelEndpoint.owner.desc(), ModelEndpoint.created_at).first() - return query.filter(ModelEndpoint.owner == None).order_by(ModelEndpoint.created_at).first() # noqa: E711 - - -def _caller_owns_session(sess_owner, caller) -> bool: - """Strict session-ownership gate for the token-authenticated sync-chat - endpoint (`POST /api/v1/chat`). - - Mirrors ``_verify_session_owner`` in session_routes.py and the null-owner - gates in notes/calendar/gallery: a caller may resume a session ONLY when - its owner matches them exactly. A null/empty session owner (legacy or - migrated rows) is deliberately NOT resumable by an arbitrary token — the - old ``sess_owner and sess_owner != caller`` form skipped the check whenever - ``sess_owner`` was falsy, so any chat-scoped token (e.g. a paired mobile - device) could resume such a session, inject a message, and read back its - history and reuse the owner's endpoint credentials. Fail closed: an - unresolvable caller also returns False. - """ - if not caller: - return False - return sess_owner == caller - - -def setup_webhook_routes( - webhook_manager: WebhookManager, - auth_manager, - session_manager=None, - api_key_manager=None, -) -> APIRouter: - - @router.get("/webhooks") - def list_webhooks(request: Request): - _require_admin(request) - db = SessionLocal() - try: - hooks = db.query(Webhook).all() - return [ - { - "id": w.id, - "name": w.name, - "url": w.url, - "has_secret": bool(w.secret), - "events": w.events.split(",") if w.events else [], - "is_active": w.is_active, - "last_triggered_at": w.last_triggered_at.isoformat() if w.last_triggered_at else None, - "last_status_code": w.last_status_code, - "last_error": w.last_error, - "created_at": w.created_at.isoformat() if w.created_at else None, - } - for w in hooks - ] - finally: - db.close() - - @router.post("/webhooks") - def create_webhook( - request: Request, - name: str = Form(""), - url: str = Form(""), - secret: str = Form(""), - events: str = Form(""), - ): - _require_admin(request) - name = name.strip()[:MAX_NAME_LEN] - if not name: - raise HTTPException(400, "Webhook name is required") - try: - url = validate_webhook_url(url) - except ValueError as e: - raise HTTPException(400, str(e)) - try: - events = validate_events(events) - except ValueError as e: - raise HTTPException(400, str(e)) - - secret_val = secret.strip()[:MAX_SECRET_LEN] or None - # Encrypt the secret at rest using the same Fernet key as API keys - encrypted_secret = None - if secret_val and api_key_manager: - encrypted_secret = api_key_manager.encrypt_api_key(secret_val) - elif secret_val: - encrypted_secret = secret_val # Fallback if no encryption available - - webhook_id = str(uuid.uuid4())[:8] - db = SessionLocal() - try: - db.add(Webhook( - id=webhook_id, - name=name, - url=url, - secret=encrypted_secret, - events=events, - is_active=True, - )) - db.commit() - finally: - db.close() - - return {"id": webhook_id, "name": name} - - @router.post("/webhooks/{webhook_id}/test") - async def test_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() - if not wh: - raise HTTPException(404, "Webhook not found") - url, secret = wh.url, wh.secret - finally: - db.close() - - await webhook_manager.deliver_test(webhook_id, url, secret) - return {"status": "sent"} - - @router.patch("/webhooks/{webhook_id}") - def toggle_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - wh = db.query(Webhook).filter(Webhook.id == webhook_id).first() - if not wh: - raise HTTPException(404, "Webhook not found") - wh.is_active = not wh.is_active - db.commit() - return {"id": webhook_id, "is_active": wh.is_active} - finally: - db.close() - - @router.delete("/webhooks/{webhook_id}") - def delete_webhook(request: Request, webhook_id: str): - _require_admin(request) - db = SessionLocal() - try: - deleted = db.query(Webhook).filter(Webhook.id == webhook_id).delete() - db.commit() - if not deleted: - raise HTTPException(404, "Webhook not found") - finally: - db.close() - return {"status": "deleted"} - - # ================================================================ - # Sync Chat Endpoint (for n8n / Make / Activepieces) - # ================================================================ - - # Known provider base URLs — auto-resolved from api_key prefix or model name - KNOWN_PROVIDERS = { - "deepseek": "https://api.deepseek.com/v1", - "openai": "https://api.openai.com/v1", - "mistral": "https://api.mistral.ai/v1", - "groq": "https://api.groq.com/openai/v1", - "together": "https://api.together.xyz/v1", - "openrouter": "https://openrouter.ai/api/v1", - "ollama": "https://ollama.com/api", - "opencode-zen": "https://opencode.ai/zen/v1", - "opencode-go": "https://opencode.ai/zen/go/v1", - "fireworks": "https://api.fireworks.ai/inference/v1", - "venice": "https://api.venice.ai/api/v1", - "kimi-code": "https://api.kimi.com/coding/v1", - "kimicode": "https://api.kimi.com/coding/v1", - } - - # Model prefix → provider mapping for auto-detection - MODEL_PROVIDER_MAP = { - "deepseek": "deepseek", - "gpt-": "openai", - "o1": "openai", - "o3": "openai", - "o4": "openai", - "mistral": "mistral", - "llama": "groq", - "mixtral": "groq", - "kimi-for-coding": "kimi-code", - "kimi": "kimi-code", - } - - def _resolve_base_url(model: Optional[str], provider: Optional[str]) -> Optional[str]: - """Try to auto-resolve a base URL from provider name or model prefix.""" - if provider and provider.lower() in KNOWN_PROVIDERS: - return KNOWN_PROVIDERS[provider.lower()] - if model: - model_lower = model.lower() - for prefix, prov in MODEL_PROVIDER_MAP.items(): - if model_lower.startswith(prefix): - return KNOWN_PROVIDERS[prov] - return None - - class SyncChatRequest(BaseModel): - message: str = Field(..., max_length=MAX_MESSAGE_LEN) - model: Optional[str] = Field(None, max_length=200) - session: Optional[str] = Field(None, max_length=100) - api_key: Optional[str] = Field(None, max_length=256) - base_url: Optional[str] = Field(None, max_length=MAX_URL_LEN) - provider: Optional[str] = Field(None, max_length=50) - - @router.post("/v1/chat") - async def sync_chat(request: Request, body: SyncChatRequest): - if not getattr(request.state, "api_token", False): - raise HTTPException(403, "This endpoint requires an API token") - scopes = set(getattr(request.state, "api_token_scopes", []) or []) - if "chat" not in scopes: - raise HTTPException(403, "API token is not scoped for chat") - token_owner = getattr(request.state, "api_token_owner", None) - - from core.models import ChatMessage - from src.llm_core import llm_call_async - from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base - - message = body.message.strip() - if not message: - raise HTTPException(400, "Message is required") - - session_id = body.session - sess = None - - # --- Case 1: Resume an existing session --- - if session_id and session_manager: - try: - sess = session_manager.get_session(session_id) - except (KeyError, Exception): - raise HTTPException(404, "Session not found") - # SECURITY: verify the API-token's user owns this session — without - # this any token holder could resume any user's chat by passing its - # ID. The token's user is on request.state.user (set by API-token - # middleware); fall back to require_user if not present. - try: - from src.auth_helpers import get_current_user as _gcu - _tok_user = token_owner or getattr(request.state, "user", None) or _gcu(request) - except Exception: - _tok_user = None - # Strict ownership (see _caller_owns_session): fail closed so a - # null-owner / cross-owner session can't be resumed by an arbitrary - # chat-scoped token. - _sess_owner = getattr(sess, "owner", None) - if not _caller_owns_session(_sess_owner, _tok_user): - raise HTTPException(404, "Session not found") - - # --- Case 2: Direct API key + model (no pre-configured endpoint needed) --- - if not sess and body.api_key: - api_key = body.api_key.strip() - model = body.model or "deepseek-chat" - - # Validate only token-supplied direct base_url; auto-resolved known-provider - # URLs are not subject to extra local/LAN blocking beyond existing provider logic. - direct_base_url = body.base_url.strip().rstrip("/") if body.base_url else None - if direct_base_url: - try: - base_url = validate_public_http_url(direct_base_url) - except ValueError as e: - detail = str(e).replace("URL", "base_url", 1) - raise HTTPException(400, detail) - else: - base_url = _resolve_base_url(model, body.provider) - if not base_url: - raise HTTPException(400, - "Could not auto-detect provider. Pass base_url (e.g. 'https://api.deepseek.com/v1') " - "or provider ('deepseek', 'openai', 'groq', etc.)") - base_url = normalize_base(base_url) - endpoint_url = build_chat_url(base_url) - - if not session_manager: - raise HTTPException(500, "Session manager not available") - - sid = str(uuid.uuid4()) - sess = session_manager.create_session( - session_id=sid, name="API Chat", endpoint_url=endpoint_url, - model=model, owner=token_owner, - ) - sess.headers = build_headers(api_key, base_url) - session_manager.save_sessions() - session_id = sid - - # --- Case 3: Fall back to first configured ModelEndpoint --- - if not sess: - db = SessionLocal() - try: - ep = _select_api_chat_fallback_endpoint(db, token_owner) - finally: - db.close() - - if not ep: - raise HTTPException(400, - "No session, api_key, or configured endpoints. " - "Pass api_key + model, or configure an endpoint in Admin.") - - base_url = normalize_base(ep.base_url) - endpoint_url = build_chat_url(base_url) - model = body.model or "auto" - api_key = ep.api_key - if getattr(ep, "provider_auth_id", None): - try: - from src.endpoint_resolver import resolve_endpoint_runtime - base_url, api_key = resolve_endpoint_runtime(ep, owner=token_owner) - endpoint_url = build_chat_url(base_url) - except Exception: - raise HTTPException(500, "Could not resolve endpoint credentials") - - if model == "auto": - try: - async with httpx.AsyncClient(timeout=5) as client: - models_url = build_models_url(base_url) - hdrs = build_headers(api_key, base_url) - if models_url: - resp = await client.get(models_url, headers=hdrs) - resp.raise_for_status() - data = resp.json() - items = data if isinstance(data, list) else (data.get("data") or []) - ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] - if not ids and isinstance(data, dict): - ids = [ - m.get("name") or m.get("model") - for m in (data.get("models") or []) - if m.get("name") or m.get("model") - ] - else: - import json as _json - ids = _json.loads(ep.cached_models or "[]") - model = ids[0] if ids else "auto" - except Exception: - raise HTTPException(500, "Could not discover models from endpoint") - - if not session_manager: - raise HTTPException(500, "Session manager not available") - - sid = str(uuid.uuid4()) - sess = session_manager.create_session( - session_id=sid, name="API Chat", endpoint_url=endpoint_url, - model=model, owner=token_owner, - ) - if api_key: - sess.headers = build_headers(api_key, base_url) - session_manager.save_sessions() - session_id = sid - - # --- Send message and get response --- - sess.add_message(ChatMessage("user", message)) - - messages = [{"role": m.role, "content": m.content} for m in sess.history] - - reply = await llm_call_async( - sess.endpoint_url, sess.model, messages, - headers=sess.headers, timeout=120, - ) - sess.add_message(ChatMessage("assistant", reply)) - session_manager.save_sessions() - - webhook_manager.fire_and_forget("chat.completed", { - "session_id": session_id, "model": sess.model, - "user_message": message[:2000], "response": reply[:2000], - }) - - return {"response": reply, "session_id": session_id, "model": sess.model} - - return router +_sys.modules[__name__] = _canonical diff --git a/scripts/demo_email/demo_account.py b/scripts/demo_email/demo_account.py index 9555b6791..8a0f1190a 100755 --- a/scripts/demo_email/demo_account.py +++ b/scripts/demo_email/demo_account.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Create/remove the switchable, non-default 'Demo' EmailAccount in Odysseus. +"""Create/remove the switchable 'Demo' EmailAccount in Odysseus. Mirrors the existing local-Dovecot account (localhost:31143, STARTTLS) but points at the throwaway demo@odysseus.local mailbox. Password is stored Fernet-encrypted @@ -20,7 +20,14 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(ROOT)) -from core.database import SessionLocal, EmailAccount, Base, engine # noqa: E402 +from core.database import ( # noqa: E402 + Base, + EmailAccount, + SessionLocal, + engine, + lock_email_account_owner_mutations, +) +from sqlalchemy import or_ # noqa: E402 from src.secret_storage import encrypt # noqa: E402 NAME = "Demo" @@ -31,18 +38,98 @@ IMAP_PASSWORD = "demodemo" OWNER = "" -def setup() -> int: - Base.metadata.create_all(bind=engine) +def _owner_scope(query, owner: str): + if owner: + return query.filter(EmailAccount.owner == owner) + return query.filter(or_(EmailAccount.owner == None, EmailAccount.owner == "")) # noqa: E711 + + +def _discover_demo_scopes() -> set[str]: db = SessionLocal() try: - acct = db.query(EmailAccount).filter( - EmailAccount.name == NAME, EmailAccount.imap_user == IMAP_USER - ).first() + return { + row.owner or "" + for row in db.query(EmailAccount).filter( + EmailAccount.name == NAME, + EmailAccount.imap_user == IMAP_USER, + ).all() + } + finally: + db.close() + + +def _lock_and_load_demo_rows(db, scopes: set[str]): + """Reload Demo rows under every observed owner lock.""" + scopes = set(scopes) or {OWNER} + while True: + lock_email_account_owner_mutations(db, *scopes) + rows = ( + db.query(EmailAccount) + .filter( + EmailAccount.name == NAME, + EmailAccount.imap_user == IMAP_USER, + ) + .order_by(EmailAccount.created_at.asc(), EmailAccount.id.asc()) + .all() + ) + current_scopes = {row.owner or "" for row in rows} + if current_scopes.issubset(scopes) or db.get_bind().dialect.name == "sqlite": + return rows + db.rollback() + scopes.update(current_scopes) + + +def _promote_oldest_enabled(db, owner: str, excluded_ids: list[str]) -> None: + remaining = _owner_scope( + db.query(EmailAccount).filter( + EmailAccount.enabled == True, # noqa: E712 + ~EmailAccount.id.in_(excluded_ids), + ), + owner, + ) + if remaining.filter(EmailAccount.is_default == True).first() is not None: # noqa: E712 + return + promote = remaining.order_by( + EmailAccount.created_at.asc(), EmailAccount.id.asc() + ).first() + if promote is not None: + promote.is_default = True + + +def setup() -> int: + Base.metadata.create_all(bind=engine) + scopes = _discover_demo_scopes() | {OWNER} + db = SessionLocal() + try: + rows = _lock_and_load_demo_rows(db, scopes) + acct = rows[0] if rows else None if acct is None: acct = EmailAccount(id=uuid.uuid4().hex, name=NAME) db.add(acct) + old_scope = acct.owner or "" + was_default = bool(acct.is_default) + if old_scope != OWNER: + # Move a non-default row first so the unique index cannot see two + # defaults transiently while SQLAlchemy flushes the owner move and + # old-scope promotion in separate UPDATE statements. + acct.is_default = False + acct.owner = OWNER + db.flush() + if was_default: + _promote_oldest_enabled(db, old_scope, [acct.id]) + + target_default = _owner_scope( + db.query(EmailAccount).filter( + EmailAccount.id != acct.id, + EmailAccount.is_default == True, # noqa: E712 + ), + OWNER, + ).first() acct.owner = OWNER - acct.is_default = False # never default — user switches to it + # Keep Demo non-default when a real default exists. If it is the only + # enabled account, it must be default to preserve normal create + # semantics and avoid leaving the owner partition without one. + acct.is_default = target_default is None acct.enabled = True acct.imap_host = "localhost" acct.imap_port = 31143 @@ -57,20 +144,27 @@ def setup() -> int: acct.smtp_password = encrypt(IMAP_PASSWORD) acct.from_address = IMAP_USER db.commit() - print(f"'{NAME}' account ready (id={acct.id}, non-default, switchable).") + state = "default" if acct.is_default else "non-default" + print(f"'{NAME}' account ready (id={acct.id}, {state}, switchable).") return 0 finally: db.close() def teardown() -> int: + scopes = _discover_demo_scopes() db = SessionLocal() try: - rows = db.query(EmailAccount).filter( - EmailAccount.name == NAME, EmailAccount.imap_user == IMAP_USER - ).all() + rows = _lock_and_load_demo_rows(db, scopes) + deleted_ids = [row.id for row in rows] + default_scopes = {row.owner or "" for row in rows if row.is_default} for r in rows: db.delete(r) + # Ensure the old default DELETE reaches the database before a + # replacement UPDATE; the unique index is enforced per statement. + db.flush() + for owner in default_scopes: + _promote_oldest_enabled(db, owner, deleted_ids) db.commit() print(f"removed {len(rows)} '{NAME}' account row(s).") return 0 diff --git a/services/memory/__init__.py b/services/memory/__init__.py index 53fc80bd8..31fa1d5fa 100644 --- a/services/memory/__init__.py +++ b/services/memory/__init__.py @@ -2,7 +2,7 @@ """Memory service — persistent memory storage and retrieval.""" from .service import MemoryService, Memory, MemorySearchResult -from .memory import MemoryManager +from .memory import MemoryManager, MemoryStoreUnreadable from .memory_vector import MemoryVectorStore __all__ = [ @@ -10,5 +10,6 @@ __all__ = [ "Memory", "MemorySearchResult", "MemoryManager", + "MemoryStoreUnreadable", "MemoryVectorStore", ] diff --git a/services/memory/memory.py b/services/memory/memory.py index 031c13ac4..b9aaaa2a8 100644 --- a/services/memory/memory.py +++ b/services/memory/memory.py @@ -5,6 +5,16 @@ application runtime instantiates ``src.memory.MemoryManager``, so keeping a parallel implementation here risks silent drift between import paths. """ -from src.memory import MemoryManager, get_text_similarity, tokenize +from src.memory import ( + MemoryManager, + MemoryStoreUnreadable, + get_text_similarity, + tokenize, +) -__all__ = ["MemoryManager", "get_text_similarity", "tokenize"] +__all__ = [ + "MemoryManager", + "MemoryStoreUnreadable", + "get_text_similarity", + "tokenize", +] diff --git a/services/memory/memory_extractor.py b/services/memory/memory_extractor.py index e5f609250..11539263b 100644 --- a/services/memory/memory_extractor.py +++ b/services/memory/memory_extractor.py @@ -17,6 +17,8 @@ import os import re from typing import Optional +from src.memory import MemoryStoreUnreadable + logger = logging.getLogger(__name__) @@ -387,7 +389,13 @@ async def extract_and_store( # Get owner from session _owner = getattr(session, 'owner', None) - existing = memory_manager.load_all() + # Strict load: this is a read-modify-write. Degrading to [] here would + # save only the newly extracted facts and drop the entire store. + try: + existing = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Skipping auto memory extraction, store unreadable: %s", e) + return added = 0 for fact in facts: @@ -626,7 +634,18 @@ async def audit_memories( # Merge audited entries back with other users' entries if owner: - all_entries = memory_manager.load_all() + # Strict load: the merge below reconstructs the whole file. If this + # degraded to [] we would save only this owner's audited slice and + # destroy every other tenant's memories. + try: + all_entries = memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Aborting memory audit save, store unreadable: %s", e) + return { + "before": before_count, + "after": before_count, + "error": "store_unreadable", + } audited_ids = {e["id"] for e in final_entries} other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)] # Also keep legacy entries that weren't part of this audit diff --git a/services/memory/skill_format.py b/services/memory/skill_format.py index 2b2dfb1b3..633f4bec5 100644 --- a/services/memory/skill_format.py +++ b/services/memory/skill_format.py @@ -50,7 +50,7 @@ import json import logging import re from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) @@ -100,6 +100,18 @@ def _parse_scalar(raw: str) -> Any: if raw.lower() in ("null", "none", "~"): return None if (raw[0] == raw[-1]) and raw[0] in ("'", '"'): + if raw[0] == '"': + # _emit_scalar writes double-quoted scalars with json.dumps, so + # decode the escapes instead of only stripping the quotes. Without + # this, `\"` / `\\` / `\uXXXX` stayed verbatim in the value and the + # next save escaped their backslashes again, doubling them on every + # load/save cycle (issue #5210). + try: + return json.loads(raw) + except ValueError: + # Hand-written file using escapes JSON rejects (e.g. a bare + # Windows path). Keep the previous literal reading. + pass return raw[1:-1] # Try number try: @@ -171,6 +183,26 @@ def parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]: return fm, body +# Characters that force a quoted scalar. The punctuation would otherwise change +# how the value reads back; the second row is every character str.splitlines() +# treats as a line break, and parse_frontmatter() reads one scalar per line, so +# emitting one of those bare would split the value across lines. +_FM_MUST_QUOTE = ( + ":", "#", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@", + "\n", "\r", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029", +) + +# json.dumps escapes every C0 control character, but with ensure_ascii=False it +# passes NEL / LINE SEPARATOR / PARAGRAPH SEPARATOR through literally, and +# str.splitlines() still breaks on all three. Re-escape exactly those, which +# json.loads decodes again on the way in, so the pair stays symmetric. +_FM_POST_DUMPS_ESCAPES = ( + ("\x85", "\\u0085"), + ("\u2028", "\\u2028"), + ("\u2029", "\\u2029"), +) + + def _emit_scalar(v: Any) -> str: if v is None: return "null" @@ -181,8 +213,15 @@ def _emit_scalar(v: Any) -> str: if isinstance(v, list): return "[" + ", ".join(_emit_scalar(x) for x in v) + "]" s = str(v) - if any(c in s for c in (":", "#", "\n", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@")): - return json.dumps(s) + if any(c in s for c in _FM_MUST_QUOTE): + # ensure_ascii=False keeps non-ASCII text as itself. SKILL.md is UTF-8 at + # both ends (skills.py reads it, atomic_write_text writes it), so the + # \uXXXX form bought nothing and leaked into the parsed value (#5210). + out = json.dumps(s, ensure_ascii=False) + for ch, esc in _FM_POST_DUMPS_ESCAPES: + if ch in out: + out = out.replace(ch, esc) + return out return s @@ -441,4 +480,4 @@ class Skill: def _now_iso() -> str: - return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/services/tts/tts_service.py b/services/tts/tts_service.py index 2120d7720..dd37865a7 100644 --- a/services/tts/tts_service.py +++ b/services/tts/tts_service.py @@ -2,6 +2,7 @@ """Multi-provider TTS service — dispatches to local Kokoro, OpenAI-compatible API, or browser.""" import io +import os import wave import logging import hashlib @@ -41,6 +42,11 @@ class TTSService: self.cache_dir = Path(cache_dir) self.cache_dir.mkdir(parents=True, exist_ok=True) self._kokoro = None # lazy-init + + try: + self.max_cache_bytes = int(os.getenv("ODYSSEUS_TTS_CACHE_MAX_BYTES", 500 * 1024 * 1024)) + except ValueError: + self.max_cache_bytes = 500 * 1024 * 1024 # ── Settings ── @@ -89,6 +95,53 @@ class TTSService: ext = ".mp3" if (len(data) >= 3 and (data[:3] == b'ID3' or (data[0] == 0xff and (data[1] & 0xe0) == 0xe0))) else ".wav" (self.cache_dir / f"{key}{ext}").write_bytes(data) + self._enforce_cache_limit() + + def _enforce_cache_limit(self): + """Evicts oldest files if the cache exceeds the configured byte limit.""" + if self.max_cache_bytes <= 0: + return + + try: + files = [] + total_size = 0 + + # Safely scan files and sum sizes, ignoring files deleted mid-scan + for f in self.cache_dir.iterdir(): + try: + if f.is_file() and f.suffix.lower() in (".mp3", ".wav"): + files.append(f) + total_size += f.stat().st_size + except OSError: + continue + + if total_size > self.max_cache_bytes: + logger.info( + f"TTS cache ({total_size} bytes) exceeded limit ({self.max_cache_bytes} bytes). Evicting oldest files." + ) + + # Sort files by modification time (oldest first) + try: + files.sort(key=lambda f: f.stat().st_mtime) + except OSError as e: + logger.warning(f"Failed to sort cache files by mtime: {e}") + + # Trim down to 80% of max capacity + target_size = self.max_cache_bytes * 0.8 + + while files and total_size > target_size: + f = files.pop(0) + try: + size = f.stat().st_size + f.unlink() + total_size -= size + except OSError as e: + logger.warning(f"Failed to evict cache file {f}: {e}") + continue + + except Exception as e: + logger.warning(f"Error enforcing TTS cache limit: {e}", exc_info=True) + def clear_cache(self): count = 0 for f in self.cache_dir.glob("*.*"): diff --git a/src/agent_tools/subprocess_tools.py b/src/agent_tools/subprocess_tools.py index 15041c76e..1c407b112 100644 --- a/src/agent_tools/subprocess_tools.py +++ b/src/agent_tools/subprocess_tools.py @@ -6,6 +6,7 @@ import sys import time import collections from typing import Optional, Callable, Awaitable, Tuple, Dict +from core.platform_compat import IS_WINDOWS, find_bash from src.constants import MAX_OUTPUT_CHARS DEFAULT_BASH_TIMEOUT = 60 * 60 # 1 hour @@ -16,6 +17,27 @@ PROGRESS_TAIL_LINES = 12 TMUX_CAPTURE_LINES = 2000 +async def _create_bash_subprocess(command: str, **kwargs): + """Start the agent shell with Bash semantics on every supported OS. + + ``asyncio.create_subprocess_shell`` delegates to ``cmd.exe`` on native + Windows. That contradicts the Bash tool contract and makes POSIX commands + such as ``pwd``, ``ls -la``, and ``cat`` unreliable even when the launcher + has found Git Bash. Pass the selected workspace as a structural ``cwd`` + argument; Git Bash inherits that native Windows directory and exposes it + using its normal ``/c/...`` representation. + """ + if IS_WINDOWS: + bash = find_bash() + if not bash: + raise RuntimeError( + "Git Bash is required for the Bash tool on Windows; " + "install Git for Windows and restart Odysseus" + ) + return await asyncio.create_subprocess_exec(bash, "-c", command, **kwargs) + return await asyncio.create_subprocess_shell(command, **kwargs) + + def _tmux_session_name(session_id: Optional[str]) -> str: raw = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(session_id or "default")).strip("-") return f"ody-agent-{raw[:80] or 'default'}" @@ -280,7 +302,10 @@ class BashTool: progress_cb = ctx.get("progress_cb") _subproc_env = ctx.get("subproc_env") session_id = ctx.get("session_id") - if session_id and shutil.which("tmux"): + # tmux is a POSIX persistence path. A stray MSYS/Cygwin tmux.exe on + # native Windows must not bypass the Git Bash launcher below: the tmux + # setup hard-codes /bin/bash and cannot safely consume a native cwd. + if session_id and not IS_WINDOWS and shutil.which("tmux"): stdout, stderr, rc, timed_out = await _run_tmux_bash( content, session_id=str(session_id), @@ -307,13 +332,16 @@ class BashTool: "tmux_session": _tmux_session_name(str(session_id)), } - proc = await asyncio.create_subprocess_shell( - content, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=_subproc_env, - cwd=agent_cwd(), - ) + try: + proc = await _create_bash_subprocess( + content, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=_subproc_env, + cwd=agent_cwd(), + ) + except RuntimeError as e: + return {"error": f"bash: {e}", "exit_code": 1} stdout, stderr, rc, timed_out = await _run_subprocess_streaming( proc, timeout=DEFAULT_BASH_TIMEOUT, diff --git a/src/ai_interaction.py b/src/ai_interaction.py index 9ee97368f..e777ca32a 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -22,6 +22,7 @@ import time from typing import Any, Awaitable, Callable, Dict, Optional, Tuple from src.constants import GENERATED_IMAGES_DIR +from src.memory import MemoryStoreUnreadable logger = logging.getLogger(__name__) @@ -384,7 +385,15 @@ async def do_manage_memory(content: str, session_id: Optional[str] = None, owner return {"error": "Memory text cannot be empty"} entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) - memories = _memory_manager.load_all() + # Strict load: this is a read-modify-write, and it is the path an + # ordinary "remember that I prefer X" takes. Degrading to [] here would + # save just this one entry over a store we only failed to read, + # atomically destroying every memory in it (issue #5673). + try: + memories = _memory_manager.load_all_for_update() + except MemoryStoreUnreadable as e: + logger.error("Refusing to add memory, store unreadable: %s", e) + return {"error": "Memory store is temporarily unreadable — nothing was saved."} memories.append(entry) _memory_manager.save(memories) diff --git a/src/builtin_actions.py b/src/builtin_actions.py index 68817467f..216bb0360 100644 --- a/src/builtin_actions.py +++ b/src/builtin_actions.py @@ -20,6 +20,395 @@ from src.interactive_gate import wait_for_interactive_quiet logger = logging.getLogger(__name__) +def _read_email_urgency_state(state_path): + """Read one atomic urgency checkpoint, tolerating the legacy shape.""" + from pathlib import Path + + state_path = Path(state_path) + try: + state = ( + json.loads(state_path.read_text(encoding="utf-8")) + if state_path.exists() + else {} + ) + except Exception: + return {} + return state if isinstance(state, dict) else {} + + +def _email_urgency_account_generations(state): + """Return normalized per-account checkpoint/complete generations. + + Checkpoint generations fence every accepted state mutation. Complete + generations advance only for a non-stale complete scan. Missing metadata + is the legacy generation zero. + """ + raw = state.get("account_generations", {}) if isinstance(state, dict) else {} + if not isinstance(raw, dict): + return {} + + generations = {} + for account_id, value in raw.items(): + if isinstance(value, dict): + checkpoint = value.get("checkpoint", 0) + complete = value.get("complete", 0) + else: + # Tolerate an intermediate scalar representation as one completed + # checkpoint generation instead of discarding its fence. + checkpoint = value + complete = value + try: + checkpoint = max(0, int(checkpoint)) + except (TypeError, ValueError): + checkpoint = 0 + try: + complete = max(0, int(complete)) + except (TypeError, ValueError): + complete = 0 + generations[str(account_id)] = { + "checkpoint": checkpoint, + "complete": complete, + } + return generations + + +def _email_urgency_string_set(value): + if not isinstance(value, (list, tuple, set, frozenset)): + return set() + return {str(item) for item in value if isinstance(item, (str, int))} + + +def _acquire_email_urgency_state_lock( + state_path, + lock_db_path, + cancel_event, + timeout_seconds=120, +): + """Acquire the cross-process urgency lock without blocking the app loop.""" + import sqlite3 + import time + from pathlib import Path + + state_path = Path(state_path) + state_path.parent.mkdir(parents=True, exist_ok=True) + deadline = time.monotonic() + timeout_seconds + + while not cancel_event.is_set(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise sqlite3.OperationalError("timed out waiting for urgency state lock") + conn = sqlite3.connect( + str(lock_db_path), + timeout=min(0.25, max(0.01, remaining)), + check_same_thread=False, + ) + try: + conn.execute("BEGIN IMMEDIATE") + except sqlite3.OperationalError as exc: + conn.close() + if "locked" not in str(exc).lower(): + raise + cancel_event.wait(min(0.05, max(0.0, remaining))) + continue + except BaseException: + conn.close() + raise + + if cancel_event.is_set(): + conn.rollback() + conn.close() + return None, None + return conn, _read_email_urgency_state(state_path) + + return None, None + + +def _close_email_urgency_state_lock(conn): + if conn is None: + return + try: + try: + conn.rollback() + except Exception: + pass + finally: + conn.close() + + +def _commit_email_urgency_state(conn, state_path, next_state): + """Atomically publish JSON before releasing the SQLite write lock.""" + import uuid + from pathlib import Path + + state_path = Path(state_path) + temp_path = state_path.with_name( + f".{state_path.name}.{uuid.uuid4().hex}.tmp" + ) + try: + temp_path.write_text(json.dumps(next_state), encoding="utf-8") + temp_path.replace(state_path) + conn.commit() + except BaseException: + conn.rollback() + raise + finally: + temp_path.unlink(missing_ok=True) + conn.close() + + +async def _run_email_urgency_state_transaction( + state_path, + lock_db_path, + operation, +): + """Serialize one urgency decision while keeping async work on this loop. + + Only lock acquisition waits in a worker thread. ``operation`` is awaited + on the caller's long-lived event loop, where shared async clients, locks, + and the browser-notification queue belong. Cancellation rolls back the + SQLite transaction and never publishes a checkpoint. + """ + import asyncio + import threading + + loop = asyncio.get_running_loop() + cancel_event = threading.Event() + acquire_future = loop.run_in_executor( + None, + _acquire_email_urgency_state_lock, + state_path, + lock_db_path, + cancel_event, + ) + try: + conn, prior = await asyncio.shield(acquire_future) + except asyncio.CancelledError as cancelled: + cancel_event.set() + # The acquisition worker owns any connection until it returns. Wait + # for its short busy-poll to observe cancellation, then close a lock it + # may have won concurrently with the cancellation request. + while True: + try: + conn, _prior = await asyncio.shield(acquire_future) + break + except asyncio.CancelledError: + continue + except Exception: + conn = None + break + _close_email_urgency_state_lock(conn) + raise cancelled + + if conn is None: + raise asyncio.CancelledError + + try: + result, next_state = await operation(prior) + # Keep this small atomic publish synchronous. There is no await between + # the successful operation and commit, so cancellation cannot be + # observed and then followed by a checkpoint. + try: + _commit_email_urgency_state(conn, state_path, next_state) + finally: + conn = None + return result + except BaseException: + _close_email_urgency_state_lock(conn) + raise + + +def _email_urgency_account_key(message_key): + return str(message_key).split(":", 1)[0] + + +def _email_urgency_payload_account_ids(state): + """Return account IDs that still own user-visible urgency payload.""" + if not isinstance(state, dict): + return set() + + per_uid = state.get("per_uid", {}) + per_uid_keys = per_uid if isinstance(per_uid, dict) else {} + return { + _email_urgency_account_key(key) for key in per_uid_keys + } | { + _email_urgency_account_key(key) + for key in _email_urgency_string_set(state.get("notified_uids", [])) + } + + +def _email_urgency_known_account_ids(state): + """Return payload owners plus generation-only active/retired markers.""" + return _email_urgency_payload_account_ids(state) | set( + _email_urgency_account_generations(state) + ) + + +def _email_urgency_stale_accounts( + prior, + base_account_generations, + account_ids, +): + prior_generations = _email_urgency_account_generations(prior) + base_generations = _email_urgency_account_generations( + {"account_generations": base_account_generations} + ) + return { + str(account_id) + for account_id in account_ids + if prior_generations.get(str(account_id), {}).get("checkpoint", 0) + != base_generations.get(str(account_id), {}).get("checkpoint", 0) + } + + +def _merge_email_urgency_state( + prior, + *, + owner, + per_uid_scores, + notified_uids, + all_unread_keys, + fully_scanned_account_ids, + base_account_generations, + timestamp, + retired_account_ids=(), + base_payload_account_ids=(), + known_account_ids=(), +): + """Merge a scan without letting an older snapshot erase newer facts.""" + prior_per_uid = prior.get("per_uid", {}) + if not isinstance(prior_per_uid, dict): + prior_per_uid = {} + complete = {str(account_id) for account_id in fully_scanned_account_ids} + prior_generations = _email_urgency_account_generations(prior) + retire_requested = {str(account_id) for account_id in retired_account_ids} + observed_accounts = { + _email_urgency_account_key(key) for key in per_uid_scores + } | complete | retire_requested + stale_accounts = _email_urgency_stale_accounts( + prior, + base_account_generations, + observed_accounts, + ) + prior_payload_accounts = _email_urgency_payload_account_ids(prior) + base_payload_accounts = { + str(account_id) for account_id in base_payload_account_ids + } + # A selected account can be absent from the base snapshot. If another + # worker creates its first payload before this transaction wins the lock, + # membership itself is a fence even when both snapshots normalize to the + # legacy generation zero. + retired_accounts = { + account_id + for account_id in retire_requested - stale_accounts + if not ( + account_id in prior_payload_accounts + and account_id not in base_payload_accounts + ) + } + fresh_complete = complete - stale_accounts - retired_accounts + changed_accounts = set(fresh_complete) + + merged_per_uid = { + key: value + for key, value in prior_per_uid.items() + if _email_urgency_account_key(key) not in retired_accounts + } + for key in list(merged_per_uid): + account_id = _email_urgency_account_key(key) + if account_id in fresh_complete: + merged_per_uid.pop(key, None) + changed_accounts.add(account_id) + # Partial scans may add or refresh facts, but absence from a partial scan + # is not evidence that another checkpoint or UI row is stale. When another + # worker committed after this scan captured its base generation, discard + # this account's whole stale snapshot. A key absent from the newer state + # may have been removed/read, so even a stale-only key is not safely + # additive without another fresh scan. + for key, value in per_uid_scores.items(): + account_id = _email_urgency_account_key(key) + if account_id in stale_accounts or account_id in retired_accounts: + continue + if merged_per_uid.get(key) != value: + changed_accounts.add(account_id) + merged_per_uid[key] = value + + prior_notified = _email_urgency_string_set(prior.get("notified_uids", [])) + merged_notified = { + key + for key in prior_notified + if _email_urgency_account_key(key) not in retired_accounts + } + for key in _email_urgency_string_set(notified_uids) - prior_notified: + account_id = _email_urgency_account_key(key) + if account_id in stale_accounts or account_id in retired_accounts: + continue + merged_notified.add(key) + changed_accounts.add(account_id) + for key in list(merged_notified): + if ( + _email_urgency_account_key(key) in fresh_complete + and key not in all_unread_keys + ): + merged_notified.discard(key) + changed_accounts.add(_email_urgency_account_key(key)) + + next_generations = { + account_id: dict(value) + for account_id, value in prior_generations.items() + } + for account_id in changed_accounts: + generation = next_generations.setdefault( + account_id, + {"checkpoint": 0, "complete": 0}, + ) + generation["checkpoint"] += 1 + if account_id in fresh_complete: + generation["complete"] += 1 + for account_id in {str(value) for value in known_account_ids}: + next_generations.setdefault( + account_id, + {"checkpoint": 0, "complete": 0}, + ) + for account_id in retired_accounts: + # Every authoritative absence advances its generation, even when the + # prior state is already a payload-empty tombstone. A re-enabled scan + # may have captured that previous tombstone immediately before the + # account was disabled/deleted again; monotonic advancement is what + # makes that in-flight scan stale. + generation = next_generations.setdefault( + account_id, + {"checkpoint": 0, "complete": 0}, + ) + generation["checkpoint"] += 1 + + total_unread = 0 + total_urgent = 0 + max_score = 0 + for value in merged_per_uid.values(): + if not isinstance(value, dict): + continue + try: + score = max(0, min(3, int(value.get("score", 0)))) + except (TypeError, ValueError): + score = 0 + max_score = max(max_score, score) + if value.get("unread"): + total_unread += 1 + if score >= 2: + total_urgent += 1 + + return { + "ts": timestamp, + "owner": owner or "", + "total_unread": total_unread, + "total_urgent": total_urgent, + "max_score": max_score, + "per_uid": merged_per_uid, + "notified_uids": sorted(merged_notified), + "account_generations": next_generations, + } + + class TaskNoop(BaseException): """Raised by an action when it determined there's nothing to do. @@ -1878,6 +2267,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: # filename for single-user installs (matches prior behaviour). _owner_slug = "".join(c if (c.isalnum() or c in "-_.@") else "_" for c in (owner or "default")) STATE_PATH = _P(DATA_DIR) / f"email_urgency_state_{_owner_slug}.json" + STATE_LOCK_DB = STATE_PATH.with_suffix(".lock.sqlite3") CACHE_DIR = _P(EMAIL_URGENCY_CACHE_DIR) CACHE_DIR.mkdir(parents=True, exist_ok=True) STATE_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -1892,35 +2282,144 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: "shopping", "social", "work", "personal", "legal", "support", "promo", } - # ── 1. Resolve LLM candidates (utility primary + utility fallbacks; fall - # through to default chat as a last resort). + # Resolve with the task owner as before, but defer the availability + # gate until after authoritative account cleanup. State retirement must + # still run when no model is configured. from src.task_endpoint import resolve_task_candidates candidates = resolve_task_candidates(owner=owner) - if not candidates: - return "No LLM endpoint available", False - target_account_id = _email_task_account_id(kwargs) - # ── 2. Enumerate enabled accounts. Match this task's owner AND fall + # ── 1. Enumerate enabled accounts. Match this task's owner AND fall # back to the legacy "unowned account whose imap_user / from_address # == this owner" pattern — same rule `_get_email_config` uses, so a # pre-multi-user account row still gets picked up for the seeded task. - db = _SL() - try: - from sqlalchemy import and_ as _and, or_ as _or - q = db.query(_EA).filter(_EA.enabled == True) # noqa: E712 - if owner: - unowned = _or(_EA.owner == None, _EA.owner == "") # noqa: E711 - same_mailbox = _or(_EA.imap_user == owner, _EA.from_address == owner) - q = q.filter(_or(_EA.owner == owner, _and(unowned, same_mailbox))) - if target_account_id: - q = q.filter(_EA.id == target_account_id) - accounts = q.all() - finally: - db.close() + def _enumerate_enabled_accounts(): + db = _SL() + try: + from sqlalchemy import and_ as _and, or_ as _or + q = db.query(_EA).filter(_EA.enabled == True) # noqa: E712 + if owner: + unowned = _or(_EA.owner == None, _EA.owner == "") # noqa: E711 + same_mailbox = _or( + _EA.imap_user == owner, + _EA.from_address == owner, + ) + q = q.filter( + _or(_EA.owner == owner, _and(unowned, same_mailbox)) + ) + if target_account_id: + q = q.filter(_EA.id == target_account_id) + return q.all() + finally: + db.close() + + initial_accounts = _enumerate_enabled_accounts() + initial_account_ids = { + str(account.id) for account in initial_accounts + } + + # Register every account before IMAP work, including its first-ever + # scan. A concurrent zero-account cleanup can then advance this marker + # and fence delivery even before the scan has produced payload. + registered_state = None + if initial_account_ids: + async def _register_accounts(prior): + next_state = _merge_email_urgency_state( + prior, + owner=owner, + per_uid_scores={}, + notified_uids=prior.get("notified_uids", []), + all_unread_keys=set(), + fully_scanned_account_ids=set(), + base_account_generations=( + _email_urgency_account_generations(prior) + ), + timestamp=_time.time(), + known_account_ids=initial_account_ids, + ) + # Return the exact state committed by registration. This is + # the scan's generation token: adopting a later checkpoint + # after account cleanup would let the stale scan appear fresh. + return next_state, next_state + + registered_state = await _run_email_urgency_state_transaction( + STATE_PATH, + STATE_LOCK_DB, + _register_accounts, + ) + + # Revalidate after registration. If deletion/disable and its cleanup + # completed before the marker was published, this second enumeration + # observes the absence and this action retires its own marker instead + # of starting IMAP. Accounts newly appearing between the two reads are + # left for the next pass rather than scanned without prior registration. + verified_accounts = _enumerate_enabled_accounts() + enabled_account_ids = { + str(account.id) for account in verified_accounts + } + accounts = [ + account + for account in verified_accounts + if str(account.id) in initial_account_ids + ] + + # Capture the checkpoint basis before cleanup or IMAP. A full + # owner-wide enumeration authoritatively retires all known state IDs + # absent from the current enabled/visible set. A scoped task may retire + # only its selected missing/disabled account. Existing accounts remain + # present even if their later network scan fails, so transient IMAP + # failure never erases their last known state. + base_state = ( + registered_state + if registered_state is not None + else _read_email_urgency_state(STATE_PATH) + ) + base_account_generations = _email_urgency_account_generations( + base_state + ) + base_payload_account_ids = _email_urgency_payload_account_ids(base_state) + known_state_account_ids = _email_urgency_known_account_ids(base_state) + if target_account_id: + retired_account_ids = ( + {str(target_account_id)} + if str(target_account_id) not in enabled_account_ids + else set() + ) + else: + retired_account_ids = ( + known_state_account_ids - enabled_account_ids + ) + + if retired_account_ids: + async def _retire_accounts(prior): + next_state = _merge_email_urgency_state( + prior, + owner=owner, + per_uid_scores={}, + notified_uids=prior.get("notified_uids", []), + all_unread_keys=set(), + fully_scanned_account_ids=set(), + base_account_generations=base_account_generations, + timestamp=_time.time(), + retired_account_ids=retired_account_ids, + base_payload_account_ids=base_payload_account_ids, + ) + return None, next_state + + await _run_email_urgency_state_transaction( + STATE_PATH, + STATE_LOCK_DB, + _retire_accounts, + ) if not accounts: raise TaskNoop("no email accounts configured") + # ── 2. Account retirement above is state maintenance and does not + # depend on model availability. Scanning still requires the utility + # primary/fallback candidates resolved for this task owner. + if not candidates: + return "No LLM endpoint available", False + urgency_prompt = settings.get("urgent_email_prompt", "") per_uid_scores = {} # key = ":" → {"score": 0-3, "reason": "..."} all_unread_keys = set() @@ -1929,6 +2428,7 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: failed_classifications = [] tag_write_details = [] scanned = 0 + fully_scanned_account_ids = set() def _heuristic_email_verdict(item: dict) -> dict: blob = ( @@ -2024,16 +2524,27 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: def _scan_one(account=acc, cache_uids=cache.get("uids", {})): """Sync IMAP work runs in a thread.""" results = [] + scan_complete = True conn = _imap_connect(account.id) try: - conn.select("INBOX", readonly=True) + select_status, _select_data = conn.select("INBOX", readonly=True) + if select_status != "OK": + return results, False # Tag recent inbox mail, not only unread mail. Urgency # reminders below still only notify for unread messages. since_str = AGE_CUTOFF.strftime("%d-%b-%Y") status, data = conn.uid("SEARCH", None, f'(SINCE {since_str})') - if status != "OK" or not data or not data[0]: - return results - uids = data[0].split()[-30:] + if status != "OK": + return results, False + if not data or not data[0]: + return results, True + matching_uids = data[0].split() + if len(matching_uids) > 30: + # The scale guard deliberately processes only the most + # recent 30. That is a partial account snapshot, so it + # cannot justify pruning older checkpoint facts. + scan_complete = False + uids = matching_uids[-30:] for uid_b in uids: uid = uid_b.decode() if isinstance(uid_b, bytes) else str(uid_b) key = f"{account.id}:{uid}" @@ -2041,12 +2552,41 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: cached_ok = isinstance(cached, dict) and cached.get("triage_version") == TRIAGE_VERSION results.append({"key": key, "uid": uid, "cached": cached if cached_ok else None}) if cached_ok: - # Already classified — skip the fetch. + # Cached verdicts still need a lightweight FLAGS + # refresh. Without it a cached unread message looks + # read and its successful notification checkpoint + # is pruned on the next pass. + try: + st, flag_data = conn.uid("FETCH", uid_b, "(UID FLAGS)") + if st != "OK" or not flag_data: + scan_complete = False + results.pop() + continue + flag_parts = [] + for part in flag_data: + if isinstance(part, (bytes, bytearray)): + flag_parts.append(bytes(part)) + elif ( + isinstance(part, tuple) + and part + and isinstance(part[0], (bytes, bytearray)) + ): + flag_parts.append(bytes(part[0])) + flags_blob = b" ".join(flag_parts) + results[-1]["unread"] = b"\\Seen" not in flags_blob + except Exception as _fe: + scan_complete = False + results.pop() + logger.debug( + f"urgency: flag fetch for uid {uid} failed: {_fe}" + ) continue # Pull headers + first ~800 chars of plaintext body. try: st, msg_data = conn.uid("FETCH", uid_b, "(UID FLAGS RFC822.HEADER BODY.PEEK[TEXT]<0.800>)") if st != "OK" or not msg_data: + scan_complete = False + results.pop() continue flags_blob = b" ".join( part[0] for part in msg_data @@ -2060,6 +2600,8 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: if isinstance(part, tuple) and part[1]: raw += part[1] + b"\n\n" if not raw: + scan_complete = False + results.pop() continue msg = _email_mod.message_from_bytes(raw) # Skip Odysseus-generated reminders so the scanner @@ -2115,17 +2657,21 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: "unread": is_unread, }) except Exception as _fe: + scan_complete = False + results.pop() logger.debug(f"urgency: header fetch for uid {uid} failed: {_fe}") finally: try: conn.logout() except Exception: pass - return results + return results, scan_complete try: - items = await _aio.to_thread(_scan_one) + items, scan_complete = await _aio.to_thread(_scan_one) except Exception as e: logger.warning(f"urgency: IMAP scan failed for account {acc.id}: {e}") continue + if scan_complete: + fully_scanned_account_ids.add(str(acc.id)) for item in items: scanned += 1 @@ -2262,13 +2808,13 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: logger.debug(f"urgency: LLM classify failed for {key}: {e}") continue - # ── Prune cache entries for UIDs that are no longer in the recent - # scan window. Read messages remain cached because tags are useful - # on read mail too; unread state is refreshed per scan above. - seen_uids = {it["uid"] for it in items} - cache_uids = cache.get("uids", {}) - for stale in [u for u in cache_uids if u not in seen_uids]: - cache_uids.pop(stale, None) + if scan_complete: + # Only a complete account scan proves a cached UID left the + # recent window. Partial/failing scans preserve prior facts. + seen_uids = {it["uid"] for it in items} + cache_uids = cache.get("uids", {}) + for stale in [u for u in cache_uids if u not in seen_uids]: + cache_uids.pop(stale, None) try: cache_file.write_text(_json.dumps(cache), encoding="utf-8") @@ -2372,40 +2918,34 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: # ── 4. Aggregate state. urgent = score ≥ 2. urgent_keys = [k for k, v in per_uid_scores.items() if v.get("score", 0) >= 2 and v.get("unread")] - max_score = max((v.get("score", 0) for v in per_uid_scores.values()), default=0) - total_urgent = len(urgent_keys) - # Load prior state to know which urgent UIDs we've already notified. - try: - prior = _json.loads(STATE_PATH.read_text(encoding="utf-8")) if STATE_PATH.exists() else {} - except Exception: - prior = {} - notified_uids = set(prior.get("notified_uids", [])) - - # ── 5. Fire reminder ONLY when a previously-unnotified UID scores urgent. - new_urgent = [k for k in urgent_keys if k not in notified_uids] + # ── 5. Fire a reminder only when a previously-unnotified UID scores + # urgent. The read, decision, delivery, and checkpoint are serialized + # below so two scheduler workers cannot both act on the same stale + # state or overwrite each other's successful checkpoint. newly_notified = set() notify_failed = set() - if new_urgent: - title = "Urgent email" if total_urgent == 1 else f"{total_urgent} urgent emails" - # Build a real listing — subject · sender · reason for each urgent - # one — so the reminder email tells you which messages to act on, - # not just "4 needing reply". Optional deep-link when the user has - # `app_public_url` configured in Settings (so the email row links - # straight into the Odysseus Email tab). - # Sort: highest-scored UIDs first; cap at 10 to keep the email tidy. + + def _urgency_reminder_payload(reminder_keys): + total = len(reminder_keys) + title = "Urgent email" if total == 1 else f"{total} urgent emails" sorted_urgent = sorted( - ((k, per_uid_scores[k]) for k in urgent_keys), - key=lambda kv: kv[1].get("score", 0), reverse=True, + ((key, per_uid_scores[key]) for key in reminder_keys), + key=lambda item: item[1].get("score", 0), + reverse=True, )[:10] _pub = (settings.get("app_public_url") or "").strip().rstrip("/") from urllib.parse import quote as _quote - lines = [f"{total_urgent} email" + ("" if total_urgent == 1 else "s") + " need an urgent reply:", ""] - for i, (k, v) in enumerate(sorted_urgent, 1): - subj = (v.get("subject") or "(no subject)")[:160] - frm = v.get("from") or "" - why = v.get("reason") or "" - uid_for_link = str(k).split(":", 1)[-1] + lines = [ + f"{total} email" + ("" if total == 1 else "s") + + " need an urgent reply:", + "", + ] + for i, (key, value) in enumerate(sorted_urgent, 1): + subj = (value.get("subject") or "(no subject)")[:160] + frm = value.get("from") or "" + why = value.get("reason") or "" + uid_for_link = str(key).split(":", 1)[-1] hash_link = f"#email={_quote('INBOX', safe='')}:{uid_for_link}" open_link = f"{_pub}/{hash_link}" if _pub else hash_link line = f"{i}. {subj}" @@ -2415,57 +2955,94 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: line += f" · {why}" lines.append(line) lines.append(f" Open email: {open_link}") - if total_urgent > len(sorted_urgent): + if total > len(sorted_urgent): lines.append("") - lines.append(f"…and {total_urgent - len(sorted_urgent)} more.") - body = "\n".join(lines) - try: - # Call dispatch_reminder DIRECTLY (no HTTP/auth roundtrip — the - # endpoint version 401's the background scheduler because it - # has no session cookie). - from routes.note_routes import dispatch_reminder - dispatch_result = await dispatch_reminder( - title=title, note_body=body, note_id="urgent-email", - owner=owner or "", - ) - channel = (settings.get("reminder_channel") or "browser").strip().lower() - delivered = bool(dispatch_result.get("browser_sent")) - if channel == "email": - delivered = bool(dispatch_result.get("email_sent")) - elif channel == "ntfy": - delivered = bool(dispatch_result.get("ntfy_sent")) - elif channel == "webhook": - delivered = bool(dispatch_result.get("webhook_sent")) - if delivered: - newly_notified.update(new_urgent) - else: + lines.append(f"…and {total - len(sorted_urgent)} more.") + return title, "\n".join(lines) + + async def _dispatch_urgency_reminder(reminder_keys): + # Call dispatch_reminder directly: a scheduler has no browser + # session cookie with which to call the HTTP endpoint. + from routes.note_routes import dispatch_reminder + title, body = _urgency_reminder_payload(reminder_keys) + return await dispatch_reminder( + title=title, + note_body=body, + note_id="urgent-email", + owner=owner or "", + ) + + async def _dispatch_and_checkpoint(prior): + notified_uids = _email_urgency_string_set( + prior.get("notified_uids", []) + ) + observed_accounts = { + _email_urgency_account_key(key) for key in per_uid_scores + } | fully_scanned_account_ids + stale_accounts = _email_urgency_stale_accounts( + prior, + base_account_generations, + observed_accounts, + ) + # Generation fencing must happen before delivery, not only during + # merge. A stale-only unread UID may have been removed, read, or + # downgraded by the newer completed scan. + deliverable_urgent = [ + key + for key in urgent_keys + if _email_urgency_account_key(key) not in stale_accounts + ] + new_urgent = [ + key + for key in deliverable_urgent + if key not in notified_uids + ] + if new_urgent: + try: + dispatch_result = await _dispatch_urgency_reminder( + deliverable_urgent + ) + channel = (settings.get("reminder_channel") or "browser").strip().lower() + delivered = bool(dispatch_result.get("browser_sent")) + if channel == "email": + delivered = bool(dispatch_result.get("email_sent")) + elif channel == "ntfy": + delivered = bool(dispatch_result.get("ntfy_sent")) + elif channel == "webhook": + delivered = bool(dispatch_result.get("webhook_sent")) + if delivered: + newly_notified.update(new_urgent) + notified_uids.update(new_urgent) + else: + notify_failed.update(new_urgent) + logger.warning( + "urgency: reminder dispatch returned no successful " + f"delivery path: {dispatch_result}" + ) + except Exception as e: + logger.warning(f"urgency: reminder dispatch failed: {e}") notify_failed.update(new_urgent) - logger.warning(f"urgency: reminder dispatch returned no successful delivery path: {dispatch_result}") - except Exception as e: - logger.warning(f"urgency: reminder dispatch failed: {e}") - notify_failed.update(new_urgent) - # Mark only successfully delivered UIDs as notified so a transient - # SMTP/ntfy/browser failure retries instead of lying forever. - notified_uids.update(newly_notified) - # Prune notified_uids that aren't unread anymore (so a future re-urgent - # message with the same UID — rare but possible after archive→unarchive - # — can re-notify). Keep only UIDs still in `all_unread_keys`. - notified_uids = {u for u in notified_uids if u in all_unread_keys} + next_state = _merge_email_urgency_state( + prior, + owner=owner, + per_uid_scores=per_uid_scores, + notified_uids=notified_uids, + all_unread_keys=all_unread_keys, + fully_scanned_account_ids=fully_scanned_account_ids, + base_account_generations=base_account_generations, + timestamp=_time.time(), + ) + return notified_uids, next_state - state = { - "ts": _time.time(), - "owner": owner or "", - "total_unread": len(all_unread_keys), - "total_urgent": total_urgent, - "max_score": max_score, - "per_uid": per_uid_scores, - "notified_uids": sorted(notified_uids), - } try: - STATE_PATH.write_text(_json.dumps(state), encoding="utf-8") + await _run_email_urgency_state_transaction( + STATE_PATH, + STATE_LOCK_DB, + _dispatch_and_checkpoint, + ) except Exception as e: - logger.warning(f"urgency: state write failed: {e}") + logger.warning(f"urgency: state transaction failed: {e}") # ── 6. Activity-log summary — counts line on top, then per-tier # bulleted breakdown so the user can see WHICH emails ranked where diff --git a/src/endpoint_resolver.py b/src/endpoint_resolver.py index 71f260fa2..1bb8fc3af 100644 --- a/src/endpoint_resolver.py +++ b/src/endpoint_resolver.py @@ -443,28 +443,14 @@ def resolve_endpoint_by_id( def resolve_chat_fallback_candidates(owner: Optional[str] = None) -> list: - """Build the configured default-chat fallback chain as a list of - (chat_url, model, headers) tuples, skipping any that can't resolve. + """Compatibility shim for the retired default-chat fallback chain.""" - The primary model is NOT included — callers prepend their session's - current (url, model, headers) so per-session model overrides are honored. - """ - return _resolve_fallback_candidates("default_model_fallbacks", owner=owner) + del owner + return [] def resolve_utility_fallback_candidates(owner: Optional[str] = None) -> list: """Configured fallback chain for the Utility model (`utility_model_fallbacks`).""" - try: - from src.settings import get_user_setting, load_settings - settings = load_settings() - utility_ep = (get_user_setting("utility_endpoint_id", owner or "", settings.get("utility_endpoint_id", "")) or "").strip() - if not utility_ep: - utility_chain = get_user_setting("utility_model_fallbacks", owner or "", settings.get("utility_model_fallbacks") or []) or [] - if utility_chain: - return _resolve_fallback_candidates("utility_model_fallbacks", owner=owner) - return _resolve_fallback_candidates("default_model_fallbacks", owner=owner) - except Exception: - pass return _resolve_fallback_candidates("utility_model_fallbacks", owner=owner) diff --git a/src/foreground_model_routing.py b/src/foreground_model_routing.py new file mode 100644 index 000000000..241ccf26b --- /dev/null +++ b/src/foreground_model_routing.py @@ -0,0 +1,31 @@ +"""Foreground Chat and Agent model-routing policy. + +The selected session model is strict by default. Historical +``default_model_fallbacks`` values remain stored for compatibility, but this +policy intentionally does not read or migrate them. +""" + +from typing import Any, Dict, Optional + + +def resolve_foreground_fallback_candidates(owner: Optional[str] = None) -> list: + """Return fallback candidates for a foreground Chat or Agent request. + + Foreground routing is strict, so no alternate endpoint/model is eligible. + ``owner`` is accepted to keep this policy boundary owner-aware. + """ + + del owner + return [] + + +def build_foreground_model_candidates( + endpoint_url: str, + model: str, + headers: Optional[Dict[str, Any]] = None, + owner: Optional[str] = None, +) -> list: + """Build the ordered candidate list for a foreground request.""" + + primary = (endpoint_url, model, headers or {}) + return [primary] + resolve_foreground_fallback_candidates(owner=owner) diff --git a/src/integrations.py b/src/integrations.py index aa6c4982e..52dd4b2d1 100644 --- a/src/integrations.py +++ b/src/integrations.py @@ -1,11 +1,14 @@ +import ipaddress import json import os +import time import uuid import logging import re from typing import Dict, List, Optional, Any from urllib.parse import urljoin, urlparse, urlunparse +import httpcore import httpx from fastapi import HTTPException @@ -354,6 +357,152 @@ def _find_integration(identifier: str) -> Optional[Dict[str, Any]]: return None +# httpcore raises its own exception hierarchy; map the ones a simple request can +# surface back to their httpx equivalents so the caller's `except httpx.*` blocks +# below behave exactly as they did with the default transport. +_HTTPCORE_TO_HTTPX_EXC = { + httpcore.ConnectError: httpx.ConnectError, + httpcore.ConnectTimeout: httpx.ConnectTimeout, + httpcore.NetworkError: httpx.NetworkError, + httpcore.PoolTimeout: httpx.PoolTimeout, + httpcore.ProtocolError: httpx.ProtocolError, + httpcore.ReadError: httpx.ReadError, + httpcore.ReadTimeout: httpx.ReadTimeout, + httpcore.RemoteProtocolError: httpx.RemoteProtocolError, + httpcore.TimeoutException: httpx.TimeoutException, + httpcore.WriteError: httpx.WriteError, + httpcore.WriteTimeout: httpx.WriteTimeout, +} + + +class _PinnedAsyncBackend(httpcore.AsyncNetworkBackend): + """Network backend that connects only to the pre-validated IPs, in order. + + Every address here came out of the single SSRF resolution, so moving to the + next one after a connect failure is not re-resolution — it's ordinary + multi-address fallback restricted to the set the guard already approved. + httpcore takes TLS SNI and the ``Host`` header from the request URL rather + than the connect host, so pinning the socket destination leaves certificate + validation and vhost routing pointed at the original hostname. + """ + + def __init__(self, ips: List[ipaddress._BaseAddress]): + self._ips = [str(ip) for ip in ips] + self._real = httpcore.AnyIOBackend() + + async def connect_tcp(self, host, port, timeout=None, local_address=None, + socket_options=None): + # One shared connect budget: each attempt gets the time left until the + # original deadline, so N dead addresses can't stretch the connect + # phase to N * timeout. + deadline = None if timeout is None else time.monotonic() + timeout + last_exc: Optional[Exception] = None + for ip in self._ips: + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + try: + return await self._real.connect_tcp( + ip, port, remaining, local_address, socket_options + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_exc = exc + if deadline is not None and time.monotonic() >= deadline: + break + raise last_exc + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + return await self._real.connect_unix_socket(path, timeout, socket_options) + + async def sleep(self, seconds: float) -> None: + return await self._real.sleep(seconds) + + +class _PinnedAsyncTransport(httpx.AsyncBaseTransport): + """httpx transport that pins the TCP connect to the pre-resolved IP(s). + + Kept local, mirroring the per-module pinned transports web fetch and + webhook delivery already carry, rather than coupling api_call to the + webhook subsystem. The request URL passes through unchanged, so SNI and the + ``Host`` header stay the original hostname; only the socket destination is + pinned, which is what closes the rebinding window. + """ + + def __init__(self, ips: List[ipaddress._BaseAddress]): + self._pinned_ips = list(ips) + self._pool = httpcore.AsyncConnectionPool( + # Reuse the CA trust the default httpx client would build (certifi + # plus SSL_CERT_FILE / SSL_CERT_DIR when trust_env is set) so + # swapping in this transport doesn't quietly change which chains + # verify. ssl.create_default_context() would use system roots. + ssl_context=httpx.create_ssl_context(), + http1=True, + http2=False, + network_backend=_PinnedAsyncBackend(ips), + ) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + core_req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + try: + core_resp = await self._pool.handle_async_request(core_req) + content = b"".join([chunk async for chunk in core_resp.aiter_stream()]) + await core_resp.aclose() + except Exception as exc: + mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc)) + if mapped is not None: + raise mapped(str(exc)) from exc + raise + return httpx.Response( + status_code=core_resp.status, + headers=core_resp.headers, + content=content, + extensions=core_resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() + + +def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]: + """Return every entry that parses as an IP address, de-duplicated, order + preserved. + + check_outbound_url only reports ok when *all* of these classify as safe, so + the whole list is guard-approved and any of them is a legitimate connect + target. Skipping unparseable entries mirrors how the guard walks the same + resolver output. + + De-duplication matters because the resolver is getaddrinfo(host, None) with + no socktype filter, so glibc reports the same address once per socktype + (SOCK_STREAM/SOCK_DGRAM/SOCK_RAW) — a single-homed host comes back three + times. Without this, the connect fallback would spend the shared deadline + retrying one dead address instead of moving on to a genuinely different one. + """ + ips: List[ipaddress._BaseAddress] = [] + seen = set() + for raw in raw_ips: + if not isinstance(raw, str): + continue + try: + ip = ipaddress.ip_address(raw.split("%")[0]) # strip IPv6 zone id + except ValueError: + continue + if ip in seen: + continue + seen.add(ip) + ips.append(ip) + return ips + + async def execute_api_call( integration_id: str, method: str, @@ -409,13 +558,31 @@ async def execute_api_call( # loopback for locked-down deployments. Private stays allowed by default # because LAN integrations (Home Assistant, Miniflux, ntfy) are the # primary use case. - from src.url_safety import check_outbound_url + from src.url_safety import check_outbound_url, _default_resolver block_private = os.getenv( "INTEGRATION_API_BLOCK_PRIVATE_IPS", "false" ).lower() == "true" - ok, reason = check_outbound_url(url, block_private=block_private) + # Resolve the host exactly once and remember the IPs the guard validated so + # the request below can be pinned to them. check_outbound_url only reports + # (ok, reason); a plain httpx client re-resolves the host at connect time, + # which reopens a DNS-rebinding TOCTOU — a base_url host that answers with a + # public IP for the guard and then flips to 169.254.169.254 for the connect + # would reach cloud metadata with the integration's auth headers attached. + resolved_ips: List[str] = [] + + def _recording_resolver(host: str) -> List[str]: + ips = _default_resolver(host) + resolved_ips[:] = ips + return ips + + ok, reason = check_outbound_url( + url, block_private=block_private, resolver=_recording_resolver + ) if not ok: return {"error": f"URL rejected: {reason}", "exit_code": 1} + pinned_ips = _validated_ips(resolved_ips) + if not pinned_ips: + return {"error": "URL rejected: host did not resolve to a usable address", "exit_code": 1} method = method.upper() @@ -455,7 +622,9 @@ async def execute_api_call( auth = httpx.BasicAuth(parts[0], parts[1]) try: - async with httpx.AsyncClient(timeout=30.0) as client: + async with httpx.AsyncClient( + timeout=30.0, transport=_PinnedAsyncTransport(pinned_ips) + ) as client: response = await client.request( method, url, diff --git a/src/llm_core.py b/src/llm_core.py index 4dec32376..e69661fb7 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -1,6 +1,7 @@ # src/llm_core.py import httpx import asyncio +import copy import time import json import logging @@ -644,7 +645,7 @@ def _build_ollama_payload( if options: payload["options"] = options if tools: - payload["tools"] = tools + payload["tools"] = _alias_harmony_tools(tools, model) return payload @@ -1055,6 +1056,57 @@ def _model_disallows_reasoning_effort_with_chat_tools(model: str) -> bool: return bool(re.match(r"^(?:openai/)?gpt-5(?:[.\-]\d+)?(?:[-_:].*)?$", m)) +# gpt-oss (harmony) ships BUILT-IN tools named `python` and `browser`, invoked +# with the raw body as the argument (`to=python` + bare source), while custom +# functions use `to=functions.NAME` + JSON. A tool we expose under a built-in's +# name therefore gets called with the built-in convention: the model emits raw +# code, the server tries to parse it as JSON, and the whole request dies +# ("error parsing tool call: raw='import sys, ...'"). In streaming mode Ollama +# does not even report it — it truncates the stream, so the turn looks like an +# empty response. `bash` collides the same way in practice. +# +# Measured on gpt-oss:20b via Ollama /v1 with a fixed agentic prompt: +# tools named python+bash ............ 2/6 succeeded (4 parse failures) +# python renamed ..................... 5/6 +# python and bash renamed ............ 6/6 +# +# So rename the colliding tools on the way out and map the names back on the +# way in. Confined to the transport layer: callers keep using the real names. +_HARMONY_TOOL_ALIASES = { + "python": "run_python_code", + "bash": "run_shell_command", + "browser": "web_browser_tool", +} +_HARMONY_TOOL_ALIASES_REVERSE = {v: k for k, v in _HARMONY_TOOL_ALIASES.items()} + + +def _is_harmony_model(model: str) -> bool: + """True for gpt-oss / harmony-format models, which have built-in tool names.""" + return "gpt-oss" in (model or "").lower() + + +def _alias_harmony_tools(tools: Optional[List[Dict]], model: str) -> Optional[List[Dict]]: + """Rename tools that collide with harmony built-ins. Returns a copy.""" + if not tools or not _is_harmony_model(model): + return tools + out = [] + for t in tools: + fn = t.get("function") or {} + alias = _HARMONY_TOOL_ALIASES.get(fn.get("name")) + if alias: + t = copy.deepcopy(t) + t["function"]["name"] = alias + out.append(t) + return out + + +def _unalias_harmony_tool_name(name: str, model: str) -> str: + """Map an aliased tool name in a model response back to the real name.""" + if not _is_harmony_model(model): + return name + return _HARMONY_TOOL_ALIASES_REVERSE.get(name, name) + + def _scrub_openai_chat_tool_reasoning(payload: Dict, target_url: str, model: str) -> None: if not payload.get("tools"): return @@ -1237,15 +1289,27 @@ def _anthropic_rejects_temperature(model: str) -> bool: return False # `(?= 4.7. Dated 4.7+ snapshots (`claude-opus-4-7- - # 20260201`) keep their explicit minor and are still matched. - match = re.search(r"(?= 4.7 (issue #5753). Without + # this, every Opus 5 call kept `temperature` and failed with HTTP 400 — visible + # only on paths that pass a temperature, e.g. scheduled tasks inheriting + # `stream_agent_loop`'s 0.3 default, which returned empty responses. + match = re.search( + r"(?= (4, 7) + major = int(match.group(1)) + minor = int(match.group(2)) if match.group(2) else 0 + return (major, minor) >= (4, 7) # Reasoning effort level sent to Mistral thinking-capable models. Mistral's # API accepts "high", "medium", "low", "none" — see @@ -1255,8 +1319,8 @@ _MISTRAL_REASONING_EFFORT = os.getenv("ODYSSEUS_MISTRAL_REASONING_EFFORT", "high # Models that support structured thinking — may output without opening tag _THINKING_MODEL_PATTERNS = ( - "qwen3", "qwq", "deepseek-r1", "deepseek-reasoner", "minimax", - "m2-reap", "gemma", "stepfun", "step-3", "step3", + "qwen3", "qwq", "deepseek-r1", "deepseek-reasoner", "deepseek-v4", + "minimax", "m2-reap", "gemma", "stepfun", "step-3", "step3", "magistral", "mistral-small", "mistral-medium", ) @@ -1885,11 +1949,10 @@ def _dedupe_candidates(candidates): """Filter malformed entries and drop a later repeat of an already-seen ``(url, model)`` route, preserving order (first occurrence wins). - The chain is the primary target followed by the configured fallbacks, so a - fallback that repeats the session's current model — a common misconfiguration, - since callers prepend the live ``(url, model)`` to ``default_model_fallbacks`` - — would otherwise make the chain re-attempt the very route that just failed: - a wasted round-trip plus a spurious ``fallback`` notice for a switch that did + The chain is the primary target followed by any caller-authorized + fallbacks. A fallback that repeats the session's current model would + otherwise make the chain re-attempt the very route that just failed: a + wasted round-trip plus a spurious ``fallback`` notice for a switch that did not happen. Headers are not part of the key; the first tuple (with its headers) is the one kept. """ @@ -2097,7 +2160,17 @@ async def llm_call_async( response = _parse_ollama_response(data) else: msg = data["choices"][0]["message"] - response = msg.get("content") or msg.get("reasoning_content") or "" + content = msg.get("content") + if isinstance(content, list): + # Mistral structured content — extract thinking + text + # (same contract as llm_call / stream_llm; see #5435). + text_part, thinking_part = _normalize_mistral_content(content) + if thinking_part: + response = thinking_part + "\n\n" + (text_part or "") + else: + response = text_part or msg.get("reasoning_content") or "" + else: + response = content or msg.get("reasoning_content") or "" _set_cached_response(cache_key, response) return response except Exception: @@ -2214,7 +2287,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" payload[tok_key] = max_tokens if tools: - payload["tools"] = tools + payload["tools"] = _alias_harmony_tools(tools, model) elif tool_choice_none: payload["tool_choice"] = "none" # Mistral thinking-capable models — send reasoning_effort so Mistral @@ -2348,7 +2421,7 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat if fn.get("name"): _ollama_tool_calls.append({ "id": tc.get("id") or f"call_{len(_ollama_tool_calls)}", - "name": fn.get("name") or "", + "name": _unalias_harmony_tool_name(fn.get("name") or "", model), "arguments": json.dumps(fn.get("arguments") or {}), }) if j.get("done"): @@ -2728,7 +2801,10 @@ async def _stream_llm_inner(url: str, model: str, messages: List[Dict], temperat if tc.get("extra_content"): _tc_acc[idx]["extra_content"] = tc["extra_content"] if func.get("name"): - _tc_acc[idx]["name"] = func["name"] + # Map harmony aliases back to real + # tool names before anything + # downstream sees them. + _tc_acc[idx]["name"] = _unalias_harmony_tool_name(func["name"], model) if "arguments" in func: # Guard against a null arguments delta: `func` can be # {"arguments": None} (JSON null), and a raw `+= None` diff --git a/src/memory.py b/src/memory.py index 1d8cdbc1e..92efbf5b2 100644 --- a/src/memory.py +++ b/src/memory.py @@ -10,6 +10,18 @@ from datetime import datetime logger = logging.getLogger(__name__) + +class MemoryStoreUnreadable(RuntimeError): + """memory.json exists on disk but could not be read or parsed. + + "The contents are unknown" is categorically different from "there are no + memories". A read-modify-write caller that conflates the two appends to an + empty view and then persists it, destroying the whole store — the writes + are atomic, so the loss is durable. Raised by + :meth:`MemoryManager.load_all_for_update` so those callers fail closed. + """ + + def tokenize(text: str) -> List[str]: """Simple tokenizer that splits on whitespace and removes punctuation.""" return [word.strip('.,!?";') for word in text.split()] @@ -110,21 +122,69 @@ class MemoryManager: with open(self.memory_file, 'w', encoding='utf-8') as f: json.dump([], f, ensure_ascii=False, indent=2) - def load_all(self) -> List[Dict]: - """Load all memory entries from JSON file (unfiltered).""" + def _read_entries(self) -> List[Dict]: + """Parse the store, or raise :class:`MemoryStoreUnreadable`. + + Returns ``[]`` only when the file genuinely does not exist. Every other + failure mode raises, so callers can tell "no memories" apart from + "couldn't read the memories". + """ if not os.path.exists(self.memory_file): return [] try: with open(self.memory_file, "r", encoding="utf-8") as f: data = json.load(f) - if isinstance(data, list): - return self._validate_entries(data) - except (json.JSONDecodeError, PermissionError) as e: - logger.error("Error loading memory.json: %s", e) - return self._migrate_from_legacy() + except OSError as e: + # PermissionError is an OSError (a scanner holding the file, a + # permissions problem, bad media). + raise MemoryStoreUnreadable( + f"cannot read {self.memory_file}: {e}" + ) from e + except json.JSONDecodeError as e: + # This is the branch that actually destroyed stores: the file reads + # back fine, so nothing stops the save that follows. A truncated + # memory.json is reachable because core/database.py rewrites it with + # a plain open(..,"w") + json.dump during migration. + # + # Preserved behaviour: a corrupt store still gets one shot at the + # pre-JSON memory.txt migration. Only raise when that finds nothing, + # so we never report "empty" for a store we simply failed to parse. + legacy = self._migrate_from_legacy() + if legacy: + return legacy + raise MemoryStoreUnreadable( + f"{self.memory_file} is not valid JSON: {e}" + ) from e - return [] + if not isinstance(data, list): + raise MemoryStoreUnreadable( + f"{self.memory_file} is not a JSON array (got {type(data).__name__})" + ) + return self._validate_entries(data) + + def load_all(self) -> List[Dict]: + """Load all memory entries from JSON file (unfiltered). + + Lenient by design: this feeds display, search, and context-injection + paths, so an unreadable store degrades to an empty list rather than + breaking chat. Never build a value from this that you intend to save + back — use :meth:`load_all_for_update` for that. + """ + try: + return self._read_entries() + except MemoryStoreUnreadable as e: + logger.error("Error loading memory.json: %s", e) + return [] + + def load_all_for_update(self) -> List[Dict]: + """Load for a read-modify-write cycle. + + Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]`` + so a caller can never append to an empty view and persist it over a + store that was only temporarily unreadable (issue #5673). + """ + return self._read_entries() def load(self, owner: str = None) -> List[Dict]: """Load memory entries, optionally filtered by owner.""" @@ -135,7 +195,12 @@ class MemoryManager: def claim_ownerless(self, owner: str): """Assign all ownerless memory entries to the given owner.""" - entries = self.load_all() + try: + entries = self.load_all_for_update() + except MemoryStoreUnreadable as e: + # Skip the sweep rather than rewrite the store from an unknown view. + logger.error("Skipping ownerless claim, memory store unreadable: %s", e) + return changed = False claimed = 0 for entry in entries: @@ -235,7 +300,12 @@ class MemoryManager: if not ids: return id_set = set(ids) - entries = self.load_all() + try: + entries = self.load_all_for_update() + except MemoryStoreUnreadable as e: + # Best-effort counter; never worth rewriting the store blind. + logger.error("Skipping uses bump, memory store unreadable: %s", e) + return changed = False for e in entries: if e.get("id") in id_set: diff --git a/src/memory_provider.py b/src/memory_provider.py index 925c59192..8974a6e84 100644 --- a/src/memory_provider.py +++ b/src/memory_provider.py @@ -157,7 +157,11 @@ class NativeMemoryProvider(MemoryProvider): if metadata: entry["metadata"] = dict(metadata) - memories = self.memory_manager.load_all() + # Strict load: read-modify-write. `load_all` degrades an unreadable + # store to [], which would save this single entry over everything + # already stored (issue #5673). The provider API has no error channel, + # so MemoryStoreUnreadable propagates to the caller. + memories = self.memory_manager.load_all_for_update() memories.append(entry) self.memory_manager.save(memories) @@ -223,7 +227,10 @@ class NativeMemoryProvider(MemoryProvider): ] async def delete(self, memory_id: str, *, owner: Optional[str] = None) -> bool: - memories = self.memory_manager.load_all() + # Strict load for the same reason: `remaining` is derived from this + # list and saved back, so it must never be built from a store we + # failed to read. + memories = self.memory_manager.load_all_for_update() remaining = [] deleted_id = None diff --git a/src/settings.py b/src/settings.py index 5836765f1..da08717d5 100644 --- a/src/settings.py +++ b/src/settings.py @@ -138,14 +138,13 @@ DEFAULT_SETTINGS = { # Email replies use email_writing_style instead because greetings, # signatures, and mailbox identity rules are medium-specific. "document_writing_style": "", - # Ordered fallback chain for the default chat model. Each entry is - # {"endpoint_id": "...", "model": "..."}. If the primary model fails - # before producing output (endpoint offline / errors), the chat - # dispatch retries the next entry in order. + # Legacy ordered fallback chain for the default chat model. Values remain + # stored for compatibility and rollback reference, but model routing no + # longer reads this key. "default_model_fallbacks": [], - # When True, non-admin users inherit global default model/endpoint/fallbacks - # when they have no personal defaults. When False, users only use their - # personal defaults (no global fallback). Default is False. + # When True, non-admin users inherit the global default model/endpoint when + # they have no personal defaults. When False, users only use their personal + # defaults. Default is False. "share_defaults_with_users": False, "utility_endpoint_id": "", "utility_model": "", diff --git a/src/task_endpoint.py b/src/task_endpoint.py index b9c290d65..ae57a81f7 100644 --- a/src/task_endpoint.py +++ b/src/task_endpoint.py @@ -32,7 +32,7 @@ def resolve_task_candidates( 2. Utility endpoint/model 3. Default endpoint/model 4. Utility fallback chain - 5. Default fallback chain + 5. Retired default-fallback compatibility hook (currently empty) """ candidates = [] diff --git a/src/teacher_escalation.py b/src/teacher_escalation.py index 49134991c..4b6206a8d 100644 --- a/src/teacher_escalation.py +++ b/src/teacher_escalation.py @@ -233,7 +233,8 @@ async def _call_teacher(teacher_model_spec: str, prompt: str, owner: Optional[str] = None) -> Optional[str]: """Call the configured teacher endpoint with the escalation prompt.""" from src.llm_core import llm_call_async - from src.ai_interaction import _resolve_model, _TEACHER_SYSTEM_PROMPT + from src.ai_interaction import _resolve_model + from src.agent_tools.model_interaction_tools import _TEACHER_SYSTEM_PROMPT try: url, model, headers = await asyncio.to_thread(_resolve_model, teacher_model_spec, owner=owner) except Exception as e: diff --git a/src/tool_parsing.py b/src/tool_parsing.py index 2885cc00f..b13f3b0a1 100644 --- a/src/tool_parsing.py +++ b/src/tool_parsing.py @@ -187,9 +187,13 @@ _FUNCTION_MODEL_NAME_RE = re.compile( _FUNCTION_MODEL_PARAMS_OPEN_RE = re.compile(r"\s*", re.IGNORECASE) _FUNCTION_MODEL_PARAMS_CLOSE_RE = re.compile(r"", re.IGNORECASE) _QWEN_ROLE_MARKER_RE = re.compile(r"?|?", re.IGNORECASE) +# At least one pipe is required around `end`. Both pipes used to be optional +# (`\|?end\|?`), which also matched a bare `end` on its own line and deleted it +# from ordinary prose and from Ruby/Lua/shell snippets that close blocks with +# one; see #5547. `|end`, `end|`, `|end|` and `/|end|` still strip as before. _QWEN_BARE_MARKER_RE = re.compile( - r"(?:^|[\t\r\n ])(?:\|?end\|?|/?\|end\|)(?=[\t\r\n ]|$)|" - r"(?:^|[\t\r\n ])assistan(?:t)?(?=[\t\r\n ]|$)", + r"(?:^|[\t\r\n ])(?:/?\|end\||\|end|end\|)(?=[\t\r\n ]|$)|" + r"(?:^|[\r\n])[ \t]*assistan(?:t)?[ \t]*(?=[\r\n]|$)", re.IGNORECASE, ) @@ -925,6 +929,46 @@ def _parse_xml_direct_tool(name, body) -> Optional[ToolBlock]: return function_call_to_tool_block(mapped, json.dumps(params)) +def _looks_like_json_body(body: str) -> bool: + """True when a wrapper body is JSON, not XML markup.""" + return body.lstrip()[:1] in ("{", "[") + + +def _parse_json_tool_call_body(body: str) -> Optional[ToolBlock]: + """Parse a Qwen/Hermes text-mode wrapper body: bare JSON inside . + + + {"name": "bash", "arguments": {"command": "mkdir -p agent-test"}} + + + Strict by design (issue #5187 / tracker #5333): the body must decode to an + object with a string "name", and "arguments" — when present — must itself + be an object. Anything else returns None rather than being coerced, so a + malformed call is dropped instead of dispatching with mangled arguments. + raw_decode tolerates trailing chatter after the JSON object; the trailing + text is never scanned for tool markup. Conversion goes through + function_call_to_tool_block so aliases and per-tool argument formatting + stay identical to the XML invoke path. + """ + stripped = body.strip() + if not stripped.startswith("{"): + return None + try: + parsed, _end = json.JSONDecoder().raw_decode(stripped) + except json.JSONDecodeError: + return None + if not isinstance(parsed, dict): + return None + name = parsed.get("name") + if not isinstance(name, str) or not name.strip(): + return None + if "arguments" in parsed and not isinstance(parsed["arguments"], dict): + return None + args = parsed.get("arguments", {}) + from src.tool_schemas import function_call_to_tool_block + return function_call_to_tool_block(name.strip().lower(), json.dumps(args)) + + def _iter_stepfun_tool_calls(text: str): """Yield StepFun native tool-call token bodies without regex backtracking.""" pos = 0 @@ -1326,10 +1370,21 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: if blocks: return blocks # Try wrapped: ... + # A wrapper body that is JSON (Qwen/Hermes text mode, issue #5187) is + # parsed as JSON or dropped — never scanned by the XML iterators, so + # XML-like text inside JSON argument values stays data instead of + # selecting a different tool. + json_body_seen = False for _ms, inner_start, inner_end, _me in _iter_delimited( text, _XML_TOOL_CALL_OPEN_RE, _XML_TOOL_CALL_CLOSE_RE ): body = text[inner_start:inner_end] + if _looks_like_json_body(body): + json_body_seen = True + block = _parse_json_tool_call_body(body) + if block: + blocks.append(block) + continue for inv_name, inv_body in _iter_xml_invoke(body): block = _parse_xml_invoke(inv_name, inv_body) if block: @@ -1344,6 +1399,13 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: if not blocks: for m in _XML_OPEN_TOOL_CALL_RE.finditer(text): body = m.group(1) + if _looks_like_json_body(body): + # Same fail-closed rule as above for an unclosed wrapper. + json_body_seen = True + block = _parse_json_tool_call_body(body) + if block: + blocks.append(block) + break for inv_name, inv_body in _iter_xml_invoke(body): block = _parse_xml_invoke(inv_name, inv_body) if block: @@ -1354,8 +1416,11 @@ def parse_tool_blocks(text: str, skip_fenced: bool = False) -> List[ToolBlock]: block = _parse_xml_direct_tool(d_name, d_body) if block: blocks.append(block) - # Try bare without wrapper - if not blocks: + # Try bare without wrapper. Skipped when a JSON wrapper body + # was seen but produced no block: this rescan covers the full text, + # wrapper bodies included, and markup inside a (possibly + # malformed) JSON payload must stay data rather than dispatch. + if not blocks and not json_body_seen: for inv_name, inv_body in _iter_xml_invoke(text): block = _parse_xml_invoke(inv_name, inv_body) if block: diff --git a/src/tools/calendar.py b/src/tools/calendar.py index e6572ba40..6dda5a0e3 100644 --- a/src/tools/calendar.py +++ b/src/tools/calendar.py @@ -196,6 +196,9 @@ async def do_manage_calendar(content: str, owner: Optional[str] = None) -> Dict: try: if action == "list_calendars": _ensure_default_calendar(db, owner) + # This read path intentionally persists the lazily-created default; + # event creation commits it in the event's transaction instead. + db.commit() cals = _calendar_query().all() result = [{"name": c.name, "href": c.id} for c in cals] if result: diff --git a/src/tools/system.py b/src/tools/system.py index 813d57df2..c2eb9ceab 100644 --- a/src/tools/system.py +++ b/src/tools/system.py @@ -46,7 +46,9 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict: except ValueError: return {"error": "Invalid JSON arguments", "exit_code": 1} - action = (args.get("action") or "").lower() + action = (args.get("action") or "").strip().lower() + if not action: + return {"error": "action is required (list|view|view_ref|add|edit|patch|publish|delete|search)", "exit_code": 1} from services.memory.skills import SkillsManager from services.memory.skill_format import Skill, slugify from src.constants import DATA_DIR @@ -55,7 +57,7 @@ async def do_manage_skills(content: str, owner: Optional[str] = None) -> Dict: # Accept legacy `skill_id` as an alias for `name`. name = (args.get("name") or args.get("skill_id") or "").strip() - if action in ("list", "index", ""): + if action in ("list", "index"): all_skills = sm.load(owner=owner) if not all_skills: return {"results": "No skills yet. Create one with action='add'."} diff --git a/src/upload_handler.py b/src/upload_handler.py index ce0b4b129..e2907699d 100644 --- a/src/upload_handler.py +++ b/src/upload_handler.py @@ -35,6 +35,16 @@ import logging logger = logging.getLogger(__name__) +UploadIndexFileSignature = tuple[ + str, + Optional[int], + Optional[int], + Optional[int], + Optional[int], + Optional[int], +] +UploadIndexSignature = tuple[UploadIndexFileSignature, ...] + class UploadCleanupSafetyError(RuntimeError): """Raised when cleanup cannot prove that destructive work is safe.""" @@ -242,7 +252,7 @@ class UploadHandler: # In-memory index cache to avoid O(N) disk I/O on every request self._index_cache: Optional[Dict[str, Any]] = None - self._index_mtime: float = 0.0 + self._index_signature: Optional[UploadIndexSignature] = None def inside_base_dir(self, path: str) -> bool: """Check if path is inside base directory""" @@ -727,62 +737,119 @@ class UploadHandler: # Update cache if this is the main index if path.endswith("uploads.json"): self._index_cache = data + self._index_signature = self._upload_index_signature( + (path, path + ".bak") + ) + + @staticmethod + def _upload_index_signature( + paths: tuple[str, ...], + ) -> Optional[UploadIndexSignature]: + """Return file identities strong enough to validate the index cache. + + Modification time alone is insufficient: a torn write can change a + file without receiving a strictly newer timestamp on some filesystems. + Size, inode, and nanosecond change times make those mutations visible + while preserving the cache fast path for unchanged files. + """ + signature: list[UploadIndexFileSignature] = [] + for candidate in paths: try: - self._index_mtime = os.path.getmtime(path) + stat_result = os.stat(candidate) + except FileNotFoundError: + signature.append((candidate, None, None, None, None, None)) + continue except OSError: - self._index_mtime = time.time() + return None + signature.append( + ( + candidate, + stat_result.st_dev, + stat_result.st_ino, + stat_result.st_size, + stat_result.st_mtime_ns, + stat_result.st_ctime_ns, + ) + ) + return tuple(signature) def _load_upload_index(self, *, fail_on_error: bool = False) -> Dict[str, Any]: - """Load the upload index from disk/cache. Uses mtime-based validation - to avoid redundant parsing on hot paths. When ``fail_on_error`` is - true, a missing, malformed, or unreadable live index raises so - destructive callers cannot mistake corruption for an empty store. + """Load the upload index from disk/cache. Uses file-identity validation + to avoid redundant parsing on hot paths without missing same-timestamp + mutations. When ``fail_on_error`` is true, a missing, malformed, or + unreadable live index raises so destructive callers cannot mistake + corruption for an empty store. """ uploads_db_path = os.path.join(self.upload_dir, "uploads.json") candidates = (uploads_db_path, uploads_db_path + ".bak") - if fail_on_error: - # A backup is intentionally the previous snapshot. It is useful for - # non-destructive reads, but cannot authorize deletion when the live - # index is missing or corrupt. - if not os.path.exists(uploads_db_path): - raise ValueError("live uploads database is missing") - existing_candidates = [uploads_db_path] - else: - existing_candidates = [path for path in candidates if os.path.exists(path)] - if not existing_candidates: - self._index_cache = {} - self._index_mtime = 0.0 - return {} + for _attempt in range(3): + signature = self._upload_index_signature(candidates) + if fail_on_error: + # A backup is intentionally the previous snapshot. It is useful for + # non-destructive reads, but cannot authorize deletion when the live + # index is missing or corrupt. + if not os.path.exists(uploads_db_path): + raise ValueError("live uploads database is missing") + existing_candidates = [uploads_db_path] + else: + existing_candidates = [ + path for path in candidates if os.path.exists(path) + ] + if not existing_candidates: + self._index_cache = {} + self._index_signature = signature + return {} - # Check cache validity - try: - mtime = max(os.path.getmtime(path) for path in existing_candidates) + # Check cache validity if ( not fail_on_error + and signature is not None and self._index_cache is not None - and mtime <= self._index_mtime + and signature == self._index_signature ): return self._index_cache - except OSError: - mtime = 0.0 - # Try the live file first, fall back to the .bak sibling if the - # live file is truncated/corrupted. - for candidate in existing_candidates: - try: - with open(candidate, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, dict): - self._index_cache = data - self._index_mtime = mtime - return data - except Exception as e: - logger.warning(f"Failed to read uploads database ({candidate}): {e}") + # Try the live file first, fall back to the .bak sibling if the + # live file is truncated/corrupted. A candidate parsed from an old + # inode is accepted only when the whole index signature stays + # stable through the read; otherwise retry so the cache cannot pair + # stale data with a fresh replacement signature. + index_changed_during_read = False + for candidate in existing_candidates: + try: + with open(candidate, "r", encoding="utf-8") as f: + data = json.load(f) + verified_signature = self._upload_index_signature(candidates) + if ( + signature is not None + and verified_signature is not None + and verified_signature != signature + ): + index_changed_during_read = True + break + if isinstance(data, dict): + self._index_cache = data + self._index_signature = verified_signature + return data + except Exception as e: + logger.warning(f"Failed to read uploads database ({candidate}): {e}") + verified_signature = self._upload_index_signature(candidates) + if ( + signature is not None + and verified_signature is not None + and verified_signature != signature + ): + index_changed_during_read = True + break + continue + if index_changed_during_read: continue + break if fail_on_error: raise ValueError("live uploads database is unreadable") self._index_cache = {} + self._index_signature = self._upload_index_signature(candidates) return {} def get_upload_info(self, upload_id: str) -> Optional[Dict[str, Any]]: diff --git a/static/app.js b/static/app.js index 97f0ae77e..dae6b31a6 100644 --- a/static/app.js +++ b/static/app.js @@ -10,18 +10,25 @@ import modelsModule from './js/models.js?v=20260715startupcalm2'; import ragModule from './js/rag.js'; import presetsModule from './js/presets.js'; import searchModule from './js/search.js'; -import chatModule from './js/chat.js?v=20260722ctxheader4'; +import chatModule from './js/chat.js?v=20260801fix1'; import compareModule from './js/compare/index.js?v=20260723compareicon2'; import documentModule from './js/document.js?v=20260722emailfastindex1'; import searchChatModule from './js/search-chat.js'; import { makeWindowDraggable } from './js/windowDrag.js'; +import { + revealApplicationShellAfterPaint, + runDeferredRouteOpener, + deferRouteOpener, + settleSessionHydration +} from './js/startupShell.js'; import markdownModule from './js/markdown.js'; import chatRenderer from './js/chatRenderer.js?v=20260722emailfastindex1'; -import sessionModule from './js/sessions.js?v=20260722ctxheader4'; +import sessionModule from './js/sessions.js'; import memoryModule from './js/memory.js?v=20260722memoryloading1'; import voiceRecorderModule from './js/voiceRecorder.js'; import censorModule from './js/censor.js'; import galleryModule from './js/gallery.js'; +import { UI_VIS_DEFAULT_OFF, resolveVisibility } from './js/ui_visibility.js'; import tasksModule from './js/tasks.js?v=20260723tasksbulkfeedback1'; import calendarModule from './js/calendar.js'; import notesModule from './js/notes.js'; @@ -1217,12 +1224,13 @@ function initializeEventListeners() { '/library': () => sessionModule && sessionModule.openLibrary && sessionModule.openLibrary(), }; const _opener = _routeOpen[urlPath]; - // Defer the opener — at this point in init, the modules whose handlers - // we trigger (#rail-new-session click handler, the email-section header - // click handler in emailInbox, sessionModule's loaded session list) are - // still being wired up further down in this same function. Stash the - // opener so it runs from sessionModule.loadSessions().finally() below. - if (_opener) window._odysseusRouteOpener = _opener; + // Defer the opener — at this point in init, the modules whose handlers we + // trigger (#rail-new-session click handler, the email-section header click + // handler in emailInbox, sessionModule) are still being wired up further + // down in this same function. startupShell decides when it can run: as soon + // as wiring completes, or — for the routes that read the session list — + // once /api/sessions has settled. + deferRouteOpener(urlPath, _opener); // Archive browser tool button const toolLibraryBtn = el('tool-library-btn'); @@ -1689,12 +1697,20 @@ function initializeEventListeners() { const newMemoryInput = el('new-memory-input'); if (newMemoryInput) { - newMemoryInput.addEventListener('keypress', (e) => { - if (e.key === 'Enter') { + // keydown, not the deprecated keypress: keypress is not guaranteed to + // fire for Enter everywhere, which left the Add Memory form with no + // working submit path (#5828). + newMemoryInput.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && !e.isComposing) { + e.preventDefault(); memoryModule.addNewMemory(); } }); } + const newMemoryAddBtn = el('new-memory-add-btn'); + if (newMemoryAddBtn) { + newMemoryAddBtn.addEventListener('click', () => memoryModule.addNewMemory()); + } // Voice recording is handled by the dual-purpose send/mic button (see below) @@ -2710,46 +2726,6 @@ function initializeEventListeners() { // ── UI Visibility (Customize UI modal) ── const UI_VIS_KEY = 'odysseus-ui-visibility'; - // Selector map: key → CSS selector(s) for targets - const UI_VIS_MAP = { - 'sidebar-brand': '.sidebar-brand-title', - 'sidebar-new-chat': '#sidebar-new-chat-btn', - 'sidebar-search': '#sidebar-search-btn', - 'sessions-section': '#sessions-section', - 'email-section': '#email-section', - 'tools-section': '#tools-section', - // Per-tool visibility — fine-grained control over which entries show - // inside the Tools section in the sidebar. - 'tool-calendar': '#tool-calendar-btn', - 'tool-compare': '#tool-compare-btn', - 'tool-cookbook': '#tool-cookbook-btn', - 'tool-research': '#tool-research-btn', - 'tool-gallery': '#tool-gallery-btn', - 'tool-library': '#tool-library-btn', - 'tool-memory': '#tool-memory-btn', - 'tool-notes': '#tool-notes-btn', - 'tool-tasks': '#tool-tasks-btn', - 'tool-theme': '#tool-theme-btn', - 'user-bar': '#user-bar-profile', - 'sidebar-settings-btn':'#user-bar-settings', - 'chat-meta': '.chat-meta-overlay', - 'welcome-text': '.welcome-name, .welcome-sub, #welcome-tip', - 'incognito-btn': '.incognito-btn', - 'web-toggle-btn': '#web-toggle-btn', - 'doc-toggle-btn': '#overflow-doc-btn', - 'rag-toggle-btn': '#overflow-rag-btn', - 'bash-toggle-btn': '#bash-toggle-btn', - 'overflow-plus-btn': '.overflow-wrapper', - 'mode-toggle': '.mode-toggle', - 'preset-mini-btn': '#overflow-preset-btn', - 'attach-btn': '#overflow-attach-btn', - 'research-btn': '#overflow-research-btn', - 'rail-new-chat': '#rail-new-session', - }; - - // Keys hidden by default on first run (no localStorage yet) - const UI_VIS_DEFAULT_OFF = new Set(['rag-toggle-btn', 'text-emojis', 'chat-fullwidth']); - // Keys that need admin to toggle off (reserved for future use) const UI_VIS_ADMIN_ONLY = new Set([]); @@ -2762,14 +2738,14 @@ function initializeEventListeners() { } function applyUIVis(state) { - Object.entries(UI_VIS_MAP).forEach(([key, selector]) => { - // section-drag-reorder uses a body class instead of inline styles - if (key === 'section-drag-reorder') return; - const visible = key in state ? state[key] !== false : !UI_VIS_DEFAULT_OFF.has(key); + // resolveVisibility computes selector→visible (pure; ui_visibility.js), + // including the tools-section parent rule that hides every tool rail + // launcher when Tools is off. Apply the result to the DOM here. + for (const [selector, visible] of Object.entries(resolveVisibility(state))) { document.querySelectorAll(selector).forEach(el => { el.style.display = visible ? '' : 'none'; }); - }); + } // Drag reorder: use body class so dynamically created handles are covered const dragEnabled = state['section-drag-reorder'] === true; document.body.classList.toggle('rearrange-mode', dragEnabled); @@ -3908,85 +3884,10 @@ function startOdysseusApp() { const messageInput = el('message'); const modelPickerWrap = document.getElementById('model-picker-wrap'); - function _readComposerPromptHistory() { - const chatBox = document.getElementById('chat-history'); - if (!chatBox) return []; - return Array.from(chatBox.querySelectorAll('.msg-user')) - .reverse() - .map(msg => { - const body = msg.querySelector('.body'); - return msg.dataset?.raw || (body ? body.textContent : '') || ''; - }) - .filter(Boolean); - } - - if (messageInput && !messageInput._odysseusPromptRecallCapture) { - messageInput._odysseusPromptRecallCapture = true; - let recallHistory = []; - let recallIndex = -1; - let lastRecalled = ''; - const norm = (v) => String(v || '').replace(/\r\n/g, '\n').trimEnd(); - messageInput.addEventListener('input', () => { - if (norm(messageInput.value) === norm(lastRecalled)) return; - recallHistory = []; - recallIndex = -1; - lastRecalled = ''; - try { delete messageInput.dataset.odysseusRecallIndex; } catch {} - }, true); - messageInput.addEventListener('keydown', (e) => { - if (e.key !== 'ArrowUp' && e.key !== 'ArrowDown') return; - if (e.shiftKey || e.altKey || e.ctrlKey || e.metaKey || e.isComposing) return; - if (window._ghostAutocomplete?.isActive?.()) return; - const fresh = _readComposerPromptHistory(); - const history = fresh.length ? fresh : recallHistory; - if (!history.length) return; - const current = norm(messageInput.value); - let currentIndex = current ? history.findIndex(item => norm(item) === current) : -1; - if (current && currentIndex < 0 && current === norm(lastRecalled)) currentIndex = recallIndex; - if (current && currentIndex < 0) { - const markedIndex = Number(messageInput.dataset.odysseusRecallIndex); - if (Number.isInteger(markedIndex) && markedIndex >= 0 && markedIndex < history.length) { - currentIndex = markedIndex; - } - } - e.preventDefault(); - e.stopPropagation(); - e.stopImmediatePropagation(); - if (e.key === 'ArrowDown') { - if (currentIndex < 0) return; - const nextIndex = currentIndex - 1; - if (nextIndex < 0) { - recallHistory = history; - recallIndex = -1; - lastRecalled = ''; - try { delete messageInput.dataset.odysseusRecallIndex; } catch {} - messageInput.value = ''; - try { messageInput.selectionStart = messageInput.selectionEnd = 0; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - return; - } - const recalled = history[nextIndex]; - recallHistory = history; - recallIndex = nextIndex; - lastRecalled = recalled; - try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {} - messageInput.value = recalled; - try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - return; - } - const nextIndex = currentIndex >= 0 ? Math.min(currentIndex + 1, history.length - 1) : 0; - const recalled = history[nextIndex]; - if (!recalled) return; - recallHistory = history; - recallIndex = nextIndex; - lastRecalled = recalled; - try { messageInput.dataset.odysseusRecallIndex = String(nextIndex); } catch {} - messageInput.value = recalled; - try { messageInput.selectionStart = messageInput.selectionEnd = recalled.length; } catch {} - try { uiModule.autoResize(messageInput); } catch {} - }, true); - } + // ArrowUp/ArrowDown prompt recall on #message lives in + // static/js/composerArrowUpRecall.js (wired from chat.js). Do not re-add a + // copy here: two capture-phase listeners on the same textarea meant the one + // without the draft guard won and ate unsent multi-line prompts (#5862). const _sendIcon = ''; const _micIcon = ''; @@ -4382,6 +4283,10 @@ function startOdysseusApp() { // Load initial data presetsModule.loadPresets(uiModule.showError); + // Core wiring is complete for this turn — reveal the shell independently of + // the session-list request. + revealApplicationShellAfterPaint(); + if (sessionModule) { sessionModule.initDependencies({ API_BASE: API_BASE, @@ -4393,21 +4298,19 @@ function startOdysseusApp() { scrollHistory: uiModule.scrollHistoryInstant }); - // Load sessions first (critical path) — remove loader when done - sessionModule.loadSessions() - .catch(e => console.warn('loadSessions error:', e)) - .finally(() => { - const loader = document.getElementById('app-loader'); - if (loader) { loader.style.opacity = '0'; setTimeout(() => loader.remove(), 300); } - // Fire any URL route opener now that sessions + module wiring are - // ready. Deferred from up top of init for exactly this reason. - if (window._odysseusRouteOpener) { - try { window._odysseusRouteOpener(); } catch (_) {} - window._odysseusRouteOpener = null; - } - }); + // sessionModule is now wired, so every route opener has the modules it + // drives. The ones that read no session data open here rather than + // queueing behind /api/sessions. + runDeferredRouteOpener(); + + // The shell is already usable at this point; session hydration is + // sidebar-local and settles on its own schedule. + settleSessionHydration(() => sessionModule.loadSessions()); } else { console.error('Session module not loaded!'); + // Nothing will hydrate. Settle immediately so the sidebar exposes the + // failure; session-dependent routes must remain unopened without data. + settleSessionHydration(null); } const runNonCriticalStartup = (fn, delay = 4000) => { diff --git a/static/index.html b/static/index.html index 8257660fe..8dc076173 100644 --- a/static/index.html +++ b/static/index.html @@ -248,11 +248,20 @@ }, { once: true }); })(); - - - + + + + + + - + @@ -286,7 +295,13 @@ if(!document.getElementById('app-loader')){clearInterval(iv);return} render(); },150); - setTimeout(function(){var l=document.getElementById('app-loader');if(l){l.style.opacity='0';setTimeout(function(){l.remove()},300)}},5000); + // startupShell.js hides the loader as soon as the shell is wired; it calls + // back here to stop the wave because this interval is owned by this script. + window.__odysseusLoaderWaveStop=function(){clearInterval(iv)}; + // Last-resort fallback for a boot that never reaches app.js at all. Must + // still REMOVE the node: sessions.js reads its presence as "startup in + // progress" and stops clearing the composer while it is around. + setTimeout(function(){var l=document.getElementById('app-loader');if(l){clearInterval(iv);l.style.opacity='0';setTimeout(function(){l.remove()},300)}},5000); })(); @@ -365,6 +380,7 @@ Add a memory — e.g. 'I prefer concise replies' +
@@ -812,7 +828,13 @@
-
+
+ +
+ Loading chats… +
+
-
+