mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-10-22 06:42:20 +02:00
Compare commits
2
Commits
d16b849c3e
...
df2fad2881
| Author | SHA1 | Date | |
|---|---|---|---|
|
df2fad2881 | ||
|
Astarte
|
f13e54de6e |
+112
-5
@@ -3,13 +3,16 @@ import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import unquote, urlparse
|
||||
from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.engine import Engine, make_url
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
from sqlalchemy.ext.declarative import declarative_base, declared_attr
|
||||
from sqlalchemy.orm import relationship, sessionmaker, backref
|
||||
|
||||
from src.runtime_paths import get_app_root
|
||||
from core.platform_compat import safe_chmod, IS_WINDOWS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,12 +45,28 @@ def _default_database_url() -> str:
|
||||
|
||||
|
||||
def _normalize_sqlite_url(url: str) -> str:
|
||||
if not url.startswith("sqlite:///"):
|
||||
"""Resolve relative ordinary SQLite paths without rewriting URI filenames."""
|
||||
try:
|
||||
parsed = make_url(url)
|
||||
except Exception:
|
||||
return url
|
||||
db_path = url.replace("sqlite:///", "", 1)
|
||||
if db_path == ":memory:" or os.path.isabs(db_path):
|
||||
|
||||
if parsed.get_backend_name() != "sqlite":
|
||||
return url
|
||||
return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}"
|
||||
|
||||
db_path = parsed.database
|
||||
if (
|
||||
not db_path
|
||||
or db_path == ":memory:"
|
||||
or str(db_path).lower().startswith("file:")
|
||||
or os.path.isabs(str(db_path))
|
||||
):
|
||||
return url
|
||||
|
||||
absolute_path = (Path(get_app_root()) / str(db_path)).resolve().as_posix()
|
||||
return parsed.set(database=absolute_path).render_as_string(
|
||||
hide_password=False
|
||||
)
|
||||
|
||||
|
||||
# Get database URL from environment, default to SQLite in DATA_DIR
|
||||
@@ -59,6 +78,59 @@ engine = create_engine(
|
||||
connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {}
|
||||
)
|
||||
|
||||
|
||||
# Sidecar files SQLite can create next to the main DB. -journal is the default
|
||||
# rollback journal; -wal/-shm appear once WAL is enabled. Each can hold copies of
|
||||
# secret-bearing pages, so they get the same 0o600 lockdown as the DB itself.
|
||||
_SQLITE_SIDECARS = ("-journal", "-wal", "-shm")
|
||||
|
||||
|
||||
def _sqlite_db_path(url) -> Optional[str]:
|
||||
"""Return the filesystem path for a file-backed SQLite URL.
|
||||
|
||||
SQLite query parameters such as ``mode=memory`` only affect filename
|
||||
semantics when SQLAlchemy enables URI handling with ``uri=true``. Ordinary
|
||||
file URLs must therefore remain file-backed even when they contain a query
|
||||
parameter named ``mode``.
|
||||
|
||||
For SQLite ``file:`` URIs, an empty authority or ``localhost`` identifies a
|
||||
local path. Other authorities are retained as UNC-style paths.
|
||||
"""
|
||||
if url.get_backend_name() != "sqlite":
|
||||
return None
|
||||
|
||||
db_path = url.database
|
||||
if not db_path or db_path == ":memory:":
|
||||
return None
|
||||
|
||||
db_path = str(db_path)
|
||||
query = {
|
||||
str(key).lower(): str(value).strip().lower()
|
||||
for key, value in dict(getattr(url, "query", {}) or {}).items()
|
||||
}
|
||||
uri_enabled = query.get("uri") in {"1", "true", "yes", "on"}
|
||||
is_file_uri = db_path.lower().startswith("file:")
|
||||
|
||||
if not uri_enabled or not is_file_uri:
|
||||
return db_path
|
||||
|
||||
if (
|
||||
db_path.lower().startswith("file::memory:")
|
||||
or query.get("mode") == "memory"
|
||||
):
|
||||
return None
|
||||
|
||||
parsed = urlparse(db_path)
|
||||
fs_path = parsed.path or ""
|
||||
if not fs_path or fs_path == ":memory:":
|
||||
return None
|
||||
|
||||
authority = parsed.netloc
|
||||
if authority and authority.lower() != "localhost":
|
||||
fs_path = f"//{authority}{fs_path}"
|
||||
|
||||
return unquote(fs_path)
|
||||
|
||||
# Create session factory
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
@@ -1819,6 +1891,41 @@ def init_db():
|
||||
"""
|
||||
_migrate_model_endpoints()
|
||||
Base.metadata.create_all(bind=engine)
|
||||
# Lock the DB file (and any SQLite sidecars) to 0o600 — it holds bearer-token
|
||||
# + bcrypt hashes and encrypted provider keys. POSIX only; safe_chmod no-ops
|
||||
# on Windows (ACL-restricted profile dir) and the path helper returns None for
|
||||
# Postgres / in-memory. Must stay AFTER create_all: the file is born here at
|
||||
# the umask default, and nothing below resets the mode. The path comes from
|
||||
# engine.url (SQLAlchemy's parsed URL), so a driver-qualified or query-tagged
|
||||
# DATABASE_URL still resolves to the real file instead of slipping through.
|
||||
db_path = _sqlite_db_path(engine.url)
|
||||
if db_path is not None:
|
||||
# Fail closed-loud on the main file: this is the only access control on
|
||||
# it, so if the chmod genuinely fails (read-only FS, foreign owner) an
|
||||
# operator should hear about it. safe_chmod also returns False as a
|
||||
# Windows no-op, so guard on IS_WINDOWS to avoid a spurious warning there.
|
||||
if not safe_chmod(db_path, 0o600) and not IS_WINDOWS:
|
||||
logger.warning(
|
||||
"Could not restrict %s to 0o600; it holds secrets and may be "
|
||||
"world-readable. Check filesystem permissions and ownership.",
|
||||
db_path,
|
||||
)
|
||||
# Re-lock any sidecars present at startup. New ones inherit the main
|
||||
# file's mode (now 0o600, since we set it first), and they're usually
|
||||
# absent here, but a stale -wal/-shm/-journal left by an older 0o644
|
||||
# install could still expose secret pages. Absent sidecars are the
|
||||
# normal case, not an error — only a failed chmod warrants a warning.
|
||||
for suffix in _SQLITE_SIDECARS:
|
||||
sidecar = db_path + suffix
|
||||
if (
|
||||
os.path.exists(sidecar)
|
||||
and not safe_chmod(sidecar, 0o600)
|
||||
and not IS_WINDOWS
|
||||
):
|
||||
logger.warning(
|
||||
"Could not restrict %s to 0o600; it may expose DB pages.",
|
||||
sidecar,
|
||||
)
|
||||
_migrate_add_hidden_models_column()
|
||||
_migrate_add_cached_models_column()
|
||||
_migrate_add_pinned_models_column()
|
||||
|
||||
@@ -27,7 +27,6 @@ class DataConfig(BaseSettings):
|
||||
uploads_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "uploads", description="Directory for uploaded files")
|
||||
sessions_file: Path = Field(default=Path(_DATA_DIR_CONST) / "sessions.json", description="Sessions storage file")
|
||||
memory_file: Path = Field(default=Path(_DATA_DIR_CONST) / "memory.json", description="Memory storage file")
|
||||
memory_doc: Path = Field(default=Path(_DATA_DIR_CONST) / "memory_doc.md", description="Memory document file")
|
||||
personal_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "personal_docs", description="Personal documents directory")
|
||||
runbook_dir: Path = Field(default=Path(_DATA_DIR_CONST) / "personal_docs" / "runbook", description="Runbook directory")
|
||||
|
||||
@@ -162,7 +161,6 @@ class AppConfig(BaseSettings):
|
||||
"uploads_dir": data_dir / "uploads",
|
||||
"sessions_file": data_dir / "sessions.json",
|
||||
"memory_file": data_dir / "memory.json",
|
||||
"memory_doc": data_dir / "memory_doc.md",
|
||||
"personal_dir": data_dir / "personal_docs",
|
||||
"runbook_dir": data_dir / "personal_docs" / "runbook",
|
||||
"max_upload_size": max_upload_size,
|
||||
|
||||
@@ -17,7 +17,6 @@ DATA_DIR = os.getenv("ODYSSEUS_DATA_DIR", get_default_data_dir())
|
||||
# re-deriving paths from __file__ or a relative "data" literal.
|
||||
SESSIONS_FILE = os.path.join(DATA_DIR, "sessions.json")
|
||||
MEMORY_FILE = os.path.join(DATA_DIR, "memory.json")
|
||||
MEMORY_DOC = os.path.join(DATA_DIR, "memory_doc.md")
|
||||
PERSONAL_DIR = os.path.join(DATA_DIR, "personal_docs")
|
||||
RUNBOOK_DIR = os.path.join(PERSONAL_DIR, "runbook")
|
||||
UPLOAD_DIR = os.path.join(DATA_DIR, "uploads")
|
||||
|
||||
+214
-110
@@ -1,124 +1,228 @@
|
||||
# Module Organization Summary
|
||||
# Frontend Module Organization Summary
|
||||
|
||||
## Purpose
|
||||
This document describes what each JavaScript module is responsible for.
|
||||
|
||||
> **Note:** This file is a partial, historical overview — not a complete authoritative
|
||||
> inventory. The authoritative module set is the current `static/js/` tree plus the
|
||||
> scripts loaded by `static/index.html`. As of this writing that tree holds **65 `.js`
|
||||
> files** across **8 subdirectories** (`calendar/`, `color/`, `compare/`, `editor/`,
|
||||
> `emailLibrary/`, `markdown/`, `research/`, `util/`), and `static/index.html` loads
|
||||
> **35** `/static…` script tags. The catalog below covers only the original core
|
||||
> modules and is not kept in sync with every module.
|
||||
> **Scope:** This document describes the architecture of the Odysseus no-build
|
||||
> frontend. The app is a collection of native ES6 modules loaded from
|
||||
> `static/`. The authoritative source is the current `static/js/` tree and the
|
||||
> top-level orchestrator `static/app.js`.
|
||||
|
||||
---
|
||||
|
||||
## Core Modules (in static/js/)
|
||||
## 1. Top-level Application Orchestrator
|
||||
|
||||
### 1. **ui.js**
|
||||
- UI helper functions and utilities
|
||||
- Toast notifications (`showToast`, `showError`)
|
||||
- Element getter (`el()`)
|
||||
- Clipboard operations (`copyToClipboard`)
|
||||
- Scroll management (`scrollHistory`, `setAutoScroll`)
|
||||
- Auto-resize textarea
|
||||
- Debounce utility
|
||||
### `static/app.js`
|
||||
*Main application entry point.*
|
||||
|
||||
### 2. **markdown.js**
|
||||
- Markdown processing and rendering
|
||||
- Convert markdown to HTML (`mdToHtml`)
|
||||
- Code block handling with syntax highlighting
|
||||
- Content rendering for message arrays
|
||||
- Text cleanup (`squashOutsideCode`)
|
||||
- Imports all feature modules.
|
||||
- Exposes a few modules on `window` for legacy inter-module reachability
|
||||
(`themeModule`, `sessionModule`, `uiModule`, `adminModule`, `cookbookModule`).
|
||||
- Patches `fetch` so any `401` redirects the user to `/login`.
|
||||
- Fetches the default chat configuration and handles deep-link route openers
|
||||
(`/notes`, `/calendar`, `/email`, `/memory`, `/gallery`, `/cookbook`, `/library`, `/tasks`).
|
||||
- Wires global event listeners: chat-history scrolling, popups, Escape handling,
|
||||
drag-and-drop/paste attachment handling, transcription export, sidebar toggles,
|
||||
rail/tool buttons, and session sorting.
|
||||
- Loads auth status and applies per-user privilege restrictions.
|
||||
|
||||
### 3. **sessions.js**
|
||||
- Session/chat management
|
||||
- Create, load, delete, switch sessions
|
||||
- Session history loading
|
||||
- Direct chat creation with models
|
||||
- Session renaming
|
||||
|
||||
### 4. **memory.js**
|
||||
- AI memory management
|
||||
- Load, add, edit, delete memories
|
||||
- Memory search/filtering
|
||||
- Memory UI rendering
|
||||
- Memory count updates
|
||||
|
||||
### 5. **fileHandler.js**
|
||||
- File attachment handling
|
||||
- File picker dialog
|
||||
- File upload to server
|
||||
- Attachment strip rendering
|
||||
- Pending files management
|
||||
- File preview/removal
|
||||
|
||||
### 6. **voiceRecorder.js**
|
||||
- Voice recording functionality
|
||||
- Start/stop recording
|
||||
- Audio file creation
|
||||
- Microphone permission handling
|
||||
- Recording UI updates
|
||||
|
||||
### 7. **models.js**
|
||||
- Model scanning and display
|
||||
- Local model discovery (ports 8000-8020)
|
||||
- Provider management (OpenAI)
|
||||
- Model selection UI
|
||||
|
||||
### 8. **rag.js**
|
||||
- RAG (Retrieval Augmented Generation) management
|
||||
- Load personal documents
|
||||
- Add directories to RAG
|
||||
- Display included files/directories
|
||||
|
||||
### 9. **presets.js**
|
||||
- Conversation preset management
|
||||
- Load, save, activate presets
|
||||
- Custom preset configuration
|
||||
- Temperature, tokens, system prompt settings
|
||||
|
||||
### 10. **search.js**
|
||||
- Web search settings
|
||||
- Provider selection (DuckDuckGo, Brave, SearXNG)
|
||||
- API key management
|
||||
- Save/load search configuration
|
||||
|
||||
### 11. **chat.js** ⭐ (The Big One)
|
||||
- Main chat functionality
|
||||
- Message handling (`addMessage`)
|
||||
- Chat submission (`handleChatSubmit`)
|
||||
- Streaming response handling
|
||||
- Performance metrics display
|
||||
- Abort request management
|
||||
- Loading states and error handling
|
||||
### `static/index.html`
|
||||
*SPA shell.* Loads `app.js` as a module, includes the theme-aware inline script,
|
||||
and defines the DOM skeleton that the modules populate (chat history, composer,
|
||||
sidebar, icon rail, modals).
|
||||
|
||||
---
|
||||
|
||||
## Main Application File
|
||||
## 2. Core Foundation Modules
|
||||
|
||||
### **app.js**
|
||||
- Application initialization
|
||||
- Event listener setup
|
||||
- Drag & drop handlers
|
||||
- Keyboard shortcuts
|
||||
- Module initialization
|
||||
- Global configuration (API_BASE)
|
||||
- Coordinates all modules together
|
||||
These are imported first and used across most features.
|
||||
|
||||
| Module | Primary Exports | Responsibility |
|
||||
|---|---|---|
|
||||
| **`ui.js`** | `showToast`, `showError`, `el`, `copyToClipboard`, `scrollHistory`, `setAutoScroll`, `autoResize`, `debounce`, `esc` | Shared UI helpers, toast notifications, scroll behavior, element accessor, text escaping. |
|
||||
| **`storage.js`** | `default` storage wrapper | LocalStorage helpers and toggle state persistence. |
|
||||
| **`markdown.js`** | `mdToHtml`, `processWithThinking`, `squashOutsideCode`, `normalizeThinkingMarkup`, `extractThinkingBlocks`, `hasUnclosedThinkTag`, `startsWithReasoningPrefix` | Markdown→HTML, thinking/reasoning block parsing, code-block normalization. |
|
||||
| **`spinner.js`** | `create`, `createWhirlpool` | Loading/spinner factories for streaming and tool cards. |
|
||||
| **`keyboard-shortcuts.js`** | `initKeyboardShortcuts` | Global keyboard shortcut wiring. |
|
||||
| **`sidebar-layout.js`** | `initSidebarLayout`, `syncRailSide` | Wide sidebar ↔ icon-rail layout behavior. |
|
||||
| **`section-management.js`** | `initSectionCollapse`, `initSectionDrag` | Collapsible/draggable sidebar sections. |
|
||||
| **`modalManager.js`** | side-effect import | Unified minimize/restore behavior for floating tool modals. |
|
||||
| **`tileManager.js`** | side-effect import | Desktop window tiling and snap-to-edge behavior. |
|
||||
| **`windowDrag.js`** | `makeWindowDraggable` | Drag support for floating panels. |
|
||||
| **`modalSnap.js`**, **`toolWindowZOrder.js`**, **`windowResize.js`** | — | Modal snapping, z-index management, resize handles. |
|
||||
|
||||
---
|
||||
|
||||
## Dependency Order (Load Order in HTML)
|
||||
```html
|
||||
<script src="/static/js/sessions.js"></script> <!-- 1. Sessions first -->
|
||||
<script src="/static/js/memory.js"></script> <!-- 2. Memory -->
|
||||
<script src="/static/js/markdown.js"></script> <!-- 3. Markdown -->
|
||||
<script src="/static/js/ui.js"></script> <!-- 4. UI utilities -->
|
||||
<script src="/static/js/fileHandler.js"></script> <!-- 5. File handling -->
|
||||
<script src="/static/js/voiceRecorder.js"></script> <!-- 6. Voice -->
|
||||
<script src="/static/js/models.js"></script> <!-- 7. Models -->
|
||||
<script src="/static/js/rag.js"></script> <!-- 8. RAG -->
|
||||
<script src="/static/js/presets.js"></script> <!-- 9. Presets -->
|
||||
<script src="/static/js/search.js"></script> <!-- 10. Search -->
|
||||
<script src="/static/js/chat.js"></script> <!-- 11. Chat -->
|
||||
<script src="/static/app.js"></script> <!-- 12. Main app LAST -->
|
||||
## 3. Chat Pipeline
|
||||
|
||||
The largest and most central subsystem. Chat submission → backend SSE → progressive rendering of text, tools, research, documents, and UI events.
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`chat.js`** | Main chat controller. Handles `handleChatSubmit`, stops/continues, builds `FormData`, posts to `/api/chat_stream`, reads the SSE stream, and dispatches each JSON event to the appropriate renderer. Tracks background streams, stalls, auto-recovery, and multi-round agent state. |
|
||||
| **`chatStream.js`** | Helpers shared between streaming consumers: browser notifications, background-stream completion toasts, and `ui_control` event handling. |
|
||||
| **`chatRenderer.js`** | Message DOM construction: `addMessage`, role labels, model route labels, color coding, footers, metrics, code blocks, sources boxes (`web`/`research`/`RAG`), findings box, images, report links, ask-user cards, welcome screen, and transcript utilities. |
|
||||
| **`streamingRenderer.js`** | Incremental streaming renderer used by `chat.js`. Freezes finalized DOM blocks and only re-renders the growing tail to avoid flicker and O(N²) re-parsing. |
|
||||
| **`streamingSegmenter.js`** | Splits a token stream into display units (text vs code fences) for `streamingRenderer.js`. |
|
||||
| **`slashCommands.js`** | Slash-command registry (`/help`, `/setup`, etc.), parsing, and dispatch handlers. Exported functions are consumed by `chat.js` and `slashAutocomplete.js`. |
|
||||
| **`slashAutocomplete.js`** | Composer autocomplete popup for `/` commands. |
|
||||
| **`composerArrowUpRecall.js`** | Recall last user message with `↑` on an empty composer. |
|
||||
| **`assistant.js`** | Assistant/persona behaviors and message styling helpers. |
|
||||
| **`tts-ai.js`** | AI text-to-speech manager, enqueueing, streaming TTS, and playback button injection. |
|
||||
| **`voiceRecorder.js`** | Voice recording from the composer microphone. |
|
||||
| **`fileHandler.js`** | Attachment picker, paste/drop handling, upload, attachment strip rendering, pending-file management. |
|
||||
| **`codeRunner.js`** | Client-side execution affordances for code blocks returned by the model. |
|
||||
|
||||
---
|
||||
|
||||
## 4. Model, Endpoint, and Configuration Modules
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`models.js`** | Model discovery / scanning, local model port probing, provider management, model selection UI state. |
|
||||
| **`modelPicker.js`** | Composer model-picker dropdown and endpoint selection. |
|
||||
| **`modelSort.js`** | Sorting helpers for model lists. |
|
||||
| **`model/matchKey.js`** | Model-to-key matching helper. |
|
||||
| **`providers.js`** | Provider metadata and account-management helpers. |
|
||||
| **`providerDeviceFlow.js`** | OAuth device-flow support for providers. |
|
||||
| **`presets.js`** | Character/preset selection, custom preset saving, inject prefix/suffix handling. |
|
||||
| **`search.js`** | Web-search settings, provider selection, API key management. |
|
||||
| **`settings.js`** | Settings panel (models, search, appearance, users, MCP, RAG, embedding, tokens). |
|
||||
| **`admin.js`** | Admin panel and privileged user/endpoint configuration. |
|
||||
| **`theme.js`** | Theme presets, custom colors, fonts, backgrounds, live theme switching. |
|
||||
|
||||
---
|
||||
|
||||
## 5. Session, Sidebar, and Workspace
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`sessions.js`** | Chat session list loading, creation, switching, renaming, archiving, library modal, and direct-chat creation. Tracks current session, streaming/research indicators in the sidebar. |
|
||||
| **`workspace.js`** | Workspace folder path management for shell/file tool confinement. |
|
||||
| **`search-chat.js`** | In-chat history search. |
|
||||
| **`skills.js`** | Client-side skill library UI (load, edit, delete, test, and audit status display). |
|
||||
|
||||
---
|
||||
|
||||
## 6. Knowledge, Memory, and RAG
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`memory.js`** | AI memory CRUD, search/filter UI, memory extraction, count badge. |
|
||||
| **`rag.js`** | Personal document RAG: load documents, add directories/files, show included paths. |
|
||||
| **`group.js`** | Group-chat UI and model orchestration. |
|
||||
|
||||
---
|
||||
|
||||
## 7. Document and Editor Subsystems
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`document.js`** | Tabbed document editor, AI edit suggestions, Markdown/HTML/CSV editing, document streaming (`streamDocOpen`/`streamDocDelta`), and panel state. |
|
||||
| **`documentLibrary.js`** | Document library modal. |
|
||||
| **`editor/`** | Gallery image editor canvas modules: layers, brush, inpaint, crop, filters, state, history panel, top-bar wiring, canvas coordinate helpers, and AI model runners for inpainting/background-removal. |
|
||||
|
||||
---
|
||||
|
||||
## 8. Research UI
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`research/panel.js`** | Research panel UI, job list, and controls. |
|
||||
| **`research/jobs.js`** | Research job polling and status rendering. |
|
||||
| **`researchSynapse.js`** | Animated research-progress visualization shown inside the chat bubble during a research run. |
|
||||
|
||||
---
|
||||
|
||||
## 9. Gallery, Email, Calendar, Tasks, and Notes
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`gallery.js`** / **`galleryEditor.js`** | Gallery/image library and canvas editor entry points. |
|
||||
| **`emailInbox.js`** / **`emailLibrary.js`** | Email inbox reader and library modal. Sub-modules handle signatures, reply recipients, state, and signature folding. |
|
||||
| **`calendar.js`** / **`calendar/utils.js`** / **`calendar/reminders.js`** | Calendar views, event forms, reminders. |
|
||||
| **`tasks.js`** | Scheduled task/recurring LLM job UI. |
|
||||
| **`notes.js`** | Notes and todo panel, reminders, pinboard. |
|
||||
|
||||
---
|
||||
|
||||
## 10. Cookbook (Model Serving)
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`cookbook.js`** | Cookbook main UI: hardware fitting, presets, action panels. |
|
||||
| **`cookbook-hwfit.js`** / **`cookbook-diagnosis.js`** / **`cookbook-deps-recipes.js`** | Hardware-fit scoring, dependency diagnosis, recipe handling. |
|
||||
| **`cookbookDownload.js`** / **`cookbookServe.js`** / **`cookbookRunning.js`** / **`cookbookSchedule.js`** / **`cookbookPorts.js`** / **`cookbookProgressSignal.js`** | Model download/serve flow, running job cards, scheduling, port detection, and progress computation. |
|
||||
|
||||
---
|
||||
|
||||
## 11. Compare and Utility Modules
|
||||
|
||||
| Module | Responsibility |
|
||||
|---|---|
|
||||
| **`compare/index.js`** (with `compare/state.js`, `compare/stream.js`, `compare/panes.js`, `compare/selector.js`, `compare/scoreboard.js`, `compare/probe.js`, `compare/vote.js`, `compare/icons.js`) | Model compare mode: parallel streams, panes, scoring, vote UI. |
|
||||
| **`censor.js`** | Text/image censor overlay toggles. |
|
||||
| **`a11y.js`** | Accessibility helpers. |
|
||||
| **`platform.js`** | Platform detection (macOS/Windows/Linux) and keyboard-modifier helpers. |
|
||||
| **`escMenuStack.js`** | Stack manager for dismissible popups. |
|
||||
| **`dragSort.js`** | Drag-to-sort shared behavior. |
|
||||
| **`tourHints.js`** / **`tourAutoplay.js`** | Onboarding tour helpers. |
|
||||
| **`color/hex.js`**, **`colorPicker.js`**, **`langIcons.js`**, **`util/ordinal.js`** | Small utility modules for color, language icons, and formatting. |
|
||||
|
||||
---
|
||||
|
||||
## 12. Frontend Event Streaming Flow
|
||||
|
||||
```
|
||||
User submits composer
|
||||
└── chat.js::handleChatSubmit() builds FormData
|
||||
├── fileHandler.uploadPending() for attachments
|
||||
├── document.js saved (if a document panel is open)
|
||||
└── POST /api/chat_stream
|
||||
|
||||
Server responds with SSE stream
|
||||
└── chat.js reads chunks via res.body.getReader() + TextDecoder
|
||||
├── Lines starting with "event:" set next-error state
|
||||
└── Lines starting with "data:" carry JSON payloads
|
||||
|
||||
JSON events are dispatched by "type":
|
||||
delta → streamingRenderer → markdown → live reply text
|
||||
agent_prep → update spinner label
|
||||
tool_start → finalize text bubble; create agent-thread node with wave animation
|
||||
tool_progress → append/update live stdout/stderr tail
|
||||
tool_output → mark node done/failed, render output, diffs, screenshots
|
||||
agent_step → finalize tool thread; create new msg-continuation bubble
|
||||
doc_stream_open → document.js opens a live document
|
||||
doc_stream_delta → document.js appends content to that document
|
||||
research_progress → researchSynapse visualization + spinner timer
|
||||
research_sources → build sources box for research
|
||||
research_done → reload session history to show the report
|
||||
web_sources → build web-search sources box
|
||||
model_info → update role header with requested/actual model
|
||||
fallback → show fallback model toast + update role label
|
||||
metrics → collect/display token/cost metrics
|
||||
message_saved → store database id on the message element
|
||||
budget_exceeded → show budget banner
|
||||
rounds_exhausted → show Continue button for step-limit hits
|
||||
teacher_takeover → insert escalation banner, reset round state
|
||||
skill_saved → show skill-learned banner
|
||||
```
|
||||
|
||||
Foreground vs background streams:
|
||||
- If the user switches sessions while a stream is running, `chat.js` pauses DOM
|
||||
updates and stores the state in `_backgroundStreams`. Completion is signaled
|
||||
with a sidebar dot/notifications, and the history is reloaded when the user
|
||||
returns.
|
||||
|
||||
---
|
||||
|
||||
## 13. What Changed from the Previous Summary
|
||||
|
||||
- The frontend is now exclusively ES6-module based; the old `<script>` tag load
|
||||
order is no longer authoritative.
|
||||
- `chat.js` is the streaming controller, but message rendering has been split
|
||||
into `chatRenderer.js`, `streamingRenderer.js`, `chatStream.js`, and `researchSynapse.js`.
|
||||
- New major subsystems added since the original summary: compare mode
|
||||
(`compare/`), document editor streaming (`document.js`), research UI
|
||||
(`research/`), model cookbook (`cookbook*.js`), group chat (`group.js`),
|
||||
voice/TTS (`voiceRecorder.js`, `tts-ai.js`), skill UI (`skills.js`), and
|
||||
slash autocomplete (`slashAutocomplete.js`).
|
||||
- `sessions.js` now owns sidebar session state, streaming/research indicators,
|
||||
and the library/archive modals.
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_created_with_0600(tmp_path):
|
||||
"""app.db holds secrets — it must not be world-readable.
|
||||
|
||||
Note: under umask 077 a fresh sqlite file is born 0600 and this would pass
|
||||
even without the chmod; dev/CI umask is 022, where the chmod is what makes
|
||||
it pass. No umask machinery needed — just don't read a green here as proof
|
||||
on a 077 box.
|
||||
|
||||
A subprocess (not in-process patching) is used deliberately: the engine
|
||||
binds to DATABASE_URL at import time, so a fresh interpreter with its own
|
||||
DATABASE_URL is the clean way to exercise init_db() against a real on-disk
|
||||
file without rebinding the already-imported engine.
|
||||
"""
|
||||
db_file = tmp_path / "app.db"
|
||||
env = {**os.environ, "DATABASE_URL": f"sqlite:///{db_file}"}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
# Importing core.database runs init_db() against the temp file-backed DB.
|
||||
# cwd=repo_root so `import core` resolves (the `-c` sys.path[0] is the CWD).
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
assert db_file.exists()
|
||||
mode = db_file.stat().st_mode & 0o777
|
||||
assert mode == 0o600, f"expected 0o600, got 0o{mode:o}"
|
||||
|
||||
# Upgrade path: an already-deployed DB sitting at 0644 must be re-corrected
|
||||
# on the next startup. The chmod is unconditional (not gated on create_all
|
||||
# having created the file), so this is the common path for existing installs.
|
||||
db_file.chmod(0o644)
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
assert db_file.stat().st_mode & 0o777 == 0o600, "existing 0644 DB not re-locked on startup"
|
||||
|
||||
|
||||
def test_normalize_sqlite_url_preserves_sqlite_uri_filename():
|
||||
"""URI filenames must reach SQLAlchemy unchanged for SQLite to parse."""
|
||||
from core.database import _normalize_sqlite_url
|
||||
|
||||
url = "sqlite:///file:/tmp/app.db?mode=rwc&uri=true"
|
||||
assert _normalize_sqlite_url(url) == url
|
||||
|
||||
|
||||
def test_sqlite_db_path_handles_driver_and_query_forms():
|
||||
"""The path fed to chmod must come from SQLAlchemy's parsed URL, not a naive
|
||||
replace("sqlite:///"). A driver-qualified URL (sqlite+pysqlite://) or one
|
||||
carrying query args (?cache=shared) would otherwise resolve to the wrong
|
||||
path and leave the real file world-readable. Pure logic — runs everywhere.
|
||||
"""
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from core.database import _sqlite_db_path
|
||||
|
||||
# Plain forms (relative + absolute) resolve to the file path.
|
||||
assert _sqlite_db_path(make_url("sqlite:///data/app.db")) == "data/app.db"
|
||||
assert _sqlite_db_path(make_url("sqlite:////abs/app.db")) == "/abs/app.db"
|
||||
# A driver qualifier must not defeat detection...
|
||||
assert _sqlite_db_path(make_url("sqlite+pysqlite:///data/app.db")) == "data/app.db"
|
||||
# ...and query args must be stripped from the path.
|
||||
assert _sqlite_db_path(make_url("sqlite:///data/app.db?cache=shared")) == "data/app.db"
|
||||
assert _sqlite_db_path(make_url("sqlite+pysqlite:////abs/app.db?mode=ro")) == "/abs/app.db"
|
||||
# Nothing to lock for non-file-backed or non-sqlite databases.
|
||||
assert _sqlite_db_path(make_url("sqlite:///:memory:")) is None
|
||||
assert _sqlite_db_path(make_url("sqlite://")) is None
|
||||
assert _sqlite_db_path(make_url("postgresql+psycopg2://u:p@h/db")) is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_sidecars_relocked(tmp_path):
|
||||
"""Stale SQLite sidecars (-wal/-shm) left by an older 0o644 install hold
|
||||
copies of DB pages, so startup must re-lock them too — not just app.db.
|
||||
|
||||
The default -journal is transient (SQLite deletes it after the create_all
|
||||
commit), so it isn't asserted on here; -wal/-shm persist and are the real
|
||||
exposure once WAL has ever been enabled.
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
db_file = tmp_path / "app.db"
|
||||
sqlite3.connect(db_file).close() # a real, pre-existing DB ...
|
||||
db_file.chmod(0o644)
|
||||
sidecars = [tmp_path / f"app.db{sfx}" for sfx in ("-wal", "-shm")]
|
||||
for s in sidecars:
|
||||
s.write_bytes(b"")
|
||||
s.chmod(0o644)
|
||||
|
||||
env = {**os.environ, "DATABASE_URL": f"sqlite:///{db_file}"}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert db_file.stat().st_mode & 0o777 == 0o600
|
||||
for s in sidecars:
|
||||
assert s.stat().st_mode & 0o777 == 0o600, f"{s.name} not re-locked on startup"
|
||||
|
||||
|
||||
def test_sqlite_db_path_handles_file_uri_forms(tmp_path):
|
||||
"""SQLite URI filenames must chmod the real filesystem path, not the
|
||||
literal file: URI string. Memory URI databases should still be skipped."""
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from core.database import _sqlite_db_path
|
||||
|
||||
db_file = tmp_path / "uri-app.db"
|
||||
|
||||
assert (
|
||||
_sqlite_db_path(make_url(f"sqlite+pysqlite:///file:{db_file}?mode=rwc&uri=true"))
|
||||
== str(db_file)
|
||||
)
|
||||
assert (
|
||||
_sqlite_db_path(make_url(f"sqlite:///file:{db_file}?cache=shared&uri=true"))
|
||||
== str(db_file)
|
||||
)
|
||||
|
||||
localhost_db = tmp_path / "localhost-uri.db"
|
||||
assert (
|
||||
_sqlite_db_path(
|
||||
make_url(
|
||||
f"sqlite+pysqlite:///file://localhost{localhost_db}"
|
||||
"?mode=rwc&uri=true"
|
||||
)
|
||||
)
|
||||
== str(localhost_db)
|
||||
)
|
||||
|
||||
non_uri_mode_db = tmp_path / "mode-query-file.db"
|
||||
assert (
|
||||
_sqlite_db_path(
|
||||
make_url(
|
||||
f"sqlite+pysqlite:///{non_uri_mode_db}?mode=memory"
|
||||
)
|
||||
)
|
||||
== str(non_uri_mode_db)
|
||||
)
|
||||
assert (
|
||||
_sqlite_db_path(make_url("sqlite+pysqlite:///file::memory:?cache=shared&uri=true"))
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
_sqlite_db_path(make_url("sqlite+pysqlite:///file:memdb1?mode=memory&cache=shared&uri=true"))
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_file_uri_created_with_0600(tmp_path):
|
||||
"""Import-time DB initialization must lock SQLite file: URI databases too."""
|
||||
db_file = tmp_path / "uri-app.db"
|
||||
env = {
|
||||
**os.environ,
|
||||
"DATABASE_URL": f"sqlite+pysqlite:///file:{db_file}?mode=rwc&uri=true",
|
||||
}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert db_file.exists()
|
||||
mode = db_file.stat().st_mode & 0o777
|
||||
assert mode == 0o600, f"expected 0o600, got 0o{mode:o}"
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_localhost_file_uri_created_with_0600(tmp_path):
|
||||
"""A file://localhost URI must chmod the local path SQLite opens."""
|
||||
db_file = tmp_path / "localhost-uri.db"
|
||||
env = {
|
||||
**os.environ,
|
||||
"DATABASE_URL": (
|
||||
f"sqlite+pysqlite:///file://localhost{db_file}"
|
||||
"?mode=rwc&uri=true"
|
||||
),
|
||||
}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert db_file.exists()
|
||||
mode = db_file.stat().st_mode & 0o777
|
||||
assert mode == 0o600, f"expected 0o600, got 0o{mode:o}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_non_uri_mode_query_created_with_0600(tmp_path):
|
||||
"""mode=memory without uri=true must not hide a real SQLite file."""
|
||||
db_file = tmp_path / "mode-query-file.db"
|
||||
env = {
|
||||
**os.environ,
|
||||
"DATABASE_URL": f"sqlite+pysqlite:///{db_file}?mode=memory",
|
||||
}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert db_file.exists()
|
||||
mode = db_file.stat().st_mode & 0o777
|
||||
assert mode == 0o600, f"expected 0o600, got 0o{mode:o}"
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="POSIX mode bits (0o600) don't exist on Windows; safe_chmod no-ops there.",
|
||||
)
|
||||
def test_app_db_plain_file_uri_created_with_0600(tmp_path):
|
||||
"""The documented sqlite:///file: URI form must remain protected."""
|
||||
db_file = tmp_path / "plain-uri-app.db"
|
||||
env = {
|
||||
**os.environ,
|
||||
"DATABASE_URL": f"sqlite:///file:{db_file}?mode=rwc&uri=true",
|
||||
}
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-c", "import core.database"],
|
||||
env=env,
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
)
|
||||
|
||||
assert db_file.exists()
|
||||
mode = db_file.stat().st_mode & 0o777
|
||||
assert mode == 0o600, f"expected 0o600, got 0o{mode:o}"
|
||||
Reference in New Issue
Block a user