diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 229a4d599..5d70ee765 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,3 +256,10 @@ jobs: env: PYTEST_SHARD: ${{ matrix.shard }} run: python -m pytest -q -rs --durations=25 --shard "$PYTEST_SHARD" + + - name: Playwright mobile layout checks (shard 1/4 only) + if: steps.docs-check.outputs.docs_only != 'true' && matrix.shard == '1/4' + run: | + ODYSSEUS_TEST_PYTHON=python ODYSSEUS_LAYOUT_BROWSER=chromium npx --no-install playwright test \ + --config tests/e2e/ios-layout.config.cjs \ + --reporter=line diff --git a/.github/workflows/native-test-suites.yml b/.github/workflows/native-test-suites.yml index 2d157ff24..e85e9d43d 100644 --- a/.github/workflows/native-test-suites.yml +++ b/.github/workflows/native-test-suites.yml @@ -94,7 +94,7 @@ jobs: PLAYWRIGHT_JUNIT_OUTPUT_FILE: test-results/browser-junit.xml run: | npm run test:photo-editor -- --reporter=line,junit - python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 130 + python .github/scripts/check_test_execution.py test-results/browser-junit.xml --minimum 137 - name: Save per-test results and sanitized failure diagnostics if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/docs/maintainers/native-test-coverage.md b/docs/maintainers/native-test-coverage.md index 5b2f1ba4f..5f7a2c575 100644 --- a/docs/maintainers/native-test-coverage.md +++ b/docs/maintainers/native-test-coverage.md @@ -14,8 +14,9 @@ through an explicitly identified Playwright driver. The manifest validator fails for omitted/duplicate/unreachable entries. Node emits individual JUnit identities and propagates its failure exit. No native file is newly run in each pytest shard. -The browser job uses the existing one-worker Chromium config and all original129 -cases, plus one driver for the native email-reply contract. Its ephemeral account, +The browser job uses the existing one-worker Chromium config and all 136 Photo +Editor cases (including cursor and brush regressions), plus one driver for the native +email-reply contract (137 cases total). Its ephemeral account, SQLite DB and server are created by the config/setup; reuseExistingServer remains false. NPX startup is offline/cache-only in a private cache; Notes/editor behavior and Python MCP services remain real. No developer app or external account is used. diff --git a/mcp_servers/image_gen_server.py b/mcp_servers/image_gen_server.py index 697522e3d..327da5c03 100644 --- a/mcp_servers/image_gen_server.py +++ b/mcp_servers/image_gen_server.py @@ -33,7 +33,7 @@ async def list_tools() -> list[Tool]: "prompt": {"type": "string", "description": "Image description prompt"}, "model": {"type": "string", "description": "Model name (auto-detects if omitted)"}, "size": {"type": "string", "description": "Image size (default 1024x1024)"}, - "quality": {"type": "string", "description": "Quality: low, medium, high, auto (default medium)"}, + "quality": {"type": "string", "description": "Quality: low, medium, high, auto (default configured quality, otherwise medium)"}, }, "required": ["prompt"], }, @@ -66,7 +66,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: if not model_spec: model_spec = _settings.get("image_model", "") - if quality == "medium" and _settings.get("image_quality"): + if "quality" not in arguments and _settings.get("image_quality"): quality = _settings["image_quality"] # Auto-detect best available image model diff --git a/package.json b/package.json index aa454e017..79cf83c3d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,11 @@ "test:photo-editor": "playwright test --config tests/e2e/playwright.config.js", "test:photo-editor:install": "playwright install chromium firefox webkit", "test:photo-editor:firefox": "PHOTO_EDITOR_E2E_BROWSER=firefox playwright test --config tests/e2e/playwright.config.js", - "test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js" + "test:photo-editor:webkit": "PHOTO_EDITOR_E2E_BROWSER=webkit playwright test --config tests/e2e/playwright.config.js", + "test:ios-layout": "playwright test --config tests/e2e/ios-layout.config.cjs", + "test:ios-layout:chromium": "ODYSSEUS_LAYOUT_BROWSER=chromium playwright test --config tests/e2e/ios-layout.config.cjs", + "test:ios-layout:webkit": "ODYSSEUS_LAYOUT_BROWSER=webkit playwright test --config tests/e2e/ios-layout.config.cjs", + "test:ios-layout:install": "playwright install chromium webkit" }, "devDependencies": { "@antithesishq/bombadil": "^0.7.0", diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 1042437ac..378f22afa 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -887,6 +887,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: """Create a new integration (admin only).""" _require_admin_user(request) body = await request.json() + if not isinstance(body, dict): + raise HTTPException(400, "Request body must be a JSON object") item = add_integration(body) return {"ok": True, "integration": mask_integration_secret(item)} @@ -895,6 +897,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter: """Update an existing integration (admin only).""" _require_admin_user(request) body = await request.json() + if not isinstance(body, dict): + raise HTTPException(400, "Request body must be a JSON object") item = update_integration(integration_id, body) if not item: raise HTTPException(404, "Integration not found") diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index aa84768fd..19c8fd6a7 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -80,6 +80,18 @@ def _require_user(request: Request) -> str: return FALLBACK_OWNER +async def _json_object_body(request: Request) -> dict: + """Parse a JSON object body. An unreadable body counts as empty; a body + that is valid JSON but not an object (a list, string or number) is a 400.""" + try: + body = await request.json() + except Exception: + return {} + if not isinstance(body, dict): + raise HTTPException(400, "Request body must be a JSON object") + return body + + def _get_or_404_calendar(db, cal_id: str, owner: str) -> CalendarCal: cal = db.query(CalendarCal).filter(CalendarCal.id == cal_id).first() if not cal: @@ -994,10 +1006,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: async def save_config(request: Request): """Legacy single-account endpoint — upserts the first account.""" owner = _require_user(request) - try: - body = await request.json() - except Exception: - body = {} + body = await _json_object_body(request) accounts = _get_caldav_accounts(owner) if not (body.get("url") or "").strip(): _save_caldav_accounts(owner, []) @@ -1052,10 +1061,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: """Add a new CalDAV account.""" import uuid as _uuid owner = _require_user(request) - try: - body = await request.json() - except Exception: - body = {} + body = await _json_object_body(request) from src.caldav_sync import validate_caldav_url try: url = validate_caldav_url(body.get("url", "")) @@ -1080,10 +1086,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: async def update_caldav_account(account_id: str, request: Request): """Update an existing CalDAV account by id.""" owner = _require_user(request) - try: - body = await request.json() - except Exception: - body = {} + body = await _json_object_body(request) accounts = _get_caldav_accounts(owner) idx = next((i for i, a in enumerate(accounts) if a.get("id") == account_id), None) if idx is None: @@ -1124,10 +1127,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter: test an already-saved account. Falls back to the first saved account when nothing is provided.""" owner = _require_user(request) - try: - body = await request.json() - except Exception: - body = {} + body = await _json_object_body(request) url = (body.get("url") or "").strip() user = (body.get("username") or "").strip() pw = body.get("password") or "" diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index 856c8bdb5..0face7fd8 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -14,6 +14,7 @@ from fastapi import HTTPException from pydantic import BaseModel from routes._validators import validate_remote_host, validate_ssh_port +from routes.cookbook_output import HF_PARTIAL_BLOB_HELPER from core.platform_compat import _ssh_exec_argv logger = logging.getLogger(__name__) @@ -378,6 +379,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: """ lines = [ "import json, os, re, shutil, subprocess, urllib.request", + *HF_PARTIAL_BLOB_HELPER.splitlines(), "models = []", "seen = set()", "BLOCKED_ROOTS = ('/sys', '/proc', '/dev', '/run', '/var/run')", @@ -436,8 +438,10 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: " sz, nf, ic = 0, 0, False", " if os.path.isdir(blobs):", " for f in os.scandir(blobs):", + " if f.name.endswith('.incomplete'):", + " if not is_partial_blob(f.path): continue", + " ic = True", " if f.is_file(): nf += 1; sz += f.stat().st_size", - " if f.name.endswith('.incomplete'): ic = True", " snap = os.path.join(cache, d, 'snapshots')", " def snapshot_size():", " total, count, incomplete = 0, 0, False", @@ -1030,6 +1034,166 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None: runner_lines.append(' fi # end _odysseus_have_prebuilt guard') +ODYSSEUS_LLAMA_SHIM_MARKER = "# Auto-generated by Odysseus Cookbook: a `llama-server` lookalike" + + +def _append_llama_server_resolution_definition_lines(runner_lines: list[str]) -> None: + """Define the helper shell function that discovers, reconciles and prioritizes llama-server.""" + marker = f"'{_bash_squote(ODYSSEUS_LLAMA_SHIM_MARKER)}'" + runner_lines.append('_odysseus_resolve_llama_server() {') + runner_lines.append(' _odysseus_llama_bin=""') + runner_lines.append(f' local _marker={marker}') + runner_lines.append(' local _saved_ifs="$IFS"') + runner_lines.append(" IFS=':'") + runner_lines.append(' local -a _parts') + runner_lines.append(' read -r -a _parts <<< "$PATH"') + runner_lines.append(' IFS="$_saved_ifs"') + runner_lines.append(' local _dir _cand') + runner_lines.append(' for _dir in "${_parts[@]}"; do') + runner_lines.append(' [ -n "$_dir" ] || continue') + runner_lines.append(' _cand="$_dir/llama-server"') + runner_lines.append(' [ -x "$_cand" ] || continue') + runner_lines.append(' if [ -f "$_cand" ] && [ ! -L "$_cand" ] && grep -Fxq "$_marker" "$_cand" 2>/dev/null; then') + runner_lines.append(' continue') + runner_lines.append(' fi') + runner_lines.append(' if "$_cand" --version >/dev/null 2>&1; then') + runner_lines.append(' _odysseus_llama_bin="$_cand"') + runner_lines.append(' break') + runner_lines.append(' fi') + runner_lines.append(' done') + runner_lines.append(' if [ -f "$HOME/bin/llama-server" ] && [ ! -L "$HOME/bin/llama-server" ] && grep -Fxq "$_marker" "$HOME/bin/llama-server" 2>/dev/null; then') + runner_lines.append(' if [ -n "$_odysseus_llama_bin" ]; then') + runner_lines.append(' echo "[odysseus] Native llama-server detected — retiring Odysseus compatibility shim in ~/bin"') + runner_lines.append(' rm -f "$HOME/bin/llama-server"') + runner_lines.append(' hash -r') + runner_lines.append(' elif python3 -c "import llama_cpp.server" 2>/dev/null; then') + runner_lines.append(' _odysseus_llama_bin="$HOME/bin/llama-server"') + runner_lines.append(' else') + runner_lines.append(' echo "[odysseus] Removing stale Odysseus llama-server shim (python3 -m llama_cpp.server unavailable)"') + runner_lines.append(' rm -f "$HOME/bin/llama-server"') + runner_lines.append(' hash -r') + runner_lines.append(' fi') + runner_lines.append(' fi') + runner_lines.append(' if [ -n "$_odysseus_llama_bin" ]; then') + runner_lines.append(' local _cand_dir') + runner_lines.append(' _cand_dir="$(dirname "$_odysseus_llama_bin")"') + runner_lines.append(' if [ "${PATH%%:*}" != "$_cand_dir" ]; then') + runner_lines.append(' export PATH="$_cand_dir:$PATH"') + runner_lines.append(' fi') + runner_lines.append(' hash -r') + runner_lines.append(' return 0') + runner_lines.append(' fi') + runner_lines.append(' return 1') + runner_lines.append('}') + + +def _append_llama_server_shim_definition_lines(runner_lines: list[str]) -> None: + """Define the helper shell function that generates the Odysseus compatibility shim.""" + _append_llama_server_resolution_definition_lines(runner_lines) + marker = f"'{_bash_squote(ODYSSEUS_LLAMA_SHIM_MARKER)}'" + runner_lines.append('_odysseus_create_llama_shim() {') + runner_lines.append(' local _target="$HOME/bin/llama-server"') + runner_lines.append(f' local _marker={marker}') + runner_lines.append(' if [ -e "$_target" ] || [ -L "$_target" ]; then') + runner_lines.append(' if [ -f "$_target" ] && [ ! -L "$_target" ] && grep -Fxq "$_marker" "$_target" 2>/dev/null; then') + runner_lines.append(' : # existing Odysseus shim — safe to replace') + runner_lines.append(' else') + runner_lines.append(' echo "[odysseus] WARNING: $_target already exists and is not an Odysseus-managed shim; leaving untouched." >&2') + runner_lines.append(' return 1') + runner_lines.append(' fi') + runner_lines.append(' fi') + runner_lines.append(' mkdir -p "$HOME/bin" || return 1') + runner_lines.append(' local _tmp') + runner_lines.append(' _tmp="$(mktemp "$HOME/bin/.llama-server.odysseus.tmp.XXXXXX")" || {') + runner_lines.append(' echo "[odysseus] ERROR: unable to create secure temporary llama-server shim in $HOME/bin" >&2') + runner_lines.append(' return 1') + runner_lines.append(' }') + runner_lines.append(" if ! cat > \"$_tmp\" <<'_ODY_LLAMA_SHIM_EOF'") + runner_lines.append('#!/usr/bin/env bash') + runner_lines.append(ODYSSEUS_LLAMA_SHIM_MARKER) + runner_lines.append('# that translates the native CLI to `python -m llama_cpp.server`.') + runner_lines.append('# Lets cookbook-generated launch commands run unchanged on hosts') + runner_lines.append('# where only the pip llama-cpp-python package is installed.') + runner_lines.append('ARGS=()') + runner_lines.append('while [ $# -gt 0 ]; do') + runner_lines.append(' case "$1" in') + runner_lines.append(' -ngl|--gpu-layers|--n-gpu-layers) ARGS+=(--n_gpu_layers "$2"); shift 2 ;;') + runner_lines.append(' -c|--ctx-size) ARGS+=(--n_ctx "$2"); shift 2 ;;') + runner_lines.append(' -b|--batch-size) ARGS+=(--n_batch "$2"); shift 2 ;;') + runner_lines.append(' -ub|--ubatch-size) shift 2 ;; # llama-cpp-python has no separate ubatch') + runner_lines.append(' --flash-attn) ARGS+=(--flash_attn true); shift 2 ;;') + runner_lines.append(' --cache-type-k) ARGS+=(--type_k "$2"); shift 2 ;;') + runner_lines.append(' --cache-type-v) ARGS+=(--type_v "$2"); shift 2 ;;') + runner_lines.append(' --n-cpu-moe) ARGS+=(--n_cpu_moe "$2"); shift 2 ;;') + runner_lines.append(' --mmproj) ARGS+=(--clip_model_path "$2"); shift 2 ;;') + runner_lines.append(' --image-max-tokens) shift 2 ;; # native-only') + runner_lines.append(' --no-mmap) ARGS+=(--no_mmap true); shift ;;') + runner_lines.append(' --no-warmup) shift ;; # native-only') + runner_lines.append(' --chat-template) ARGS+=(--chat_format "$2"); shift 2 ;;') + runner_lines.append(' --fit|--split-mode|--tensor-split|--main-gpu|--parallel) shift 2 ;; # native-only') + runner_lines.append(' --mlock) ARGS+=(--use_mlock true); shift ;;') + runner_lines.append(' *) ARGS+=("$1"); shift ;;') + runner_lines.append(' esac') + runner_lines.append('done') + runner_lines.append('exec python3 -m llama_cpp.server "${ARGS[@]}"') + runner_lines.append('_ODY_LLAMA_SHIM_EOF') + runner_lines.append(' then') + runner_lines.append(' rm -f -- "$_tmp"') + runner_lines.append(' return 1') + runner_lines.append(' fi') + runner_lines.append(' chmod +x "$_tmp" || { rm -f -- "$_tmp"; return 1; }') + runner_lines.append(' mv -f -- "$_tmp" "$_target" || { rm -f -- "$_tmp"; return 1; }') + runner_lines.append(' hash -r') + runner_lines.append(' echo "[odysseus] Created llama-server shim → python -m llama_cpp.server (no native binary needed)"') + runner_lines.append('}') + + +def _append_llama_server_shim_reconciliation_lines(runner_lines: list[str]) -> None: + """Reconcile an existing ~/bin/llama-server if it is owned by Odysseus Cookbook. + + Invariants: + - Never touch an arbitrary user-owned executable or symlink. + - If ~/bin/llama-server is an Odysseus generated shim: + - If a native llama-server is present and healthy elsewhere on PATH, remove the shim + so the native binary is used. + - If python3 -m llama_cpp.server is unavailable, remove the stale shim + and hash -r so the fallback/build path is triggered. + - If no healthy native server exists and llama_cpp.server is working, preserve it. + - If a healthy native candidate exists on PATH (even later in PATH), prioritize it and + ensure the launched executable is the validated candidate. + """ + runner_lines.append('[ -r ~/.config/odysseus-llama-cpp-env ] && . ~/.config/odysseus-llama-cpp-env') + runner_lines.append('if ! declare -F _odysseus_resolve_llama_server >/dev/null 2>&1; then') + _append_llama_server_resolution_definition_lines(runner_lines) + runner_lines.append('fi') + runner_lines.append('_odysseus_resolve_llama_server || true') + + +def _append_llama_server_finalization_lines(runner_lines: list[str], platform_hint: str = "") -> None: + """Reconcile post-install launcher availability and enforce fail-closed invariant. + + Guarantees that if python3 -m llama_cpp.server is usable but the `llama-server` + command is unresolvable, the compatibility launcher is created. If no usable + launcher exists after all build/install attempts, sets ODYSSEUS_PREFLIGHT_EXIT=127. + """ + err_msg = ( + f"ERROR: llama.cpp serving is not available on {platform_hint} after install attempts." + if platform_hint + else "ERROR: llama.cpp serving is not available after install/build attempts." + ) + runner_lines.append('if ! declare -F _odysseus_resolve_llama_server >/dev/null 2>&1; then') + _append_llama_server_resolution_definition_lines(runner_lines) + runner_lines.append('fi') + runner_lines.append('if [ -z "$_odysseus_llama_bin" ] && python3 -c "import llama_cpp.server" 2>/dev/null; then') + runner_lines.append(' _odysseus_create_llama_shim') + runner_lines.append(' _odysseus_resolve_llama_server || true') + runner_lines.append('fi') + runner_lines.append('if [ -z "$_odysseus_llama_bin" ]; then') + runner_lines.append(f' echo "{err_msg}"') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('fi') + + def _llama_cpp_rebuild_cmd(update_source: bool = False) -> str: """Shell command that clears the Cookbook-managed llama.cpp build. diff --git a/routes/cookbook_output.py b/routes/cookbook_output.py index b30b18536..a8e8ed528 100644 --- a/routes/cookbook_output.py +++ b/routes/cookbook_output.py @@ -8,6 +8,20 @@ import re _FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE) +# Shared source embedded in local/remote inventory and status probes. A blank +# retry marker beside its completed, nonempty content-addressed blob is stale; +# a blank marker alone may be a download that has not received its first byte. +HF_PARTIAL_BLOB_HELPER = """def is_partial_blob(path): + if not path.endswith('.incomplete'): + return False + try: + completed = path[:-len('.incomplete')] + return (os.path.getsize(path) != 0 or not os.path.isfile(completed) + or os.path.getsize(completed) == 0) + except OSError: + return True +""" + # Probe scripts for the dead-session download check, run as # `python3 -c ` (locally or over SSH). # cache_root is the task's custom download dir, '' for the default HF cache. @@ -15,7 +29,7 @@ _FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE) # HF_HOME=, so that task's cache lives under /hub, and # the probe process's own environment knows nothing about it. HF_CACHE_COMPLETE_PROBE = ( - "import os,sys;" + "import os,sys\n" + HF_PARTIAL_BLOB_HELPER + "repo=sys.argv[1];" "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" @@ -24,18 +38,18 @@ HF_CACHE_COMPLETE_PROBE = ( "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));" "inc=False;" "blobs=os.path.join(d,'blobs');" - "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "inc=os.path.isdir(blobs) and any(is_partial_blob(os.path.join(blobs,x)) for x in os.listdir(blobs));" "sys.exit(0 if ok and not inc else 1)" ) HF_CACHE_INCOMPLETE_PROBE = ( - "import os,sys;" + "import os,sys\n" + HF_PARTIAL_BLOB_HELPER + "repo=sys.argv[1];" "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" "d=os.path.join(base,'models--'+repo.replace('/','--'));" "blobs=os.path.join(d,'blobs');" - "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "inc=os.path.isdir(blobs) and any(is_partial_blob(os.path.join(blobs,x)) for x in os.listdir(blobs));" "sys.exit(0 if inc else 1)" ) diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 02b419894..0799d3ace 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -57,9 +57,11 @@ from routes.cookbook_helpers import ( _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, _diagnose_serve_output, run_ssh_command_async, _ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache, - _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, _append_pip_install_runner_lines, _pip_install_command_without_break_system_packages, _normalize_llama_cpp_python_cache_types, + _append_llama_server_shim_definition_lines, + _append_llama_server_shim_reconciliation_lines, + _append_llama_server_finalization_lines, ModelDownloadRequest, ServeRequest, ) @@ -2161,7 +2163,7 @@ def setup_cookbook_routes() -> APIRouter: ps_lines.append('}') elif "llama_cpp" in req.cmd or "llama-server" in req.cmd: ps_lines.append('# Auto-install llama-cpp-python if missing') - ps_lines.append('try { python -c "import llama_cpp" 2>$null } catch {}') + ps_lines.append('try { python -c "import llama_cpp.server" 2>$null } catch {}') ps_lines.append('if ($LASTEXITCODE -ne 0) {') ps_lines.append(' Write-Host "Installing llama-cpp-python..."') ps_lines.append(' python -m pip install llama-cpp-python[server]') @@ -2247,14 +2249,17 @@ def setup_cookbook_routes() -> APIRouter: # ollama is found (otherwise macOS falls back to a slow source build). # /opt/homebrew = Apple Silicon, /usr/local = Intel; harmless on Linux. runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:$HOME/llama.cpp/build/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') + _append_llama_server_shim_definition_lines(runner_lines) + _append_llama_server_shim_reconciliation_lines(runner_lines) runner_lines.append('if [ -d /data/data/com.termux ]; then') runner_lines.append(' # Termux: no native build — use the Python bindings (CPU).') - runner_lines.append(' if ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' if ! python3 -c "import llama_cpp.server" 2>/dev/null; then') runner_lines.append(' pkg install -y cmake 2>/dev/null') runner_lines.append(' pip install numpy diskcache jinja2 2>/dev/null') runner_lines.append(' CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install \'llama-cpp-python[server]\' --no-build-isolation --no-cache-dir 2>&1 || true') runner_lines.append(' fi') - runner_lines.append('elif ! command -v llama-server &>/dev/null; then') + _append_llama_server_finalization_lines(runner_lines, platform_hint="Termux") + runner_lines.append('elif [ -z "$_odysseus_llama_bin" ]; then') runner_lines.append(' echo "Native llama-server not found — building from source (one-time, may take a few minutes)..."') runner_lines.append(' mkdir -p ~/bin') runner_lines.append(' cd ~ && [ -d llama.cpp ] || git clone --depth 1 https://github.com/ggml-org/llama.cpp') @@ -2271,8 +2276,7 @@ def setup_cookbook_routes() -> APIRouter: # would reuse the bad settings and fail again. CMAKE_BUILD_TYPE is # explicit so the binary is optimized (Metal auto-enables on macOS). runner_lines.append(' cd ~/llama.cpp && rm -rf build && cmake -B build -DCMAKE_BUILD_TYPE=Release \\') - runner_lines.append(' && cmake --build build -j"$NPROC" --target llama-server \\') - runner_lines.append(' && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' && cmake --build build -j"$NPROC" --target llama-server') runner_lines.append(' else') _append_llama_cpp_linux_accel_build_lines(runner_lines) runner_lines.append(' fi') @@ -2296,58 +2300,13 @@ def setup_cookbook_routes() -> APIRouter: runner_lines.append(' fi') runner_lines.append(' fi') runner_lines.append(' fi') - # SHORT-CIRCUIT before the build/pip fallback: if the - # native binary is missing but llama_cpp Python is already - # installed, drop a wrapper at ~/bin/llama-server that - # translates llama-server CLI args to llama_cpp.server's - # underscore-style flags. The user's serve command stays - # `llama-server ...` and "just works" — no build, no cmake, - # no second install. This is the path that unblocks every - # remote where pip-installed llama-cpp-python is already - # working but Cookbook used to insist on a native binary. - runner_lines.append(' if ! command -v llama-server >/dev/null 2>&1 && python3 -c "import llama_cpp" 2>/dev/null; then') - runner_lines.append(' mkdir -p ~/bin') - runner_lines.append(' cat > ~/bin/llama-server <<\'_ODY_LLAMA_SHIM_EOF\'') - runner_lines.append('#!/usr/bin/env bash') - runner_lines.append('# Auto-generated by Odysseus Cookbook: a `llama-server` lookalike') - runner_lines.append('# that translates the native CLI to `python -m llama_cpp.server`.') - runner_lines.append('# Lets cookbook-generated launch commands run unchanged on hosts') - runner_lines.append('# where only the pip llama-cpp-python package is installed.') - runner_lines.append('ARGS=()') - runner_lines.append('while [ $# -gt 0 ]; do') - runner_lines.append(' case "$1" in') - runner_lines.append(' -ngl|--gpu-layers|--n-gpu-layers) ARGS+=(--n_gpu_layers "$2"); shift 2 ;;') - runner_lines.append(' -c|--ctx-size) ARGS+=(--n_ctx "$2"); shift 2 ;;') - runner_lines.append(' -b|--batch-size) ARGS+=(--n_batch "$2"); shift 2 ;;') - runner_lines.append(' -ub|--ubatch-size) shift 2 ;; # llama-cpp-python has no separate ubatch') - runner_lines.append(' --flash-attn) ARGS+=(--flash_attn true); shift 2 ;;') - runner_lines.append(' --cache-type-k) ARGS+=(--type_k "$2"); shift 2 ;;') - runner_lines.append(' --cache-type-v) ARGS+=(--type_v "$2"); shift 2 ;;') - runner_lines.append(' --n-cpu-moe) ARGS+=(--n_cpu_moe "$2"); shift 2 ;;') - runner_lines.append(' --mmproj) ARGS+=(--clip_model_path "$2"); shift 2 ;;') - runner_lines.append(' --image-max-tokens) shift 2 ;; # native-only') - runner_lines.append(' --no-mmap) ARGS+=(--no_mmap true); shift ;;') - runner_lines.append(' --no-warmup) shift ;; # native-only') - runner_lines.append(' --chat-template) ARGS+=(--chat_format "$2"); shift 2 ;;') - runner_lines.append(' --fit|--split-mode|--tensor-split|--main-gpu|--parallel) shift 2 ;; # native-only') - runner_lines.append(' --mlock) ARGS+=(--use_mlock true); shift ;;') - runner_lines.append(' *) ARGS+=("$1"); shift ;;') - runner_lines.append(' esac') - runner_lines.append('done') - runner_lines.append('exec python3 -m llama_cpp.server "${ARGS[@]}"') - runner_lines.append('_ODY_LLAMA_SHIM_EOF') - runner_lines.append(' chmod +x ~/bin/llama-server') - runner_lines.append(' echo "[odysseus] Created llama-server shim → python -m llama_cpp.server (no native binary needed)"') - runner_lines.append(' fi') - runner_lines.append(' # If the native build failed, fall back to the Python bindings.') - runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' _odysseus_resolve_llama_server || true') + # If the native build failed, fall back to the Python bindings. + runner_lines.append(' if [ -z "$_odysseus_llama_bin" ] && ! python3 -c "import llama_cpp.server" 2>/dev/null; then') runner_lines.append(' echo "llama-server build failed — installing Python bindings as fallback..."') runner_lines.append(f" {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='pip')} || true") runner_lines.append(' fi') - runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') - runner_lines.append(' echo "ERROR: llama.cpp serving is not available after install/build attempts."') - runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') - runner_lines.append(' fi') + _append_llama_server_finalization_lines(runner_lines) runner_lines.append('fi') elif re.search(r"\bollama\s+serve\b", req.cmd): handled_ollama_serve = True @@ -4343,7 +4302,7 @@ def setup_cookbook_routes() -> APIRouter: tmux output can stop at a stale progress line if the pane/session disappears before Cookbook captures the final DOWNLOAD_OK marker. In that case, trust the cache shape: a snapshot directory with files - and no *.incomplete blobs means HuggingFace finished materializing the + and no unfinished blobs means HuggingFace finished materializing the model. cache_root is the task's custom download dir — the runner pointed HF_HOME there, so the cache lives under /hub, not wherever this probe's environment says. @@ -4368,7 +4327,8 @@ def setup_cookbook_routes() -> APIRouter: """Best-effort check for resumable HF partial blobs. A lost SSH/tmux session can leave a real download still incomplete. - Treat any *.incomplete blob as stronger evidence than stale + Treat unfinished blobs (excluding empty markers paired with a + completed blob) as stronger evidence than stale "100%" lines in the captured pane output. """ if not repo_id or "/" not in repo_id: diff --git a/routes/document/document_routes.py b/routes/document/document_routes.py index 2b987e140..2a820de75 100644 --- a/routes/document/document_routes.py +++ b/routes/document/document_routes.py @@ -781,9 +781,11 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: except Exception as e: logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e) data = {} - ids = data.get("ids") or [] + ids = data.get("ids") if isinstance(data, dict) else None if not ids: raise HTTPException(400, "No documents specified") + if not isinstance(ids, list) or not all(isinstance(i, str) for i in ids): + raise HTTPException(400, "ids must be a list of document ids") _ext = { "javascript": ".js", "python": ".py", "html": ".html", "css": ".css", "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh", diff --git a/routes/font_routes.py b/routes/font_routes.py index 3451db8c7..fbfd20458 100644 --- a/routes/font_routes.py +++ b/routes/font_routes.py @@ -19,9 +19,10 @@ def _split_family_token(token): def _derive_family(filename): """Derive a font-family name from a filename like 'JetBrainsMono-Regular.woff2' → 'JetBrains Mono'.""" name = os.path.splitext(filename)[0] - # Strip common weight/style suffixes + # Strip weight/style suffixes, including BoldItalic and Bold-Italic. name = re.sub( - r'[-_ ]?(Thin|ExtraLight|UltraLight|Light|Regular|Medium|SemiBold|DemiBold|Bold|ExtraBold|UltraBold|Black|Heavy|Italic|Oblique|Variable|VF)$', + r'[-_ ]?(Thin|ExtraLight|UltraLight|Light|Regular|Medium|SemiBold|DemiBold|Bold|ExtraBold|UltraBold|Black|Heavy|Italic|Oblique|Variable|VF)' + r'(?:[-_ ]?(?:Italic|Oblique))?$', '', name, flags=re.IGNORECASE ) # Replace dashes/underscores with spaces diff --git a/routes/gallery/gallery_routes.py b/routes/gallery/gallery_routes.py index 690c22044..8094afa9f 100644 --- a/routes/gallery/gallery_routes.py +++ b/routes/gallery/gallery_routes.py @@ -2433,11 +2433,10 @@ def setup_gallery_routes() -> APIRouter: image_b64 = body.get("image") amount = body.get("amount", 50) / 100.0 - from PIL import Image, ImageFilter + from PIL import ImageFilter import base64, io - img_bytes = _b64_bytes(image_b64) - img = Image.open(io.BytesIO(img_bytes)).convert("RGB") + img = _b64_to_pil_image(image_b64, mode="RGB") # Unsharp mask: radius=2, percent=amount*200, threshold=3 sharpened = img.filter(ImageFilter.UnsharpMask(radius=2, percent=int(amount * 200), threshold=3)) @@ -2472,8 +2471,7 @@ def setup_gallery_routes() -> APIRouter: except ImportError: raise HTTPException(500, "Server missing a required dependency") # Decode source image (RGB; Real-ESRGAN doesn't preserve alpha). - img_bytes = _b64_bytes(image_b64) - src = Image.open(io.BytesIO(img_bytes)).convert("RGB") + src = _b64_to_pil_image(image_b64, mode="RGB") try: patch_realesrgan_torchvision_compat() from realesrgan import RealESRGANer @@ -2525,8 +2523,7 @@ def setup_gallery_routes() -> APIRouter: import numpy as np except ImportError: raise HTTPException(500, "Server missing a required dependency") - img_bytes = _b64_bytes(image_b64) - src = Image.open(io.BytesIO(img_bytes)).convert("RGB") + src = _b64_to_pil_image(image_b64, mode="RGB") try: patch_realesrgan_torchvision_compat() from basicsr.archs.rrdbnet_arch import RRDBNet @@ -2778,8 +2775,7 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image import base64, io - img_bytes = _b64_bytes(image_b64) - img = Image.open(io.BytesIO(img_bytes)).convert("RGBA") + img = _b64_to_pil_image(image_b64) W, H = img.size hint = None @@ -2868,8 +2864,7 @@ def setup_gallery_routes() -> APIRouter: from PIL import Image, ImageFilter, ImageEnhance import numpy as np - img_bytes = _b64_bytes(image_b64) - img = Image.open(io.BytesIO(img_bytes)).convert("RGB") + img = _b64_to_pil_image(image_b64, mode="RGB") # Try GFPGAN first (AI face restoration) try: diff --git a/routes/note/note_routes.py b/routes/note/note_routes.py index 249453614..cabf6a2bb 100644 --- a/routes/note/note_routes.py +++ b/routes/note/note_routes.py @@ -728,32 +728,35 @@ def setup_note_routes(task_scheduler=None, upload_handler=None): ) if body.title is not None: note.title = body.title - if body.content is not None: + # Omitted fields preserve their values; explicit null clears the + # nullable fields used by the editor's removal controls. + supplied = body.model_fields_set + if "content" in supplied: note.content = body.content - if body.items is not None: - note.items = json.dumps(body.items) + if "items" in supplied: + note.items = json.dumps(body.items) if body.items is not None else None flag_modified(note, "items") if body.note_type is not None: note.note_type = body.note_type - if body.color is not None: + if "color" in supplied: note.color = body.color - if body.label is not None: + if "label" in supplied: note.label = body.label if body.pinned is not None: note.pinned = body.pinned if body.archived is not None: note.archived = body.archived - if body.due_date is not None: + if "due_date" in supplied: note.due_date = body.due_date - if body.image_url is not None: + if "image_url" in supplied: note.image_url = body.image_url - if body.gallery_id is not None: + if "gallery_id" in supplied: note.gallery_id = body.gallery_id if body.repeat is not None: note.repeat = body.repeat if body.sort_order is not None: note.sort_order = body.sort_order - if body.agent_session_id is not None: + if "agent_session_id" in supplied: note.agent_session_id = body.agent_session_id db.commit() diff --git a/routes/session_routes.py b/routes/session_routes.py index 1c903f194..388de5b20 100644 --- a/routes/session_routes.py +++ b/routes/session_routes.py @@ -1315,7 +1315,7 @@ def setup_session_routes( def auto_sort_sessions(request: Request, skip_llm: bool = False): """Use AI to categorize all sessions into folders. - Phase 1 deletes empty/throwaway sessions and Phase 2 asks the LLM + Phase 1 deletes empty/Incognito sessions and Phase 2 asks the LLM to assign folders. When `skip_llm=true` the endpoint returns after Phase 1 — used by the "Tidy (no AI)" UI affordance so users can clean junk without spending tokens. @@ -1326,21 +1326,11 @@ def setup_session_routes( single_user_mode = not user and _auth_disabled() user_sessions = session_manager.get_sessions_for_user(user) - # Delete empty and throwaway sessions before sorting + # Saved conversations are never disposable based on their title or prompt. from core.database import ChatMessage as DbMsg db = SessionLocal() deleted_empty = 0 deleted_throwaway = 0 - # Names that indicate a throwaway/test session (case-insensitive exact or prefix match) - _THROWAWAY_NAMES = { - "test", "testing", "asdf", "asd", "hello", "hi", "hey", - "yo", "sup", "hola", "hii", "hiii", "heyo", - "foo", "bar", "baz", "tmp", "temp", "scratch", "untitled", - "new chat", "delete", "remove", "junk", "trash", "xxx", - "abc", "qwerty", "blah", "stuff", "whatever", "idk", - "ok", "lol", "bruh", "hmm", "hm", "meh", - } - _THROWAWAY_MAX_MESSAGES = 4 # only delete if <= this many messages try: rows_q = db.query(DbSession).filter(DbSession.archived == False) if user: @@ -1349,15 +1339,11 @@ def setup_session_routes( rows_q = rows_q.filter(DbSession.owner == user) rows = rows_q.limit(2000).all() folder_map = {r.id: r.folder for r in rows} - # Precompute per-session message counts in TWO aggregate queries + # Precompute per-session message counts in one aggregate query # instead of 1–3 queries PER session — with many chats the per-row # loop was doing thousands of round-trips and blowing the timeout. from sqlalchemy import func as _sa_func _counts = dict(db.query(DbMsg.session_id, _sa_func.count(DbMsg.id)).group_by(DbMsg.session_id).all()) - _asst_counts = dict( - db.query(DbMsg.session_id, _sa_func.count(DbMsg.id)) - .filter(DbMsg.role == "assistant").group_by(DbMsg.session_id).all() - ) cleanup_now = utcnow_naive() for row in rows: # Never delete important sessions @@ -1367,9 +1353,10 @@ def setup_session_routes( if (row.name or "").strip() == "Incognito": should_delete = True deleted_throwaway += 1 - db.delete(row) if hasattr(session_manager, 'delete_session'): session_manager.delete_session(row.id) + else: + db.delete(row) continue if is_session_recently_active(row, now=cleanup_now): continue @@ -1378,31 +1365,11 @@ def setup_session_routes( if msg_count == 0: should_delete = True deleted_empty += 1 - elif msg_count <= _THROWAWAY_MAX_MESSAGES: - name = (row.name or "").strip().lower() - # Check first user message content (AI renames sessions, so - # "hi" becomes "Casual Greeting Exchange" — name alone won't match) - first_msg = db.query(DbMsg.content).filter( - DbMsg.session_id == row.id, DbMsg.role == "user" - ).order_by(DbMsg.timestamp).first() - first_text = (first_msg[0] or "").strip().lower() if first_msg else "" - # Count assistant messages — if user sent something but AI never replied, it's dead - assistant_count = _asst_counts.get(row.id, 0) - if name in _THROWAWAY_NAMES or name.startswith("chat:") or first_text in _THROWAWAY_NAMES: - should_delete = True - deleted_throwaway += 1 - # Single user message with no AI response = dead session - elif msg_count == 1 and assistant_count == 0: - should_delete = True - deleted_throwaway += 1 - # Short phrase (1-3 words) with no real AI conversation (<=2 msgs) - elif msg_count <= 2 and first_text and len(first_text.split()) <= 3 and len(first_text) <= 40: - should_delete = True - deleted_throwaway += 1 if should_delete: - db.delete(row) if hasattr(session_manager, 'delete_session'): session_manager.delete_session(row.id) + else: + db.delete(row) if deleted_empty or deleted_throwaway: db.commit() logger.info(f"Auto-sort: deleted {deleted_empty} empty + {deleted_throwaway} throwaway sessions") diff --git a/routes/task/task_routes.py b/routes/task/task_routes.py index 49f65b313..0b7093d15 100644 --- a/routes/task/task_routes.py +++ b/routes/task/task_routes.py @@ -20,7 +20,7 @@ from src.task_action_policy import ( is_admin_only_task_action, owner_has_admin_task_privileges, ) -from src.task_scheduler import compute_next_run, HOUSEKEEPING_DEFAULTS +from src.task_scheduler import compute_next_run, is_valid_cron, HOUSEKEEPING_DEFAULTS from routes.prefs_routes import _load_for_user, _save_for_user logger = logging.getLogger(__name__) @@ -475,10 +475,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: if req.trigger_type == "schedule" and req.schedule == "cron" and not req.cron_expression: raise HTTPException(400, "Cron expression is required for cron schedule") if req.trigger_type == "schedule" and req.schedule == "cron" and req.cron_expression: - try: - from croniter import croniter - croniter(req.cron_expression) - except Exception: + if not is_valid_cron(req.cron_expression): raise HTTPException(400, "Invalid cron expression") if req.trigger_type == "event" and not req.trigger_event: raise HTTPException(400, "Event name is required for event-triggered tasks") @@ -788,10 +785,7 @@ def setup_task_routes(task_scheduler) -> APIRouter: task.character_id = req.character_id or None if req.cron_expression is not None: if req.cron_expression: - try: - from croniter import croniter - croniter(req.cron_expression) - except Exception: + if not is_valid_cron(req.cron_expression): raise HTTPException(400, "Invalid cron expression") task.cron_expression = req.cron_expression or None diff --git a/scripts/odysseus-gallery b/scripts/odysseus-gallery index ab892d798..20e210f04 100755 --- a/scripts/odysseus-gallery +++ b/scripts/odysseus-gallery @@ -6,6 +6,7 @@ that are awkward at the shell). Filters images by tag, prompt, favorite, album, and surfaces EXIF metadata. odysseus-gallery list --limit 20 --favorites + odysseus-gallery list --tag "sunset,beach" # all tags, manual or AI odysseus-gallery show IMAGE_ID odysseus-gallery albums | jq -r '.[] | .name' odysseus-gallery search "sunset" # matches prompt + tags @@ -83,7 +84,13 @@ def cmd_list(args): fail(f"no album named {args.album!r}") q = q.filter(GalleryImage.album_id == al.id) if args.tag: - q = q.filter(GalleryImage.tags.ilike(f"%{args.tag}%")) + for tag in (value.strip() for value in args.tag.split(",")): + if not tag: + continue + q = q.filter( + GalleryImage.tags.ilike(f"%{tag}%") | + GalleryImage.ai_tags.ilike(f"%{tag}%") + ) q = q.order_by(GalleryImage.created_at.desc()).limit(args.limit) emit([_serialize_image(i) for i in q.all()], args) finally: @@ -159,7 +166,7 @@ def _build_parser(): pl.add_argument("--limit", type=int, default=50) pl.add_argument("--favorites", action="store_true") pl.add_argument("--album", help="filter by album name") - pl.add_argument("--tag", help="substring match against tags column") + pl.add_argument("--tag", help="comma-separated tags; each matches manual or AI tags by substring") pl.set_defaults(func=cmd_list) psh = sub.add_parser("show", parents=[common]) diff --git a/scripts/odysseus-mail b/scripts/odysseus-mail index e63d9f166..443541df9 100755 --- a/scripts/odysseus-mail +++ b/scripts/odysseus-mail @@ -43,7 +43,7 @@ import logging import os import re import sys -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path # Anchor the path so the script works when invoked from anywhere. @@ -136,7 +136,7 @@ def cmd_list(args) -> None: emit([], args) return all_uids = data[0].split() - # Newest first, limit + # Keep a bounded arrival batch; order its decoded rows by Date below. uids = list(reversed(all_uids))[: args.limit] out = [] for uid in uids: @@ -165,10 +165,15 @@ def cmd_list(args) -> None: sender_name, sender_addr = parseaddr(sender) date_raw = msg.get("Date", "") try: - iso = parsedate_to_datetime(date_raw).isoformat() if date_raw else "" + parsed_date = parsedate_to_datetime(date_raw) if date_raw else None + iso = parsed_date.isoformat() if parsed_date else "" + if parsed_date is not None and parsed_date.tzinfo is None: + parsed_date = parsed_date.replace(tzinfo=timezone.utc) + date_epoch = parsed_date.timestamp() if parsed_date else float("-inf") except Exception: iso = "" - out.append({ + date_epoch = float("-inf") + out.append((date_epoch, { "uid": uid.decode(), "date": iso, "from_addr": sender_addr, @@ -176,10 +181,11 @@ def cmd_list(args) -> None: "subject": subject, "is_read": "\\Seen" in flags, "is_answered": "\\Answered" in flags, - }) + })) except Exception as e: sys.stderr.write(f"warn: skipping uid {uid!r}: {e}\n") - emit(out, args) + out.sort(key=lambda item: item[0], reverse=True) + emit([row for _, row in out], args) # ─── read ──────────────────────────────────────────────────────────── diff --git a/scripts/verify_notes_browser_runtime.py b/scripts/verify_notes_browser_runtime.py index 5fedbaf33..679e3f111 100644 --- a/scripts/verify_notes_browser_runtime.py +++ b/scripts/verify_notes_browser_runtime.py @@ -11,6 +11,38 @@ import time from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def _stop_owned_server(server, identity): + """Close only identity-bound descendants, including MCP's new sessions. + + Capture before signalling the parent: an orphaned stderr holder can no + longer be attributed by ancestry. Never search by executable name or kill + the caller's process group. + """ + from src import process_lifecycle as lifecycle + + descendants = (lifecycle.bind_descendants([identity.pid]) + if identity.verdict() == lifecycle.OWNED else []) + if os.name == "nt": + if identity.verdict() == lifecycle.OWNED: + lifecycle.taskkill_tree(identity.pid) + else: + outcome = lifecycle.terminate_identities( + [*(entry.identity for entry in reversed(descendants)), identity], + steps=lifecycle.term_kill_steps(10), + ) + if outcome.survivors or outcome.unverified: + raise RuntimeError(f"Disposable Notes cleanup could not verify exit: {outcome}") + try: + logs, _ = server.communicate(timeout=2) + except subprocess.TimeoutExpired as error: + # Do not turn an unidentified pipe holder into an unbounded wait. + server.stdout.close() + raise RuntimeError("Disposable Notes output pipe remained open after owned cleanup") from error + return logs def main(): @@ -36,7 +68,10 @@ def main(): server = subprocess.Popen( [sys.executable, "-m", "uvicorn", "app:app", "--host", "127.0.0.1", "--port", str(port)], cwd=target, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + start_new_session=(os.name != "nt"), ) + from src.process_lifecycle import ProcessIdentity + server_identity = ProcessIdentity.capture(server.pid) result = None try: origin = f"http://127.0.0.1:{port}" @@ -62,12 +97,7 @@ def main(): print(result.stderr, file=sys.stderr) return result.returncode finally: - server.terminate() - try: - logs, _ = server.communicate(timeout=10) - except subprocess.TimeoutExpired: - server.kill() - logs, _ = server.communicate() + logs = _stop_owned_server(server, server_identity) if result is None or result.returncode: print(logs[-8000:], file=sys.stderr) diff --git a/services/hwfit/hf_discovery.py b/services/hwfit/hf_discovery.py index 3bea44931..c630ce115 100644 --- a/services/hwfit/hf_discovery.py +++ b/services/hwfit/hf_discovery.py @@ -202,17 +202,35 @@ def _infer_use_case(repo_id, pipeline_tag): return "general" +# HF collections mix in models no Cookbook LLM backend can serve (vision +# encoders, time-series forecasters, image/video generators). Image models +# have their own registry, so keep these out of the LLM catalog. +_NON_LLM_PIPELINE_TAGS = frozenset({ + "time-series-forecasting", + "zero-shot-image-classification", + "image-classification", + "image-feature-extraction", + "text-to-image", + "image-to-video", + "text-to-video", +}) + + def _entry_from_collection_item(collection, item, source): repo_id = item.get("id") or "" if item.get("type") != "model" or not repo_id: return None + if (item.get("pipeline_tag") or "").strip().lower() in _NON_LLM_PIPELINE_TAGS: + return None repo_prefix = source.get("repo_prefix") if repo_prefix and not repo_id.startswith(repo_prefix): return None raw_params = item.get("numParameters") or 0 - active = None + # Read the name even when HF gives a total: it is the only source of the + # MoE active size (e.g. "-35B-A3B"), which the speed estimate depends on. + name_params, active = _parse_params_from_name(repo_id) if not raw_params: - raw_params, active = _parse_params_from_name(repo_id) + raw_params = name_params param_label, raw_params = _format_params(raw_params) if not raw_params: return None diff --git a/services/memory/skill_format.py b/services/memory/skill_format.py index 633f4bec5..56a30a919 100644 --- a/services/memory/skill_format.py +++ b/services/memory/skill_format.py @@ -84,10 +84,14 @@ _FM_KEY_RE = re.compile(r"^([a-z_][a-z0-9_]*):\s*(.*)$", re.IGNORECASE) _FM_BLOCK_LIST_RE = re.compile(r"^\s*-\s*(.*)$") -def _parse_scalar(raw: str) -> Any: +def _parse_scalar(raw: str, *, string_only: bool = False) -> Any: raw = raw.strip() if raw == "": return "" + # String fields keep their spelling; quoted values still use the decoder + # below so existing escapes round-trip as before. + if string_only and not (raw[0] == raw[-1] and raw[0] in ("'", '"')): + return raw if raw.startswith("[") and raw.endswith("]"): inner = raw[1:-1].strip() if not inner: @@ -171,7 +175,7 @@ def parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]: pending_key = key fm[key] = [] else: - fm[key] = _parse_scalar(val) + fm[key] = _parse_scalar(val, string_only=(key == "version")) pending_key = None continue m2 = _FM_BLOCK_LIST_RE.match(line) diff --git a/src/ai_interaction.py b/src/ai_interaction.py index dd772eff1..90294774a 100644 --- a/src/ai_interaction.py +++ b/src/ai_interaction.py @@ -1207,7 +1207,7 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne Line 1: prompt describing the image Line 2: model name (optional, default auto-detects: prefers gpt-image-1.5 > gpt-image-1) Line 3: size (optional, defaults to 1024x1024) - Line 4: quality (optional, defaults to medium — options: low, medium, high, auto) + Line 4: quality (optional, defaults to configured quality or medium — low, medium, high, auto) """ import base64 import httpx @@ -1225,13 +1225,13 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne prompt = str(args.get('prompt') or '').strip() model_spec = str(args.get('model') or '').strip() size = str(args.get('size') or '1024x1024') - quality = str(args.get('quality') or 'medium') + quality = str(args.get('quality') or '') else: lines = content.strip().split("\n") prompt = lines[0].strip() if lines else "" model_spec = lines[1].strip() if len(lines) > 1 and lines[1].strip() else "" size = lines[2].strip() if len(lines) > 2 and lines[2].strip() else "1024x1024" - quality = lines[3].strip() if len(lines) > 3 and lines[3].strip() else "medium" + quality = lines[3].strip() if len(lines) > 3 and lines[3].strip() else "" if not prompt: return {"error": "Image prompt is required (line 1)"} @@ -1249,8 +1249,8 @@ async def do_generate_image(content: str, session_id: Optional[str] = None, owne # Use admin-configured model/quality if not specified by the tool call if not model_spec: model_spec = _settings.get("image_model", "") - if quality == "medium" and _settings.get("image_quality"): - quality = _settings["image_quality"] + if not quality: + quality = _settings.get("image_quality") or "medium" # Auto-detect best available image model if still not set if not model_spec: @@ -1482,7 +1482,7 @@ async def do_edit_image( session_id: Optional[str] = None, owner: Optional[str] = None, size: str = "auto", - quality: str = "medium", + quality: Optional[str] = None, progress_callback: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, incognito: bool = False, ) -> Dict: @@ -1509,8 +1509,8 @@ async def do_edit_image( if not model_spec: model_spec = _settings.get("image_model", "") - if quality == "medium" and _settings.get("image_quality"): - quality = _settings["image_quality"] + if quality is None: + quality = _settings.get("image_quality") or "medium" if not model_spec: return {"error": "No image model selected for image editing"} diff --git a/src/llm_core.py b/src/llm_core.py index bde17741e..861a0512d 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -432,12 +432,15 @@ class _DegenerateStreamGuard: "Summer Summer ..."). This is not a useful response and can burn context, browser memory, and GPU time. Keep the guard conservative: only fire on long same-token runs or a very dominant repeated token in the recent window. + Repetition made only of numbers is data (a zero or identity matrix, a + padded buffer), so it only stops at a long all-numeric backstop. """ def __init__(self, model: str): self.model = model or "model" self.last_token = "" self.same_run = 0 + self.numeric_run = 0 self.recent_tokens: List[str] = [] self.periodic_tokens: List[str] = [] self.total_chars = 0 @@ -458,6 +461,7 @@ class _DegenerateStreamGuard: else: self.last_token = token self.same_run = 1 + self.numeric_run = self.numeric_run + 1 if token.isdigit() else 0 self.recent_tokens.append(token) self.periodic_tokens.append(token) if len(self.recent_tokens) > 96: @@ -466,14 +470,20 @@ class _DegenerateStreamGuard: self.periodic_tokens = self.periodic_tokens[-2048:] reason = None - if self.same_run >= 28 and self.total_chars >= 100: + if self.same_run >= 28 and self.total_chars >= 100 and not self.last_token.isdigit(): reason = f"repeated '{self.last_token}' {self.same_run} times" elif len(self.recent_tokens) >= 72: top = max(set(self.recent_tokens), key=self.recent_tokens.count) count = self.recent_tokens.count(top) - if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78: + if count >= 60 and count / max(len(self.recent_tokens), 1) >= 0.78 and not top.isdigit(): reason = f"repeated '{top}' {count}/{len(self.recent_tokens)} recent tokens" - if not reason and len(self.recent_tokens) >= 48: + if ( + not reason + and len(self.recent_tokens) >= 48 + # A window that is mostly numbers is matrix/table data; its + # periodicity is real. Mixed loops still hit the exact-block check. + and sum(t.isdigit() for t in self.recent_tokens) * 2 < len(self.recent_tokens) + ): # Detect a periodic suffix instead of counting repeated n-grams. # Reused property names are normal in CSS/SVG/JSON; a collapsed # generation repeats nearly every token at a short fixed period. @@ -510,10 +520,13 @@ class _DegenerateStreamGuard: period = current_anchor - previous if period < 24 or len(history) < period * 2: continue - if history[-2 * period:-period] == history[-period:]: + if history[-2 * period:-period] == history[-period:] and not all(t.isdigit() for t in history[-period:]): reason = f"repeated an exact {period}-token block twice" break + if not reason and self.numeric_run >= 2048: + reason = f"emitted {self.numeric_run} numbers in a row" + if not reason: return None diff --git a/src/mcp_manager.py b/src/mcp_manager.py index 53112009d..b47d0e1d7 100644 --- a/src/mcp_manager.py +++ b/src/mcp_manager.py @@ -10,6 +10,7 @@ import logging import os import re import asyncio +import contextlib from typing import Any, Dict, List, Optional, Set, Tuple from src.database import McpServer, SessionLocal @@ -383,12 +384,21 @@ class McpManager: provider = build_provider(server_id, url, on_redirect=_on_redirect) stack = AsyncExitStack() - transport = await stack.enter_async_context(streamablehttp_client(url, auth=provider)) - read_stream, write_stream, _get_session_id = transport - session = await stack.enter_async_context(ClientSession(read_stream, write_stream)) - await session.initialize() + try: + transport = await stack.enter_async_context(streamablehttp_client(url, auth=provider)) + read_stream, write_stream, _get_session_id = transport + session = await stack.enter_async_context(ClientSession(read_stream, write_stream)) + await session.initialize() - tools_result = await session.list_tools() + tools_result = await session.list_tools() + except BaseException: + # Close the half-open transport here, in the task that entered + # it. Left for the garbage collector, its AnyIO cancel scope is + # exited from the wrong task and the event loop keeps spinning + # (#5518). Its own teardown error would only mask the real one. + with contextlib.suppress(Exception): + await stack.aclose() + raise tools = [] for tool in tools_result.tools: tools.append({ diff --git a/src/model_discovery.py b/src/model_discovery.py index 116951f9d..71b24d558 100644 --- a/src/model_discovery.py +++ b/src/model_discovery.py @@ -193,7 +193,10 @@ class ModelDiscovery: data = r.json() # Some OpenAI-compatible servers return a bare list, not {"data": [...]}. items = data if isinstance(data, list) else ((data or {}).get("data") or []) - ids = [m.get("id") for m in items if isinstance(m, dict) and m.get("id")] + ids = [ + m["id"] for m in items + if isinstance(m, dict) and isinstance(m.get("id"), str) and m["id"] + ] if ids: return { "host": host, diff --git a/src/pdf_form_doc.py b/src/pdf_form_doc.py index 8b8e210b8..4bc1666cc 100644 --- a/src/pdf_form_doc.py +++ b/src/pdf_form_doc.py @@ -102,8 +102,10 @@ _PLAIN_FRONT_MATTER_RE = re.compile( # - **label:** value # - **label** [opts]: value # - [x] **label** +# Accept Unicode word characters for older saved markers, whose encoder left +# non-ASCII letters/numbers literal. Newly rendered names are ASCII-safe. _FIELD_BULLET_RE = re.compile( - r'^\s*-\s+(?P.*?)\s*\s*$' + r'^\s*-\s+(?P.*?)\s*\s*$' ) @@ -115,7 +117,7 @@ def _encode_name(name: str) -> str: """ out = [] for ch in name or "": - if ch.isalnum() or ch in ("_", ".", "-"): + if (ch.isascii() and ch.isalnum()) or ch in ("_", ".", "-"): out.append(ch) else: for b in ch.encode("utf-8"): diff --git a/src/session_actions.py b/src/session_actions.py index 1efa484e3..9859ef486 100644 --- a/src/session_actions.py +++ b/src/session_actions.py @@ -12,16 +12,6 @@ from datetime import datetime, timedelta, timezone logger = logging.getLogger(__name__) -# Names that indicate a throwaway/test session -_THROWAWAY_NAMES = { - "test", "testing", "asdf", "asd", "hello", "hi", "hey", - "yo", "sup", "hola", "hii", "hiii", "heyo", - "foo", "bar", "baz", "tmp", "temp", "scratch", "untitled", - "new chat", "delete", "remove", "junk", "trash", "xxx", - "abc", "qwerty", "blah", "stuff", "whatever", "idk", - "ok", "lol", "bruh", "hmm", "hm", "meh", -} -_THROWAWAY_MAX_MESSAGES = 4 _FRESH_EMPTY_SESSION_GRACE = timedelta(minutes=10) _FRESH_SESSION_GRACE = _FRESH_EMPTY_SESSION_GRACE @@ -54,14 +44,15 @@ def is_session_recently_active(row, now=None, grace=_FRESH_SESSION_GRACE) -> boo async def run_auto_sort(owner: str, skip_llm: bool = False, delete_throwaway: bool = True) -> str: - """Run session cleanup + (optional) AI folder sort for the given owner. + """Remove empty/Incognito sessions and optionally organize saved chats. Args: owner: user whose sessions to process - skip_llm: when True, do only Phase 1 (delete empty/throwaway sessions); + skip_llm: when True, do only Phase 1 (delete empty/Incognito sessions); skip Phase 2 (AI folder assignment). Used by the built-in daily background sweep so it never burns LLM tokens. - delete_throwaway: when False, only empty/incognito sessions are deleted. + delete_throwaway: retained for caller compatibility. Ordinary sessions + with saved messages are always preserved, regardless of this flag. Returns a human-readable summary of what was done. """ @@ -71,7 +62,7 @@ async def run_auto_sort(owner: str, skip_llm: bool = False, delete_throwaway: bo db = SessionLocal() try: - # ── Phase 1: Delete empty/throwaway sessions ── + # ── Phase 1: Delete only empty and explicitly Incognito sessions ── deleted_empty = 0 deleted_throwaway = 0 @@ -93,47 +84,13 @@ async def run_auto_sort(owner: str, skip_llm: bool = False, delete_throwaway: bo if is_session_recently_active(row, now=cleanup_now): continue - msg_count = db.query(DbMsg.id).filter( + has_messages = db.query(DbMsg.id).filter( DbMsg.session_id == row.id - ).limit(_THROWAWAY_MAX_MESSAGES + 1).count() - should_delete = False - - if msg_count == 0: + ).first() is not None + if not has_messages: if is_fresh: continue - should_delete = True deleted_empty += 1 - elif delete_throwaway and msg_count <= _THROWAWAY_MAX_MESSAGES: - name = (row.name or "").strip().lower() - first_msg = db.query(DbMsg.content).filter( - DbMsg.session_id == row.id, DbMsg.role == "user" - ).order_by(DbMsg.timestamp).first() - first_text = (first_msg[0] or "").strip().lower() if first_msg else "" - assistant_count = db.query(DbMsg.id).filter( - DbMsg.session_id == row.id, DbMsg.role == "assistant" - ).limit(1).count() - - if name in _THROWAWAY_NAMES or name.startswith("chat:") or first_text in _THROWAWAY_NAMES: - should_delete = True - deleted_throwaway += 1 - elif msg_count == 1 and assistant_count == 0: - should_delete = True - deleted_throwaway += 1 - elif msg_count <= 4 and first_text and len(first_text.split()) <= 8 and len(first_text) <= 80: - # Short trivial chats — e.g. "write hi to a friend" → "Hi!" - should_delete = True - deleted_throwaway += 1 - else: - # Aggressive: total message text under 250 chars combined = trivial - msg_rows = db.query(DbMsg.content).filter( - DbMsg.session_id == row.id - ).all() - total_chars = sum(len(m[0] or "") for m in msg_rows) - if total_chars <= 250: - should_delete = True - deleted_throwaway += 1 - - if should_delete: db.delete(row) if deleted_empty or deleted_throwaway: diff --git a/src/task_scheduler.py b/src/task_scheduler.py index 12e6ed61b..06ecf8ed5 100644 --- a/src/task_scheduler.py +++ b/src/task_scheduler.py @@ -132,6 +132,23 @@ async def _cached(key: Tuple, ttl: float, fetch: Callable[[], Awaitable[Any]]) - _shared_cache_pending.pop(key, None) +def is_valid_cron(cron_expression: str) -> bool: + """True for a standard 5-field cron expression that croniter can parse. + + croniter also accepts 6- and 7-field expressions with a seconds field, but + the scheduler works at one-minute resolution and would run such a task + every second. + """ + if len((cron_expression or "").split()) != 5: + return False + try: + from croniter import croniter + croniter(cron_expression) + except Exception: + return False + return True + + def compute_next_run(schedule: str, scheduled_time: str, scheduled_day: int = None, scheduled_date: datetime = None, @@ -175,6 +192,9 @@ def compute_next_run(schedule: str, scheduled_time: str, return dt.astimezone(timezone.utc).replace(tzinfo=None) if schedule == "cron" and cron_expression: + if not is_valid_cron(cron_expression): + logger.warning(f"Invalid cron expression '{cron_expression}'") + return None try: from croniter import croniter cron = croniter(cron_expression, now) @@ -306,7 +326,7 @@ def _digest_windows(now): def _checkin_calendar_events(db, owner, start, end): - """Calendar events in [start, end] for ONE owner, for the check-in digest. + """Calendar events in [start, end) for ONE owner, for the check-in digest. Ownership lives on CalendarCal.owner; events inherit it via calendar_id. The digest query had no owner scope, so it pulled EVERY user's events into @@ -320,7 +340,7 @@ def _checkin_calendar_events(db, owner, start, end): .filter( _CC.owner == owner, _CE.dtstart >= start, - _CE.dtstart <= end, + _CE.dtstart < end, _CE.status != "cancelled", ) .order_by(_CE.dtstart) diff --git a/src/tool_schemas.py b/src/tool_schemas.py index 6468f4f45..176657243 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -41,6 +41,7 @@ _REQUIRED_NATIVE_TOOL_ARGS = { "write_file": ("path",), "edit_file": ("path",), "apply_patch": ("patch_text", "patchText", "patch"), + "generate_image": ("prompt",), "host_shell": ("command",), } diff --git a/static/css/01-agent-chat.css b/static/css/01-agent-chat.css index 9a5a836cb..87a17a117 100644 --- a/static/css/01-agent-chat.css +++ b/static/css/01-agent-chat.css @@ -4592,7 +4592,9 @@ body.bg-pattern-ascii-fireflies { width:100%; background:var(--bg); color:var(--fg); border:1px solid var(--border); border-radius:6px; padding:8px; font-family:inherit; font-size:0.95rem; - resize:vertical; outline:none; + resize:both; outline:none; + field-sizing:content; + max-width:100%; box-sizing:border-box; min-height:80px; } .edit-textarea:focus { border-color:var(--red); } diff --git a/static/css/17-research.css b/static/css/17-research.css index c4de5cb75..2c43aa2c8 100644 --- a/static/css/17-research.css +++ b/static/css/17-research.css @@ -2690,11 +2690,14 @@ body.theme-frosted .modal { max-height: 95dvh !important; } + /* Notes and Documents must respect the same viewport and safe-area + top as other mobile full-height sheets. Keep these authoritative rules + consistent with 05-documents.css and 14-email.css. */ body.doc-view .doc-editor-pane, body.notes-view .notes-pane { - inset: auto 0 0 !important; - height: 95dvh !important; - max-height: 95dvh !important; + inset: var(--sheet-top) 0 0 !important; + height: calc(var(--app-height) - var(--sheet-top)) !important; + max-height: calc(var(--app-height) - var(--sheet-top)) !important; } } diff --git a/static/index.html b/static/index.html index aafacc2e1..df3045af3 100644 --- a/static/index.html +++ b/static/index.html @@ -16,13 +16,6 @@