Squash Odysseus development history

routes/auth_routes.py

@@ -736,6 +736,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
         _INT_RANGES = {
             "agent_max_rounds": (1, 200),
             "agent_max_tool_calls": (0, 1000),  # 0 = unlimited
+            "auto_compact_threshold_percent": (50, 95),
         }
         for key in DEFAULT_SETTINGS:
             if key in RETIRED_SETTING_KEYS:

routes/calendar_routes.py

@@ -4,7 +4,7 @@ import logging
 import json
 import re
 import uuid
-from datetime import datetime, date, timedelta
+from datetime import datetime, date, timedelta, timezone
 from typing import Optional, List
 
 from fastapi import APIRouter, HTTPException, Request, UploadFile, File
@@ -13,7 +13,7 @@ from sqlalchemy import or_, and_
 from sqlalchemy.exc import IntegrityError
 from dateutil.rrule import rrulestr
 
-from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
+from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent, Note
 from src.auth_helpers import effective_user, require_user
 from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
 from src.upload_handler import reserve_upload_references
@@ -207,6 +207,7 @@ class EventCreate(BaseModel):
     calendar_href: Optional[str] = None  # calendar id
     rrule: Optional[str] = None
     color: Optional[str] = None  # per-event color override
+    reminder_minutes: Optional[int] = None
 
 
 class EventUpdate(BaseModel):
@@ -218,6 +219,7 @@ class EventUpdate(BaseModel):
     location: Optional[str] = None
     rrule: Optional[str] = None
     color: Optional[str] = None
+    reminder_minutes: Optional[int] = None
 
 
 # ── Helpers ──
@@ -621,7 +623,133 @@ def _parse_dt(s: str) -> datetime:
         raise ValueError(f"could not parse datetime: {s!r}")
 
 
-def _event_to_dict(ev: CalendarEvent) -> dict:
+def _note_due_datetime(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        text = str(value).strip()
+        if text.endswith("Z"):
+            text = text[:-1] + "+00:00"
+        due = datetime.fromisoformat(text)
+        if due.tzinfo is not None:
+            return due.astimezone(timezone.utc).replace(tzinfo=None)
+        return due
+    except Exception:
+        return None
+
+
+def _calendar_reminder_for_event(db, owner: str, ev: CalendarEvent) -> dict | None:
+    """Return the closest Notes reminder that belongs to this calendar event.
+
+    Calendar alarms are currently stored as Notes rows. Older rows do not carry
+    an event UID, so match conservatively by the generated title plus due_date
+    before the event start. This keeps existing reminder notes visible on the
+    calendar without a schema migration.
+    """
+    if not db or not owner or not ev or not ev.dtstart:
+        return None
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return None
+
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    if not notes:
+        return None
+
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    best = None
+    best_minutes = None
+    for note in notes:
+        due = _note_due_datetime(note.due_date)
+        if due is None:
+            continue
+        minutes = round((start - due).total_seconds() / 60)
+        if minutes < 0 or minutes > 7 * 24 * 60:
+            continue
+        if best is None or minutes < best_minutes:
+            best = note
+            best_minutes = minutes
+    if best is None:
+        return None
+    return {
+        "note_id": best.id,
+        "due_date": best.due_date,
+        "minutes": best_minutes,
+    }
+
+
+def _delete_calendar_reminders_for_event(db, owner: str, ev: CalendarEvent) -> int:
+    if not db or not owner or not ev:
+        return 0
+    summary = (ev.summary or "").strip()
+    if not summary:
+        return 0
+    titles = [f"Calendar reminder: {summary}", f"Reminder: {summary}"]
+    notes = (
+        db.query(Note)
+        .filter(
+            Note.owner == owner,
+            Note.archived == False,  # noqa: E712
+            Note.label == "calendar",
+            Note.source == "calendar",
+            Note.title.in_(titles),
+            Note.due_date.isnot(None),
+        )
+        .all()
+    )
+    for note in notes:
+        db.delete(note)
+    return len(notes)
+
+
+def _create_calendar_reminder_for_event(db, owner: str, ev: CalendarEvent, minutes_before: int) -> dict:
+    if not owner or not ev or not ev.dtstart:
+        return {"note_id": None, "skipped_reason": "missing event"}
+    minutes_before = max(0, int(minutes_before))
+    start = ev.dtstart
+    if getattr(start, "tzinfo", None) is not None:
+        start = start.astimezone(timezone.utc).replace(tzinfo=None)
+    remind_at = start - timedelta(minutes=minutes_before)
+    now = datetime.utcnow() if getattr(ev, "is_utc", False) else datetime.now()
+    if start <= now:
+        return {"note_id": None, "skipped_reason": "event already passed"}
+    if remind_at <= now:
+        remind_at = now
+
+    summary = (ev.summary or "(no title)").strip() or "(no title)"
+    location = (ev.location or "").strip()
+    start_fmt = start.strftime("%a %b %d") if ev.all_day else start.strftime("%a %b %d %H:%M")
+    loc = f" @ {location}" if location else ""
+    due_date = remind_at.isoformat() + ("Z" if getattr(ev, "is_utc", False) and not ev.all_day else "")
+    note = Note(
+        id=str(uuid.uuid4()),
+        owner=owner,
+        title=f"Calendar reminder: {summary}",
+        items=json.dumps([{"text": f"{summary}{loc} — {start_fmt}", "done": False, "checked": False}]),
+        note_type="todo",
+        label="calendar",
+        due_date=due_date,
+        source="calendar",
+    )
+    db.add(note)
+    return {"note_id": note.id, "due_date": due_date, "minutes": minutes_before, "skipped_reason": None}
+
+
+def _event_to_dict(ev: CalendarEvent, db=None, owner: str | None = None) -> dict:
     """Convert a CalendarEvent model to the API dict format.
 
     Timed events whose stored datetimes represent UTC (is_utc=True) are
@@ -637,6 +765,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         suffix = "Z" if getattr(ev, "is_utc", False) else ""
         start_str = ev.dtstart.isoformat() + suffix
         end_str = ev.dtend.isoformat() + suffix
+    reminder = _calendar_reminder_for_event(db, owner, ev) if db and owner else None
     return {
         "uid": ev.uid,
         "summary": ev.summary or "",
@@ -653,6 +782,10 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
         "color": ev.color or (ev.calendar.color if ev.calendar else ""),
         "event_type": getattr(ev, "event_type", None),
         "importance": getattr(ev, "importance", None) or "normal",
+        "has_reminder": bool(reminder),
+        "reminder_note_id": reminder["note_id"] if reminder else None,
+        "reminder_due_date": reminder["due_date"] if reminder else None,
+        "reminder_minutes": reminder["minutes"] if reminder else None,
     }
 
 
@@ -684,7 +817,7 @@ def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
 
 
 def _expand_rrule(
-    ev: CalendarEvent, start: datetime, end: datetime
+    ev: CalendarEvent, start: datetime, end: datetime, db=None, owner: str | None = None
 ) -> List[dict]:
     """Expand a single recurring CalendarEvent into occurrence dicts.
 
@@ -702,7 +835,7 @@ def _expand_rrule(
         # Non-recurring — return the base event as-is. list_events
         # already filters non-recurring rows with the overlap check
         # in SQL, so we don't re-check here.
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -728,7 +861,7 @@ def _expand_rrule(
         logger.warning(
             "Failed to parse rrule=%r for event %s: %s", ev.rrule, ev.uid, ex
         )
-        d = _event_to_dict(ev)
+        d = _event_to_dict(ev, db=db, owner=owner)
         d["is_recurrence"] = False
         d["series_uid"] = ev.uid
         d["truncated"] = False
@@ -746,7 +879,7 @@ def _expand_rrule(
     expand_start = start - duration
     results = []
     truncated = False
-    base = _event_to_dict(ev)
+    base = _event_to_dict(ev, db=db, owner=owner)
     exdates = set(_recurrence_exdates(ev))
 
     for occ_start in rule.xafter(expand_start, inc=True):
@@ -1185,7 +1318,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             # Expand recurring events into individual occurrences.
             expanded = []
             for e in events:
-                expanded.extend(_expand_rrule(e, start_dt, end_dt))
+                expanded.extend(_expand_rrule(e, start_dt, end_dt, db=db, owner=owner))
 
             # Sort by occurrence start time for consistent frontend ordering.
             truncated = any(e.get("truncated") for e in expanded)
@@ -1251,10 +1384,19 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 caldav_sync_pending="create" if cal.source == "caldav" else None,
             )
             db.add(ev)
+            reminder = None
+            if data.reminder_minutes is not None:
+                reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             db.commit()
+            db.refresh(ev)
             if cal.source == "caldav":
                 await _push_caldav_event_after_commit(owner, uid, "create")
-            return {"ok": True, "uid": uid}
+            return {
+                "ok": True,
+                "uid": uid,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1264,6 +1406,17 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    @router.get("/events/{uid}")
+    async def get_event(request: Request, uid: str):
+        owner = _require_user(request)
+        db = SessionLocal()
+        try:
+            base_uid = _resolve_base_uid(uid)
+            ev = _get_or_404_event(db, base_uid, owner)
+            return {"event": _event_to_dict(ev, db=db, owner=owner)}
+        finally:
+            db.close()
+
     @router.put("/events/{uid}")
     async def update_event(request: Request, uid: str, data: EventUpdate):
         owner = _require_user(request)
@@ -1300,13 +1453,24 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 ev.rrule = data.rrule
             if data.color is not None:
                 ev.color = data.color if data.color else None
+            reminder = None
+            reminder_fields = getattr(data, "model_fields_set", getattr(data, "__fields_set__", set()))
+            if "reminder_minutes" in reminder_fields:
+                _delete_calendar_reminders_for_event(db, owner, ev)
+                if data.reminder_minutes is not None:
+                    reminder = _create_calendar_reminder_for_event(db, owner, ev, data.reminder_minutes)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
             if is_caldav:
                 ev.caldav_sync_pending = "update"
             db.commit()
+            db.refresh(ev)
             if is_caldav:
                 await _push_caldav_event_after_commit(owner, base_uid, "update")
-            return {"ok": True}
+            return {
+                "ok": True,
+                "event": _event_to_dict(ev, db=db, owner=owner),
+                "reminder": reminder,
+            }
         except HTTPException:
             raise
         except Exception as e:
@@ -1328,6 +1492,8 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
             ev = _get_or_404_event(db, base_uid, owner)
             is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
             is_caldav = ev.calendar and ev.calendar.source == "caldav"
+            if scope in {"occurrence", "instance"} and not is_occurrence_delete:
+                raise HTTPException(400, "Occurrence delete requires a recurring occurrence uid")
             if is_occurrence_delete:
                 key = _occurrence_exdate_key(uid, ev)
                 if not key:
@@ -1344,6 +1510,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 return {"ok": True, "scope": "occurrence", "exdate": key}
             if is_caldav:
                 _record_caldav_delete_tombstone(db, ev, owner)
+            _delete_calendar_reminders_for_event(db, owner, ev)
             db.delete(ev)
             db.commit()
             if is_caldav:
@@ -1423,7 +1590,7 @@ def setup_calendar_routes(upload_handler=None) -> APIRouter:
                 raise HTTPException(400, f"Invalid ICS file: {e}")
 
             # Sanitize display name — length cap + strip control chars
-            raw_name = calendar_name.strip() or (file.filename or "").replace(".ics", "").replace("_", " ").strip() or "Imported"
+            raw_name = calendar_name.strip() or re.sub(r"\.(?:calendar|ics|ical)$", "", file.filename or "", flags=re.IGNORECASE).replace("_", " ").strip() or "Imported"
             cal_display = "".join(c for c in raw_name if c.isprintable())[:120] or "Imported"
 
             target_cal = db.query(CalendarCal).filter(

routes/chat_helpers.py

@@ -3,6 +3,7 @@
 import asyncio
 import json
 import logging
+import math
 import os
 import re
 import time
@@ -25,6 +26,56 @@ from fastapi import HTTPException
 
 logger = logging.getLogger(__name__)
 
+_INVISIBLE_RESPONSE_CHARS = "\u2063\u200b\u200c\u200d\ufeff"
+
+
+def _skill_run_is_complex(agent_rounds: int, agent_tool_calls: int) -> bool:
+    """Keep one-off TUI edit loops out of automatic skill extraction."""
+    return agent_tool_calls >= 4 or (agent_rounds >= 5 and agent_tool_calls >= 3)
+
+
+def clean_repeated_assistant_content(text: object) -> str:
+    """Collapse repeated terminal assistant prose before history/SFT storage."""
+    value = str(text or "")
+    for char in _INVISIBLE_RESPONSE_CHARS:
+        value = value.replace(char, "")
+    value = value.strip()
+    if not value:
+        return ""
+
+    # Stream rejoin/finalization races can concatenate the same complete
+    # answer without separators. Collapse only exact 2-4x repetitions.
+    for copies in range(4, 1, -1):
+        if len(value) % copies == 0:
+            width = len(value) // copies
+            unit = value[:width]
+            if unit and unit * copies == value:
+                value = unit.strip()
+                break
+
+    # Interrupted/rejoined streams can leave a short suffix before a closing
+    # think tag at the edge of visible prose, e.g. "ls.\n</think>\n\nHere's...".
+    edge_close_re = re.compile(r"(?is)^\s*(?!<\s*think\b)[^<\n]{0,120}\s*</\s*think\s*>\s*")
+    while True:
+        cleaned = edge_close_re.sub("", value, count=1).strip()
+        if cleaned == value:
+            break
+        value = cleaned
+
+    first_line = next((line.strip() for line in value.splitlines() if line.strip()), "")
+    if 8 <= len(first_line) <= 180:
+        matches = list(re.finditer(r"(?m)^" + re.escape(first_line) + r"\s*$", value))
+        if len(matches) >= 2:
+            value = value[matches[0].start():matches[1].start()].strip()
+
+    value = re.sub(
+        r"(?is)(?<=[.!?])(?:[a-z]{1,12}\.)\s*</\s*think\s*>\s*$",
+        "",
+        value,
+    ).strip()
+    value = re.sub(r"(?is)\s*</\s*think\s*>\s*$", "", value).strip()
+    return value
+
 _CASUAL_OPENING_RE = re.compile(
     r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
     r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
@@ -36,6 +87,14 @@ _CASUAL_BLOCKLIST_RE = re.compile(
     r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
     re.IGNORECASE,
 )
+_PERSONAL_TOOL_CONTEXT_RE = re.compile(
+    r"\b(?:"
+    r"email|emails|mail|inbox|gmail|"
+    r"calendar|events?|meetings?|appointments?|schedule|"
+    r"notes?|todo|checklist|reminders?|tasks?"
+    r")\b",
+    re.IGNORECASE,
+)
 
 
 def _is_casual_low_signal(text: str) -> bool:
@@ -51,6 +110,14 @@ def _is_casual_low_signal(text: str) -> bool:
     return len(tail_words) <= 2
 
 
+def _truthy_request_flag(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
 # Strong references to in-flight fire-and-forget tasks scheduled from this
 # module. asyncio only keeps weak references to tasks created via
 # create_task, so without this the GC can collect a task mid-execution and
@@ -60,6 +127,197 @@ _BG_TASKS: set[asyncio.Task] = set()
 _INCOGNITO_CONTEXTS: dict[str, dict[str, Any]] = {}
 _INCOGNITO_CONTEXT_TTL_SECONDS = 6 * 60 * 60
 _INCOGNITO_CONTEXT_MAX_MESSAGES = 80
+_SFT_TRACE_CAPTURE_ENV = "ODYSSEUS_SFT_TRACE_CAPTURE"
+_SFT_TRACE_DIR_ENV = "ODYSSEUS_SFT_TRACE_DIR"
+_RUNTIME_REVISION_ENV = "ODYSSEUS_RUNTIME_REVISION"
+
+
+def _sft_trace_capture_enabled(owner: str | None) -> bool:
+    flag = os.getenv(_SFT_TRACE_CAPTURE_ENV, "1").strip().lower()
+    return flag not in {"0", "false", "no", "off"} and str(owner or "").startswith("sft_")
+
+
+def _json_safe(value: Any) -> Any:
+    try:
+        json.dumps(value)
+        return value
+    except TypeError:
+        return str(value)
+
+
+def _last_user_message_for_trace(sess) -> str:
+    for msg in reversed(getattr(sess, "history", []) or []):
+        if getattr(msg, "role", None) == "user":
+            return str(getattr(msg, "content", "") or "").strip()
+    return ""
+
+
+def _append_sft_trace_record(
+    *,
+    owner: str | None,
+    session_id: str,
+    sess,
+    assistant_content: str,
+    metadata: dict,
+    message_id: Any = None,
+) -> None:
+    """Append one training-ready trace record for synthetic SFT users."""
+    if not _sft_trace_capture_enabled(owner):
+        return
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        os.makedirs(trace_dir, exist_ok=True)
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        runtime_revision = os.getenv(_RUNTIME_REVISION_ENV, "").strip()
+        record = {
+            "format": "odysseus_sft_trace_turn_v1",
+            "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "owner": owner,
+            "session_id": session_id,
+            "session_name": getattr(sess, "name", "") or "",
+            "message_id": message_id,
+            "user": _last_user_message_for_trace(sess),
+            "assistant": str(assistant_content or "").strip(),
+            "thinking": str((metadata or {}).get("thinking") or "").strip(),
+            "tool_events": _json_safe((metadata or {}).get("tool_events") or []),
+            "round_texts": _json_safe((metadata or {}).get("round_texts") or []),
+            "runtime_revision": runtime_revision,
+            "metadata": {
+                "model": (metadata or {}).get("model"),
+                "requested_model": (metadata or {}).get("requested_model"),
+                "endpoint_label": (metadata or {}).get("endpoint_label"),
+                "endpoint_id": (metadata or {}).get("endpoint_id"),
+                "response_time": (metadata or {}).get("response_time"),
+                "input_tokens": (metadata or {}).get("input_tokens"),
+                "output_tokens": (metadata or {}).get("output_tokens"),
+                "usage_buckets": _json_safe((metadata or {}).get("usage_buckets") or []),
+                "runtime_revision": runtime_revision,
+            },
+        }
+        _prune_sft_retry_rows_before_append(path, record)
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    except Exception as exc:
+        logger.warning("Failed to append SFT trace record for %s/%s: %s", owner, session_id, exc)
+
+
+def remove_session_sft_trace_rows(owner: str | None, session_id: str) -> int:
+    """Remove every captured training row for a deleted synthetic session."""
+    if not _sft_trace_capture_enabled(owner) or not str(session_id or "").strip():
+        return 0
+    try:
+        from src.constants import DATA_DIR
+
+        trace_dir = os.getenv(_SFT_TRACE_DIR_ENV) or os.path.join(DATA_DIR, "sft_traces")
+        path = os.path.join(trace_dir, f"{owner}.jsonl")
+        if not os.path.exists(path):
+            return 0
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as source:
+            for line in source:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if str(row.get("session_id") or "") != session_id:
+                    kept.append(raw)
+                    continue
+                row["deleted_from_training"] = True
+                removed.append(json.dumps(row, ensure_ascii=False))
+        if not removed:
+            return 0
+        tmp_path = f"{path}.{os.getpid()}.{time.time_ns()}.tmp"
+        with open(tmp_path, "w", encoding="utf-8") as target:
+            for raw in kept:
+                target.write(raw + "\n")
+        os.replace(tmp_path, path)
+        with open(path + ".trash", "a", encoding="utf-8") as trash:
+            for raw in removed:
+                trash.write(raw + "\n")
+        logger.info("Removed %d SFT trace row(s) for deleted session %s", len(removed), session_id)
+        return len(removed)
+    except Exception as exc:
+        logger.warning("Failed to remove SFT trace rows for session %s: %s", session_id, exc)
+        return 0
+
+
+def _prune_sft_retry_rows_before_append(path: str, record: dict[str, Any]) -> None:
+    """For SFT traces, keep only the latest retry for a repeated user send.
+
+    The browser resend flow can append a second identical user turn without
+    first calling the delete endpoint. Training wants the final attempt, not
+    both sends, so remove prior trailing rows in the same session with the same
+    user prompt before appending the replacement.
+    """
+    current_session = str(record.get("session_id") or "")
+    current_user = str(record.get("user") or "").strip()
+    if not current_session or not current_user or not os.path.exists(path):
+        return
+    kept: list[str] = []
+    parsed: list[tuple[str, dict | None]] = []
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    parsed.append((raw, json.loads(raw)))
+                except json.JSONDecodeError:
+                    parsed.append((raw, None))
+
+        last_different_same_session = -1
+        for idx, (_raw, row) in enumerate(parsed):
+            if not isinstance(row, dict) or row.get("session_id") != current_session:
+                continue
+            if str(row.get("user") or "").strip() != current_user:
+                last_different_same_session = idx
+
+        removed: list[str] = []
+        for idx, (raw, row) in enumerate(parsed):
+            should_remove = (
+                idx > last_different_same_session
+                and isinstance(row, dict)
+                and row.get("session_id") == current_session
+                and str(row.get("user") or "").strip() == current_user
+            )
+            if should_remove:
+                tombstone = dict(row)
+                tombstone["deleted_from_training"] = True
+                tombstone["delete_reason"] = "sft_retry_replaced"
+                removed.append(json.dumps(tombstone, ensure_ascii=False))
+            else:
+                kept.append(raw)
+
+        if not removed:
+            return
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        with open(path + ".trash", "a", encoding="utf-8") as f:
+            for raw in removed:
+                f.write(raw + "\n")
+        logger.info(
+            "Removed %d prior SFT retry row(s) before appending replacement for session %s",
+            len(removed),
+            current_session,
+        )
+    except Exception as exc:
+        logger.warning("Failed to prune prior SFT retry rows for %s: %s", current_session, exc)
+
+
+def strip_tui_local_context(content: Any) -> Any:
+    """Remove client-only workspace metadata before persistence/display."""
+    if not isinstance(content, str):
+        return content
+    return re.sub(r"\s*<local_context\b[^>]*>.*?</local_context>\s*", "", content, flags=re.IGNORECASE | re.DOTALL).strip()
 
 
 def _spawn_bg(coro) -> asyncio.Task:
@@ -113,6 +371,8 @@ class PresetInfo:
     max_tokens: Optional[int]
     system_prompt: Optional[str]
     character_name: Optional[str]
+    persona_memory: Optional[str] = None
+    persona_memory_schema: str = "general"
 
 
 @dataclass
@@ -251,6 +511,14 @@ def needs_auto_name(name: str) -> bool:
     return False
 
 
+def fallback_session_title(text: str, *, max_words: int = 6) -> str:
+    words = re.findall(r"[A-Za-z0-9@._'-]+", text)
+    if not words:
+        return "New chat"
+    title = " ".join(words[:max_words]).strip()
+    return title[:60] or "New chat"
+
+
 async def auto_name_session(session_manager, sess):
     """Generate a short title for a session from its first user message."""
     try:
@@ -273,6 +541,17 @@ async def auto_name_session(session_manager, sess):
         if not first_msg:
             return
 
+        endpoint_url = str(getattr(sess, "endpoint_url", "") or "")
+        model_name = str(getattr(sess, "model", "") or "")
+        if (
+            "ttft" in model_name.lower()
+            or re.search(r":18\d{3}\b", endpoint_url)
+        ):
+            title = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, title)
+            logger.info(f"Auto-named session {sess.id} deterministically: {title}")
+            return
+
         owner = getattr(sess, "owner", None)
         t_url, t_model, t_headers = resolve_task_endpoint(
             sess.endpoint_url, sess.model, sess.headers, owner=owner
@@ -294,9 +573,9 @@ async def auto_name_session(session_manager, sess):
                 {"role": "user", "content": first_msg},
             ],
             temperature=0.3,
-            max_tokens=4096,
+            max_tokens=64,
             headers=t_headers,
-            timeout=60,
+            timeout=15,
         )
 
         title = title.strip().strip('"\'').strip()
@@ -304,18 +583,47 @@ async def auto_name_session(session_manager, sess):
         # via the central helper.
         from src.text_helpers import strip_think
         title = strip_think(title, prose=False, prompt_echo=False)
-        if title and len(title) < 80:
-            session_manager.update_session_name(sess.id, title)
-            logger.info(f"Auto-named session {sess.id}: {title}")
+        if not title or len(title) >= 80 or "\n" in title:
+            fallback = fallback_session_title(first_msg)
+            session_manager.update_session_name(sess.id, fallback)
+            logger.info(
+                "Auto-named session %s with fallback title after unusable model title: %s",
+                sess.id,
+                fallback,
+            )
+            return
+
+        session_manager.update_session_name(sess.id, title)
+        logger.info(f"Auto-named session {sess.id}: {title}")
 
     except Exception as e:
         import traceback
         logger.error(f"Auto-name failed for {sess.id}: {e}\n{traceback.format_exc()}")
 
 
+async def auto_name_session_after_stream(session_id: str, session_manager, sess):
+    """Delay chat title generation until the first response stream is settled."""
+    try:
+        waited = 0.0
+        while _is_session_stream_active(session_id) and waited < 30.0:
+            await asyncio.sleep(0.25)
+            waited += 0.25
+        # Let the final SSE chunk/message_saved bookkeeping clear before any
+        # title model call can contend with the user's visible response.
+        await asyncio.sleep(0.5)
+        try:
+            sess = session_manager.get_session(session_id)
+        except Exception as e:
+            logger.warning("[auto-name] Could not reload session %s before naming: %s", session_id, e)
+        await auto_name_session(session_manager, sess)
+    except Exception as e:
+        import traceback
+        logger.error(f"Deferred auto-name failed for {session_id}: {e}\n{traceback.format_exc()}")
+
+
 def extract_preset(chat_handler, preset_id) -> PresetInfo:
     """Extract preset parameters via chat_handler."""
-    temperature, max_tokens, system_prompt, char_name = (
+    temperature, max_tokens, system_prompt, char_name, persona_memory, persona_memory_schema = (
         chat_handler.validate_and_extract_preset(preset_id)
     )
     return PresetInfo(
@@ -323,6 +631,8 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo:
         max_tokens=max_tokens,
         system_prompt=system_prompt,
         character_name=char_name,
+        persona_memory=persona_memory,
+        persona_memory_schema=persona_memory_schema,
     )
 
 
@@ -406,14 +716,28 @@ def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[
     return manifest
 
 
-def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
+def add_user_message(
+    sess,
+    chat_handler,
+    preprocessed: PreprocessedMessage,
+    incognito: bool = False,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
+):
     """Add user message to session history and update session name.
     Incognito messages must not mutate persistent session history, even in
     memory, because a later normal turn can persist the same session object."""
     if incognito:
         return
-    user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
-    sess.add_message(ChatMessage("user", preprocessed.user_content, metadata=user_meta))
+    user_meta = {}
+    if preprocessed.attachment_meta:
+        user_meta["attachments"] = preprocessed.attachment_meta
+    if interaction_mode in {"chat", "agent", "research"}:
+        user_meta["interaction_mode"] = interaction_mode
+    if auto_escalated:
+        user_meta["auto_escalated"] = True
+    clean_content = strip_tui_local_context(preprocessed.user_content)
+    sess.add_message(ChatMessage("user", clean_content, metadata=user_meta or None))
     chat_handler.update_session_name_if_needed(sess, preprocessed.text_for_context)
 
 
@@ -626,6 +950,8 @@ async def build_chat_context(
     defer_context_shaping: bool = False,
     continuation_context_message: str | None = None,
     persist_user_message: bool = True,
+    interaction_mode: str | None = None,
+    auto_escalated: bool = False,
 ) -> ChatContext:
     """Build the full context (preface + messages) for an LLM call.
 
@@ -650,10 +976,23 @@ async def build_chat_context(
     # transcript store instead of session history so stale saved chats cannot
     # bleed into context and the turn is not persisted.
     if persist_user_message and incognito:
-        user_meta = {"attachments": preprocessed.attachment_meta} if preprocessed.attachment_meta else None
+        user_meta = {}
+        if preprocessed.attachment_meta:
+            user_meta["attachments"] = preprocessed.attachment_meta
+        if interaction_mode in {"chat", "agent", "research"}:
+            user_meta["interaction_mode"] = interaction_mode
+        if auto_escalated:
+            user_meta["auto_escalated"] = True
         _append_incognito_message(session_id, "user", preprocessed.user_content, user_meta)
     elif persist_user_message:
-        add_user_message(sess, chat_handler, preprocessed, incognito=False)
+        add_user_message(
+            sess,
+            chat_handler,
+            preprocessed,
+            incognito=False,
+            interaction_mode=interaction_mode,
+            auto_escalated=auto_escalated,
+        )
 
     # Fire events
     if persist_user_message and not incognito:
@@ -679,7 +1018,11 @@ async def build_chat_context(
     mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
     # Skills injection respects its own enable toggle (mirrors memory_enabled).
     # When off, the "Available skills" index is not added to the prompt.
-    skills_enabled = not incognito and uprefs.get("skills_enabled", True)
+    skills_enabled = (
+        not incognito
+        and uprefs.get("skills_enabled", True)
+        and getattr(sess, "skill_injection_enabled", True) is not False
+    )
     if not allow_tool_preprocessing:
         mem_enabled = False
         skills_enabled = False
@@ -704,8 +1047,17 @@ async def build_chat_context(
     if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
         use_rag_val = False
 
-    # If pre-fetched search context was provided (compare mode), skip live web search
-    skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
+    use_web_val = _truthy_request_flag(use_web)
+    # If pre-fetched search context was provided (compare mode), skip live web
+    # search. Personal app requests should be served by their tools; pre-search
+    # here caused calendar/email turns with use_web="false" to run irrelevant
+    # web searches before the agent even saw the tool surface.
+    skip_web = (
+        bool(search_context)
+        or not allow_tool_preprocessing
+        or casual_low_signal
+        or bool(agent_mode and _PERSONAL_TOOL_CONTEXT_RE.search(context_message or ""))
+    )
 
     # Build context preface
     # The stream path uses enhanced_message (with CoT/preprocessing applied),
@@ -722,12 +1074,13 @@ async def build_chat_context(
     _preface_kwargs = dict(
         message=_ctx_msg,
         session=sess,
-        use_web=use_web and not skip_web,
+        use_web=use_web_val and not skip_web,
         use_memory=mem_enabled,
         time_filter=time_filter,
         preset_system_prompt=preset.system_prompt,
         owner=user,
         character_name=preset.character_name,
+        persona_memory=preset.persona_memory,
         agent_mode=agent_mode,
         incognito=incognito,
         use_skills=skills_enabled,
@@ -826,10 +1179,17 @@ async def build_chat_context(
 
 
 def accumulate_token_usage(session_id: str, metrics: dict):
-    """Add input/output token counts to the session's running totals."""
+    """Add input/output token counts (and USD cost) to the session's totals."""
     in_t = metrics.get("input_tokens", 0)
     out_t = metrics.get("output_tokens", 0)
-    if not (in_t or out_t):
+    cost = metrics.get("cost_usd")
+    try:
+        cost = float(cost) if cost is not None else 0.0
+        if not math.isfinite(cost) or cost < 0:
+            cost = 0.0
+    except (TypeError, ValueError):
+        cost = 0.0
+    if not (in_t or out_t or cost):
         return
     db = SessionLocal()
     try:
@@ -837,6 +1197,8 @@ def accumulate_token_usage(session_id: str, metrics: dict):
         if db_s:
             db_s.total_input_tokens = (db_s.total_input_tokens or 0) + in_t
             db_s.total_output_tokens = (db_s.total_output_tokens or 0) + out_t
+            if cost:
+                db_s.total_cost_usd = (db_s.total_cost_usd or 0.0) + cost
             db.commit()
     except Exception:
         db.rollback()
@@ -889,6 +1251,21 @@ def _normalize_thinking(text: str) -> str:
 
     # Qwen3.5: "Thinking Process:" or "Thinking:" prefix
     if thinking_prefix_re.match(text.lstrip()):
+        # Tool-router checkpoints sometimes narrate several drafts and then
+        # emit an explicit final marker near the end. Prefer the last marker;
+        # the first ordinary-looking paragraph can still be internal review.
+        final_markers = list(re.finditer(
+            r"(?im)^\s*Final\s+(?:decision|answer|output(?:\s+generation)?)\s*:\s*",
+            text,
+        ))
+        if final_markers:
+            marker = final_markers[-1]
+            think = thinking_prefix_re.sub('', text[:marker.start()]).strip()
+            reply = text[marker.end():].strip()
+            if len(reply) >= 2 and reply[0] in {'\"', '\u201c'} and reply[-1] in {'\"', '\u201d'}:
+                reply = reply[1:-1].strip()
+            if reply:
+                return '<think>' + think + '</think>\n\n' + reply
         # Try clean boundary first
         m = re.match(
             r'^(Thinking(?:\s+Process)?:[\s\S]*?)(\n\n(?=[A-Z]|Hey|Yo|Hi|Sure|I |What|Here|Let|The |This |OK|Ok|Yes|No |So |Well |Thank|Alright|Of course|Absolutely|Great|Hello|As ))',
@@ -1017,6 +1394,23 @@ def clean_thinking_for_save(content: str, metadata: dict | None = None) -> tuple
         if info.get("time"):
             md["thinking_time"] = info["time"]
         return info["reply"], md
+    # A stopped stream can end before producing any answer prose. Preserve its
+    # partial reasoning as structured metadata so history rendering and the
+    # next Resume request can both recover it. Normal reasoning-only completed
+    # turns retain the legacy raw-content behavior.
+    if md.get("stopped"):
+        raw = str(content or "")
+        partial = re.match(
+            r'^\s*<think(?:ing)?(?:\s+time="([\d.]+)")?>([\s\S]*?)(?:</think(?:ing)?>\s*)?$',
+            raw,
+            re.IGNORECASE,
+        )
+        if partial and partial.group(2).strip():
+            md["thinking"] = partial.group(2).strip()
+            md["thinking_interrupted"] = True
+            if partial.group(1):
+                md["thinking_time"] = partial.group(1)
+            return "", md
     return content, md
 
 
@@ -1071,6 +1465,16 @@ def save_assistant_response(
     if tool_events:
         md["tool_events"] = tool_events
 
+    # The streaming route may have forwarded textual DSML/XML tool calls as
+    # deltas before the agent loop parsed them. Strip them again at the
+    # persistence boundary so raw tool markup cannot survive in history.
+    try:
+        from src.tool_parsing import strip_tool_blocks
+        full_response = strip_tool_blocks(str(full_response or "")).strip()
+    except Exception:
+        full_response = str(full_response or "")
+    full_response = clean_repeated_assistant_content(full_response)
+
     # Extract thinking into metadata (don't pollute message content with <think> tags)
     _think_info = _extract_thinking_meta(full_response)
     if _think_info:
@@ -1096,10 +1500,25 @@ def save_assistant_response(
     try:
         _last = sess.history[-1]
         _meta = getattr(_last, "metadata", None)
+        _message_id = _meta.get("_db_id") if isinstance(_meta, dict) else None
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+            message_id=_message_id,
+        )
         if isinstance(_meta, dict):
-            return _meta.get("_db_id")
+            return _message_id
     except (IndexError, AttributeError):
-        pass
+        _append_sft_trace_record(
+            owner=getattr(sess, "owner", None),
+            session_id=session_id,
+            sess=sess,
+            assistant_content=_content,
+            metadata=md,
+        )
     return None
 
 
@@ -1172,6 +1591,8 @@ def run_post_response_tasks(
     owner: str = None,
     extract_skills: bool = True,
     allow_background_extraction: bool = True,
+    preset_manager=None,
+    persona_memory_schema: str = "general",
 ):
     """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.
 
@@ -1192,7 +1613,8 @@ def run_post_response_tasks(
     # Memory extraction — only every 4th message pair to avoid excess LLM calls
     _msg_count = len(sess.history) if hasattr(sess, 'history') else 0
     _should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0)
-    if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
+    _chat_memory_extract = getattr(sess, "memory_extraction_enabled", True) is not False
+    if allow_background_extraction and not incognito and not compare_mode and _chat_memory_extract and _should_extract and uprefs.get("auto_memory", True):
         from services.memory.memory_extractor import extract_and_store
         from src.task_endpoint import resolve_task_endpoint
         t_url, t_model, t_headers = resolve_task_endpoint(
@@ -1203,6 +1625,27 @@ def run_post_response_tasks(
             t_url, t_model, t_headers,
         )))
 
+    if (
+        allow_background_extraction
+        and not incognito
+        and not compare_mode
+        and _chat_memory_extract
+        and _should_extract
+        and uprefs.get("auto_memory", True)
+        and character_name
+    ):
+        if preset_manager is not None:
+            from services.memory.memory_extractor import update_persona_memory
+            from src.task_endpoint import resolve_task_endpoint
+            p_url, p_model, p_headers = resolve_task_endpoint(
+                sess.endpoint_url, sess.model, sess.headers, owner=owner,
+            )
+            _extraction_jobs.append(("persona-memory", update_persona_memory(
+                sess, preset_manager, character_name,
+                p_url, p_model, p_headers,
+                schema=persona_memory_schema,
+            )))
+
     # Skill extraction from complex agent runs. Only when the user actually
     # chose agent mode — not a chat we auto-escalated for a notes/calendar
     # intent, and never in incognito/compare.
@@ -1217,13 +1660,17 @@ def run_post_response_tasks(
         extract_skills, auto_skills_enabled, incognito, compare_mode,
         agent_rounds, agent_tool_calls, "set" if skills_manager else "MISSING",
     )
+    # A normal inspect/edit/verify turn is commonly three calls. Treating that
+    # as a reusable skill creates one-off titles and makes the skill library
+    # noisy. Automatic extraction is reserved for runs that demonstrate a
+    # genuinely longer procedure; explicit skill tools remain unaffected.
     if (
         extract_skills
         and allow_background_extraction
         and auto_skills_enabled
         and not incognito
         and not compare_mode
-        and (agent_rounds >= 2 or agent_tool_calls >= 2)
+        and _skill_run_is_complex(agent_rounds, agent_tool_calls)
     ):
         if skills_manager is None:
             logger.warning(
@@ -1260,4 +1707,4 @@ def run_post_response_tasks(
 
     # Auto-name
     if needs_auto_name(sess.name):
-        _spawn_bg(auto_name_session(session_manager, sess))
+        _spawn_bg(auto_name_session_after_stream(session_id, session_manager, sess))

routes/contacts/contacts_routes.py

@@ -5,8 +5,10 @@ CardDAV contacts integration. Reads from local Radicale, supports
 search and adding new contacts.
 """
 
+import asyncio
 import re
 import logging
+import threading
 import uuid
 import json
 import csv
@@ -19,10 +21,11 @@ from datetime import datetime
 from urllib.parse import urljoin, urlparse, urlunparse
 
 from core.log_safety import redact_url
-from fastapi import APIRouter, Query, Depends, Response, HTTPException
+from fastapi import APIRouter, Query, Depends, Request, Response, HTTPException
 from typing import List, Dict, Optional
 
 from core.middleware import require_admin
+from src.auth_helpers import effective_user
 from src.url_safety import check_outbound_url
 
 logger = logging.getLogger(__name__)
@@ -93,22 +96,37 @@ def _normalize_contact(contact: Dict) -> Dict:
     if not name and emails:
         name = emails[0].split("@")[0]
     address = str(contact.get("address") or "").strip()
-    return {
+    out = {
         "uid": str(contact.get("uid") or uuid.uuid4()),
         "name": name,
         "emails": emails,
         "phones": phones,
         "address": address,
     }
+    owner = str(contact.get("owner") or "").strip()
+    if owner:
+        out["owner"] = owner
+    return out
 
 
-def _load_local_contacts() -> List[Dict]:
+def _contact_visible_to_owner(contact: Dict, owner: Optional[str]) -> bool:
+    owner = str(owner or "").strip()
+    row_owner = str(contact.get("owner") or "").strip()
+    if owner:
+        if row_owner:
+            return row_owner == owner
+        return not owner.startswith("sft_")
+    return True
+
+
+def _load_local_contacts(owner: Optional[str] = None) -> List[Dict]:
     try:
         if not LOCAL_CONTACTS_FILE.exists():
             return []
         data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
         rows = data.get("contacts", data) if isinstance(data, dict) else data
-        return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        contacts = [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
+        return [c for c in contacts if _contact_visible_to_owner(c, owner)]
     except Exception as e:
         logger.error(f"Failed to load local contacts: {e}")
         return []
@@ -119,7 +137,9 @@ def _save_local_contacts(contacts: List[Dict]) -> None:
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
     _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
+    _contact_cache["by_owner"] = {}
     _contact_cache["fetched_at"] = datetime.utcnow()
+    _contact_cache["failed_at"] = None
 
 
 # ── vCard parsing ──
@@ -264,7 +284,58 @@ def _build_vcard(name: str, email: str, uid: Optional[str] = None,
 
 # ── In-memory cache ──
 
-_contact_cache = {"contacts": [], "fetched_at": None}
+_CONTACT_CACHE_TTL_SECONDS = 60
+_CONTACT_FAILURE_BACKOFF_SECONDS = 120
+_CARDDAV_TIMEOUT = httpx.Timeout(5.0, connect=2.0)
+
+# CardDAV can be unavailable for a while. Keep the UI responsive by serving
+# the last known result (or an empty list on first use) while a single worker
+# attempts a refresh in the background.
+_contact_cache = {
+    "contacts": [],
+    "fetched_at": None,
+    "failed_at": None,
+    "by_owner": {},
+}
+_contact_fetch_lock = threading.Lock()
+
+
+def _cached_contacts(owner_key: str) -> List[Dict]:
+    cached = (_contact_cache.get("by_owner") or {}).get(owner_key) or {}
+    if owner_key and cached:
+        return cached.get("contacts") or []
+    return _contact_cache.get("contacts") or []
+
+
+def _mark_contact_fetch_failure(owner_key: str) -> List[Dict]:
+    now = datetime.utcnow()
+    stale_contacts = _cached_contacts(owner_key)
+    _contact_cache["failed_at"] = now
+    if owner_key:
+        _contact_cache.setdefault("by_owner", {})[owner_key] = {
+            "contacts": stale_contacts,
+            "fetched_at": now,
+        }
+    else:
+        _contact_cache["fetched_at"] = now
+    return stale_contacts
+
+
+def _contact_sync_status() -> Dict[str, str]:
+    """Return a safe, user-facing summary for contact autocomplete clients."""
+    if not _carddav_configured():
+        return {"state": "local", "message": "No contact sync is configured."}
+    if _contact_fetch_lock.locked():
+        return {"state": "syncing", "message": "Syncing contacts..."}
+    failed_at = _contact_cache.get("failed_at")
+    if failed_at:
+        age = (datetime.utcnow() - failed_at).total_seconds()
+        if age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return {
+                "state": "unavailable",
+                "message": "Contacts sync is unavailable. Try again later.",
+            }
+    return {"state": "ready", "message": ""}
 
 
 def _abs_url(href: str) -> str:
@@ -306,7 +377,7 @@ def _fetch_via_report(cfg, auth):
             "REPORT", cfg["url"],
             content=_ADDRESSBOOK_QUERY.encode("utf-8"),
             headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
-            auth=auth, timeout=10,
+            auth=auth, timeout=_CARDDAV_TIMEOUT,
         )
         if r.status_code not in (207, 200):
             return None
@@ -337,20 +408,51 @@ def _fetch_via_report(cfg, auth):
         return None
 
 
-def _fetch_contacts(force=False):
+def _fetch_contacts(force=False, owner: Optional[str] = None):
     """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
-    if not force and _contact_cache["fetched_at"]:
+    owner_key = str(owner or "").strip()
+    by_owner = _contact_cache.setdefault("by_owner", {})
+    if owner_key and not force and owner_key in by_owner:
+        cached = by_owner.get(owner_key) or {}
+        fetched_at = cached.get("fetched_at")
+        if fetched_at:
+            age = (datetime.utcnow() - fetched_at).total_seconds()
+            if age < _CONTACT_CACHE_TTL_SECONDS:
+                return cached.get("contacts") or []
+
+    if not owner_key and not force and _contact_cache["fetched_at"]:
         age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
-        if age < 60:
+        if age < _CONTACT_CACHE_TTL_SECONDS:
             return _contact_cache["contacts"]
 
+    failed_at = _contact_cache.get("failed_at")
+    if not force and failed_at:
+        failure_age = (datetime.utcnow() - failed_at).total_seconds()
+        if failure_age < _CONTACT_FAILURE_BACKOFF_SECONDS:
+            return _cached_contacts(owner_key)
+
+    # SFT users must not see the operator's personal/CardDAV contact book.
+    # Their training contacts are seeded as owner-scoped local rows.
+    if owner_key.startswith("sft_"):
+        contacts = _load_local_contacts(owner_key)
+        by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        return contacts
+
     cfg = _get_carddav_config()
     if not _carddav_configured(cfg):
-        contacts = _load_local_contacts()
-        _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        contacts = _load_local_contacts(owner_key or None)
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": datetime.utcnow()}
+        else:
+            _contact_cache["contacts"] = contacts
+            _contact_cache["fetched_at"] = datetime.utcnow()
         return contacts
 
+    # Do not let a burst of typeahead requests start parallel CardDAV timeouts.
+    # A caller that arrives during a refresh gets the most recent cache instead.
+    if not _contact_fetch_lock.acquire(blocking=False):
+        return _cached_contacts(owner_key)
+
     try:
         cfg["url"] = _carddav_base_url(cfg)
         auth = None
@@ -360,17 +462,23 @@ def _fetch_contacts(force=False):
         contacts = _fetch_via_report(cfg, auth)
         if contacts is None:
             # Fallback: plain GET, concatenated vCards, no hrefs.
-            r = httpx.get(cfg["url"], auth=auth, timeout=10)
+            r = httpx.get(cfg["url"], auth=auth, timeout=_CARDDAV_TIMEOUT)
             if r.status_code != 200:
                 logger.warning(f"CardDAV returned {r.status_code}")
-                return _contact_cache["contacts"]
+                return _mark_contact_fetch_failure(owner_key)
             contacts = _parse_vcards(r.text)
+        fetched_at = datetime.utcnow()
         _contact_cache["contacts"] = contacts
-        _contact_cache["fetched_at"] = datetime.utcnow()
+        _contact_cache["fetched_at"] = fetched_at
+        _contact_cache["failed_at"] = None
+        if owner_key:
+            by_owner[owner_key] = {"contacts": contacts, "fetched_at": fetched_at}
         return contacts
     except Exception as e:
         logger.error(f"Failed to fetch contacts: {e}")
-        return _contact_cache["contacts"]
+        return _mark_contact_fetch_failure(owner_key)
+    finally:
+        _contact_fetch_lock.release()
 
 
 def _resolve_resource_url(uid: str) -> str:
@@ -394,25 +502,31 @@ def _resolve_resource_url(uid: str) -> str:
     return _lookup() or _vcard_url(uid)
 
 
-def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
+def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None, owner: Optional[str] = None) -> bool:
     """Add a new contact via CardDAV or local contacts."""
     email = (email or "").strip()
     phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         email_l = email.lower()
         for c in contacts:
+            if owner_key and not _contact_visible_to_owner(c, owner_key):
+                continue
             if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
                 return True
             if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
                 return True
-        contacts.append(_normalize_contact({
+        row = {
             "name": name,
             "emails": [email] if email else [],
             "phones": phone_list,
             "address": address,
-        }))
+        }
+        if owner_key:
+            row["owner"] = owner_key
+        contacts.append(_normalize_contact(row))
         _save_local_contacts(contacts)
         return True
 
@@ -650,24 +764,34 @@ def _contacts_to_csv(contacts: List[Dict]) -> str:
     return out.getvalue()
 
 
-def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
+def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "", owner: Optional[str] = None) -> bool:
     """Rewrite an existing contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
         found = False
         out = []
         for c in contacts:
             if c.get("uid") == uid:
+                if owner_key and not _contact_visible_to_owner(c, owner_key):
+                    out.append(c)
+                    continue
                 # Preserve existing address when caller passes "" (only
                 # updating name/emails/phones, not touching address).
                 addr = address if address else c.get("address", "")
-                out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
+                row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}
+                if owner_key:
+                    row["owner"] = owner_key
+                out.append(_normalize_contact(row))
                 found = True
             else:
                 out.append(c)
         if not found:
-            out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
+            row = {"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}
+            if owner_key:
+                row["owner"] = owner_key
+            out.append(_normalize_contact(row))
         _save_local_contacts(out)
         return True
 
@@ -694,12 +818,16 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], a
         return False
 
 
-def _delete_contact(uid: str) -> bool:
+def _delete_contact(uid: str, owner: Optional[str] = None) -> bool:
     """Delete a contact via CardDAV or local contacts."""
     cfg = _get_carddav_config()
-    if not _carddav_configured(cfg):
+    owner_key = str(owner or "").strip()
+    if owner_key.startswith("sft_") or not _carddav_configured(cfg):
         contacts = _load_local_contacts()
-        remaining = [c for c in contacts if c.get("uid") != uid]
+        remaining = [
+            c for c in contacts
+            if c.get("uid") != uid or (owner_key and not _contact_visible_to_owner(c, owner_key))
+        ]
         _save_local_contacts(remaining)
         return True
 
@@ -739,17 +867,17 @@ def setup_contacts_routes():
     router = APIRouter(prefix="/api/contacts", tags=["contacts"])
 
     @router.get("/list")
-    async def list_contacts(_admin: str = Depends(require_admin)):
+    async def list_contacts(request: Request, _admin: str = Depends(require_admin)):
         """List all contacts."""
-        contacts = _fetch_contacts()
-        return {"contacts": contacts, "count": len(contacts)}
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
+        return {"contacts": contacts, "count": len(contacts), "sync": _contact_sync_status()}
 
     @router.get("/search")
-    async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
+    async def search_contacts(request: Request, q: str = Query(""), _admin: str = Depends(require_admin)):
         """Search contacts by name or email. Returns up to 10 matches."""
-        contacts = _fetch_contacts()
+        contacts = await asyncio.to_thread(_fetch_contacts, owner=effective_user(request))
         if not q:
-            return {"results": []}
+            return {"results": [], "sync": _contact_sync_status()}
         q_lower = q.lower()
         results = []
         for c in contacts:
@@ -760,11 +888,12 @@ def setup_contacts_routes():
                 if q_lower in em.lower():
                     results.append(c)
                     break
-        return {"results": results[:10]}
+        return {"results": results[:10], "sync": _contact_sync_status()}
 
     @router.post("/add")
-    async def add_contact(data: dict, _admin: str = Depends(require_admin)):
+    async def add_contact(data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Add a new contact."""
+        owner = effective_user(request)
         name = (data.get("name") or "").strip()
         email = (data.get("email") or "").strip()
         phone = (data.get("phone") or "").strip()
@@ -778,17 +907,20 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, phone, or address required"}
         if not name:
             name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
-        contacts = _fetch_contacts()
+        contacts = _fetch_contacts(owner=owner)
         for c in contacts:
             if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
                 return {"success": True, "message": "Already exists", "contact": c}
             if phones and any(p in (c.get("phones") or []) for p in phones):
                 return {"success": True, "message": "Already exists", "contact": c}
         create_params = inspect.signature(_create_contact).parameters
-        if "phones" in create_params:
-            ok = _create_contact(name, email, address, phones=phones)
-        elif len(create_params) >= 3:
-            ok = _create_contact(name, email, address)
+        if len(create_params) >= 3:
+            create_kwargs = {}
+            if "phones" in create_params:
+                create_kwargs["phones"] = phones
+            if "owner" in create_params:
+                create_kwargs["owner"] = owner
+            ok = _create_contact(name, email, address, **create_kwargs)
         else:
             ok = _create_contact(name, email)
         # If a phone was provided, do an immediate update to thread it
@@ -796,7 +928,7 @@ def setup_contacts_routes():
         # email + address; phones happen via update).
         if ok and phones and "phones" not in create_params:
             try:
-                fresh = _fetch_contacts(force=True)
+                fresh = _fetch_contacts(force=True, owner=owner)
                 created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
                 if created:
                     _update_contact(
@@ -804,6 +936,7 @@ def setup_contacts_routes():
                         created.get("emails", []),
                         phones,
                         address,
+                        owner=owner,
                     )
             except Exception:
                 pass
@@ -830,11 +963,16 @@ def setup_contacts_routes():
 
     @router.get("/export")
     async def export_contacts(
+        request: Request,
         format: str = Query("vcf", pattern="^(vcf|csv)$"),
         _admin: str = Depends(require_admin),
     ):
         """Export all contacts as vCard or CSV."""
-        contacts = _fetch_contacts(force=True)
+        contacts = await asyncio.to_thread(
+            _fetch_contacts,
+            force=True,
+            owner=effective_user(request),
+        )
         if format == "csv":
             content = _contacts_to_csv(contacts)
             media_type = "text/csv; charset=utf-8"
@@ -876,19 +1014,28 @@ def setup_contacts_routes():
         _save_settings(settings)
         # Force re-fetch
         _contact_cache["fetched_at"] = None
+        _contact_cache["failed_at"] = None
         return {"success": True}
 
     @router.delete("/clear")
-    async def clear_contacts(_admin: str = Depends(require_admin)):
+    async def clear_contacts(request: Request, _admin: str = Depends(require_admin)):
         """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
-        _save_local_contacts([])
+        owner = effective_user(request)
+        if owner:
+            remaining = [
+                c for c in _load_local_contacts()
+                if not _contact_visible_to_owner(c, owner)
+            ]
+            _save_local_contacts(remaining)
+        else:
+            _save_local_contacts([])
         return {"success": True}
 
     # NOTE: the /{uid} routes are declared LAST so the literal paths above
     # (/list, /search, /add, /config) win — otherwise PUT /config would
     # match PUT /{uid} with uid="config".
     @router.put("/{uid}")
-    async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
+    async def edit_contact(uid: str, data: dict, request: Request, _admin: str = Depends(require_admin)):
         """Edit an existing contact — name / emails / phones / address."""
         name = (data.get("name") or "").strip()
         emails = data.get("emails")
@@ -902,15 +1049,15 @@ def setup_contacts_routes():
             return {"success": False, "error": "Name, email, or address required"}
         if not name and emails:
             name = emails[0].split("@")[0]
-        ok = _update_contact(uid, name, emails, phones, address)
+        ok = _update_contact(uid, name, emails, phones, address, owner=effective_user(request))
         return {"success": ok}
 
     @router.delete("/{uid}")
-    async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
+    async def delete_contact(uid: str, request: Request, _admin: str = Depends(require_admin)):
         """Delete a contact by UID."""
         if not uid:
             return {"success": False, "error": "UID required"}
-        ok = _delete_contact(uid)
+        ok = _delete_contact(uid, owner=effective_user(request))
         return {"success": ok}
 
     return router

routes/cookbook_helpers.py

@@ -1085,6 +1085,10 @@ class ServeRequest(BaseModel):
     hf_token: str | None = None
     gpus: str | None = None
     platform: str | None = None    # "linux", "termux", or "windows"
+    # Optional explicit image runtime adapter. "auto" preserves compatibility
+    # with older callers; catalog-backed launches can set this without relying
+    # on model-name heuristics in the generated runner.
+    runtime_adapter: str | None = None
 
 
 def _parse_serve_phase(snapshot: str, task_type: str = "serve") -> dict:

routes/cookbook_routes.py

@@ -114,6 +114,16 @@ def _append_mlx_image_server_script(runner_lines: list[str]) -> None:
     runner_lines.append('chmod +x scripts/mlx_image_server.py 2>/dev/null || true')
 
 
+def _normalize_runtime_adapter(value: str | None) -> str:
+    """Return a shell-safe explicit image adapter name."""
+    value = (value or "auto").strip().lower()
+    if not value:
+        return "auto"
+    if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,39}", value):
+        raise HTTPException(400, "Invalid runtime adapter")
+    return value
+
+
 def _venv_root_from_serve_cmd(cmd: str) -> str:
     """Best-effort venv root from an absolute venv python in a serve command."""
     try:
@@ -664,13 +674,17 @@ def setup_cookbook_routes() -> APIRouter:
             return cmd
 
         repo_id = "cyankiwi/MiniMax-M3-AWQ-INT4"
-        snapshot = (
-            "/home/pewds/.cache/huggingface/hub/"
-            "models--cyankiwi--MiniMax-M3-AWQ-INT4/"
-            "snapshots/4082acbbec1236d21828d55b6bb0fe02ade4ab5b"
-        )
-        if body[serve_i + 1] == repo_id:
-            body[serve_i + 1] = snapshot
+        hf_home = Path(os.environ.get("HF_HOME", str(Path.home() / ".cache" / "huggingface")))
+        hf_cache = Path(os.environ.get("HUGGINGFACE_HUB_CACHE", str(hf_home / "hub")))
+        snapshot_root = hf_cache / "models--cyankiwi--MiniMax-M3-AWQ-INT4" / "snapshots"
+        if body[serve_i + 1] == repo_id and snapshot_root.is_dir():
+            installed_snapshots = sorted(
+                (p for p in snapshot_root.iterdir() if p.is_dir()),
+                key=lambda p: p.stat().st_mtime,
+                reverse=True,
+            )
+            if installed_snapshots:
+                body[serve_i + 1] = str(installed_snapshots[0])
 
         def add_env(key: str, value: str) -> None:
             if not any(p.startswith(f"{key}=") for p in env_parts):
@@ -1411,7 +1425,6 @@ def setup_cookbook_routes() -> APIRouter:
         # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection.
         host = validate_remote_host(host)
         ssh_port = validate_ssh_port(ssh_port)
-        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
 
         model_dirs = []
         if model_dir:
@@ -1423,20 +1436,17 @@ def setup_cookbook_routes() -> APIRouter:
                     model_dirs.append(d)
         paths_code = _cached_model_scan_script(model_dirs)
 
-        scan_py = TMUX_LOG_DIR / "scan_cache.py"
-        scan_py.write_text(paths_code, encoding="utf-8")
-
         async def _run_cached_scan_once():
+            # Each request owns its script bytes. A shared scan_cache.py races
+            # when the tool scans several hosts/directories concurrently.
             if host:
-                _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
-                _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
-                if platform == "windows":
-                    # Windows: use 'python' and pipe via stdin with double-quote wrapping
-                    cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
-                else:
-                    cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
-                proc = await asyncio.create_subprocess_shell(
-                    cmd,
+                ssh_args = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
+                            '-o', 'ServerAliveInterval=4', '-o', 'ServerAliveCountMax=1']
+                if ssh_port and ssh_port != '22':
+                    ssh_args.extend(['-p', ssh_port])
+                proc = await asyncio.create_subprocess_exec(
+                    *ssh_args, host, 'python -' if platform == 'windows' else 'python3 -',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
@@ -1454,12 +1464,31 @@ def setup_cookbook_routes() -> APIRouter:
                     or which_tool("py") or "python"
                 )
                 proc = await asyncio.create_subprocess_exec(
-                    local_py, str(scan_py),
+                    local_py, '-',
+                    stdin=asyncio.subprocess.PIPE,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     cwd=str(Path.home()),
                 )
-            return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode
+            try:
+                output = await asyncio.wait_for(proc.communicate(paths_code.encode('utf-8')), timeout=60)
+                return output, proc.returncode
+            finally:
+                # A timed-out/cancelled request must not abandon its scanner.
+                # This handle belongs only to this request, never a model job.
+                if proc.returncode is None:
+                    try:
+                        proc.terminate()
+                    except ProcessLookupError:
+                        pass
+                    try:
+                        await asyncio.wait_for(proc.wait(), timeout=2)
+                    except asyncio.TimeoutError:
+                        try:
+                            proc.kill()
+                        except ProcessLookupError:
+                            pass
+                        await asyncio.wait_for(proc.wait(), timeout=2)
 
         (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
         stderr_txt = stderr_b.decode(errors="replace").strip()
@@ -1974,6 +2003,7 @@ def setup_cookbook_routes() -> APIRouter:
         validate_remote_host(req.remote_host)
         req.ssh_port = validate_ssh_port(req.ssh_port)
         req.gpus = _validate_gpus(req.gpus)
+        req.runtime_adapter = _normalize_runtime_adapter(req.runtime_adapter)
         req.hf_token = req.hf_token or _load_stored_hf_token()
         _validate_token(req.hf_token)
         # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
@@ -2602,19 +2632,20 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('print(model)')
                 runner_lines.append('PY')
                 runner_lines.append(')"')
-                runner_lines.append('if printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; then')
+                runner_lines.append(f"export ODYSSEUS_MLX_IMAGE_ADAPTER='{_bash_squote(req.runtime_adapter or 'auto')}'")
+                runner_lines.append('if [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "hidream" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi hidream; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import mlx, mlx_vlm, transformers, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: HiDream MLX serving needs the model requirements in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart mlx mlx-vlm \'transformers>=4.57.0,<6.0\' huggingface_hub safetensors numpy pillow tqdm sentencepiece hf_transfer"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "boogu" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -qi boogu; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import boogu_image_mlx, mlx, huggingface_hub, safetensors, numpy, PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: Boogu MLX serving needs boogu-image-mlx in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U git+https://github.com/xocialize/boogu-image-mlx.git fastapi uvicorn python-multipart pillow"')
                 runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "ddcolor" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "ddcolor"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: DDColor MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2634,7 +2665,7 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; then')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "inpaint" ] || { [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ] && printf "%s" "$ODYSSEUS_MLX_IMAGE_MODEL" | grep -Eqi "mi-gan|migan|lama"; }; then')
                 runner_lines.append('  if ! "$ODYSSEUS_MLX_IMAGE_CMD_PY" -c "import PIL" >/dev/null 2>&1; then')
                 runner_lines.append('    echo "ERROR: LaMa / MI-GAN MLX serving needs Pillow in the launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
                 runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U fastapi uvicorn python-multipart pillow huggingface_hub"')
@@ -2654,10 +2685,12 @@ def setup_cookbook_routes() -> APIRouter:
                 runner_lines.append('      ODYSSEUS_PREFLIGHT_EXIT=127')
                 runner_lines.append('    fi')
                 runner_lines.append('  fi')
-                runner_lines.append('elif ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
-                runner_lines.append('  echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
-                runner_lines.append('  echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
-                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('elif [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "mflux" ] || [ "$ODYSSEUS_MLX_IMAGE_ADAPTER" = "auto" ]; then')
+                runner_lines.append('  if ! command -v mflux-generate >/dev/null 2>&1 && ! command -v mflux-generate-qwen >/dev/null 2>&1; then')
+                runner_lines.append('    echo "ERROR: mflux-compatible MLX image serving requires mflux-generate or mflux-generate-qwen in PATH for launch Python: $ODYSSEUS_MLX_IMAGE_CMD_PY."')
+                runner_lines.append('    echo "Install with: $ODYSSEUS_MLX_IMAGE_CMD_PY -m pip install -U mflux fastapi uvicorn python-multipart"')
+                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
+                runner_lines.append('  fi')
                 runner_lines.append('fi')
             elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                 runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
@@ -3516,12 +3549,19 @@ def setup_cookbook_routes() -> APIRouter:
             return {"ok": False, "error": str(e)}
 
     @router.get("/api/cookbook/hf-latest")
-    async def hf_latest(vram_gb: float = 0, limit: int = 10, pipeline: str = "text-generation", owner: str = Depends(require_user)):
+    async def hf_latest(
+        vram_gb: float = 0,
+        limit: int = 10,
+        pipeline: str = "text-generation",
+        official_only: bool = False,
+        owner: str = Depends(require_user),
+    ):
         """Fetch latest HuggingFace models, filtered by what fits in available VRAM.
 
         vram_gb: total available VRAM in GB. 0 = no filter (return everything).
         limit:   how many models to return (default 10).
         pipeline: HF pipeline_tag filter (text-generation, text-to-image, etc.).
+        official_only: restrict results to recognized first-party provider namespaces.
         """
         import re
         import httpx
@@ -3587,6 +3627,20 @@ def setup_cookbook_routes() -> APIRouter:
                     return True
             return False
 
+        # HF does not expose a universal "first-party" flag. Keep this as a
+        # namespace policy rather than a model-name list, so newly published
+        # provider models are included without recommending community forks.
+        OFFICIAL_NAMESPACES = {
+            "apple", "black-forest-labs", "deepseek-ai", "google", "lightricks",
+            "meta-llama", "microsoft", "mistralai", "nvidia", "openai", "qwen",
+            "stabilityai", "tencent", "runwayml",
+        }
+
+        def _is_official(entry: dict, repo_id: str) -> bool:
+            namespace = repo_id.split("/", 1)[0].strip().lower() if "/" in repo_id else ""
+            author = str(entry.get("author") or "").strip().lower()
+            return namespace in OFFICIAL_NAMESPACES and (not author or author == namespace)
+
         out = []
         for entry in raw:
             repo_id = entry.get("modelId") or entry.get("id") or ""
@@ -3601,6 +3655,8 @@ def setup_cookbook_routes() -> APIRouter:
             # Skip adapters, LoRAs, datasets, etc.
             if _is_excluded(repo_id, tags):
                 continue
+            if official_only and not _is_official(entry, repo_id):
+                continue
 
             est_fp16 = _est_vram_fp16(repo_id)
             quant_mult = _quant_factor(repo_id, tags)
@@ -3614,7 +3670,11 @@ def setup_cookbook_routes() -> APIRouter:
                     # if we cannot estimate size from the repo id/tags, do not
                     # present it as runnable on this hardware.
                     continue
-                if needed_vram > vram_gb:
+                # Leave allocator/runtime headroom instead of treating the
+                # reported total as a safe load budget. This keeps the
+                # official-only list honest on tight GPUs as well.
+                usable_vram = vram_gb * 0.90
+                if needed_vram > usable_vram:
                     continue
 
             out.append({
@@ -4412,6 +4472,7 @@ def setup_cookbook_routes() -> APIRouter:
 
             progress_text = ""
             full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""
+            _persisted_terminal = False
 
             if local_win_task:
                 # File-based liveness + output for the detached-process model.
@@ -4445,9 +4506,10 @@ def setup_cookbook_routes() -> APIRouter:
                     and bool(full_snapshot)
                     and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
                 )
-                if _task_status in {"stopped", "done", "completed",
+                _persisted_terminal = _task_status in {"stopped", "done", "completed",
                                     "crashed", "error", "failed",
-                                    "ended", "killed"} and not _persisted_serve_ready:
+                                    "ended", "killed"} and not _persisted_serve_ready
+                if _persisted_terminal:
                     is_alive = False
                     # Keep the persisted output_tail for the UI — it's
                     # what the agent uses to diagnose past failures.
@@ -4486,7 +4548,9 @@ def setup_cookbook_routes() -> APIRouter:
                 and (
                     ".incomplete" in full_snapshot
                     or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
-                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
+                    or (not _persisted_terminal and _download_cache_incomplete(
+                        _payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or ""
+                    ))
                 )
             )
             if is_alive or (local_win_task and full_snapshot):
@@ -4538,6 +4602,7 @@ def setup_cookbook_routes() -> APIRouter:
                         progress_text = "Download complete"
                 elif (
                     task_type == "download"
+                    and not _persisted_terminal
                     and not download_has_incomplete_evidence
                     and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                 ):

routes/document/document_routes.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 from typing import Dict, Any, List, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form
+from fastapi.responses import HTMLResponse
 
 from sqlalchemy import case, func, or_
 from core.database import SessionLocal, Document, DocumentVersion
@@ -479,6 +480,32 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
         finally:
             db.close()
 
+    # ---- GET /api/document/{doc_id}/visual-report ----
+    @router.get("/api/document/{doc_id}/visual-report", response_class=HTMLResponse)
+    async def document_visual_report(request: Request, doc_id: str) -> HTMLResponse:
+        """Render a Markdown document with the same standalone report UI used by Deep Research."""
+        user = get_current_user(request)
+        db = SessionLocal()
+        try:
+            doc = db.query(Document).filter(Document.id == doc_id).first()
+            if not doc:
+                raise HTTPException(404, "Document not found")
+            _verify_doc_owner(db, doc, user)
+            if (doc.language or "").lower() != "markdown":
+                raise HTTPException(400, "Visual reports are available for Markdown documents")
+
+            from src.visual_report import generate_visual_report
+
+            html_content = generate_visual_report(
+                question=doc.title or "Document",
+                report_markdown=doc.current_content or "",
+                sources=[],
+                stats={},
+            )
+            return HTMLResponse(content=html_content)
+        finally:
+            db.close()
+
     # ---- POST /api/document/{doc_id}/archive — soft-archive / restore ----
     @router.post("/api/document/{doc_id}/archive")
     async def archive_document(request: Request, doc_id: str, archived: bool = Query(True)) -> Dict[str, Any]:
@@ -575,7 +602,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
             "markdown": ".md", "json": ".json", "yaml": ".yml", "bash": ".sh",
             "sql": ".sql", "rust": ".rs", "go": ".go", "java": ".java", "c": ".c",
             "cpp": ".cpp", "typescript": ".ts", "ruby": ".rb", "php": ".php",
-            "text": ".txt", "xml": ".xml", "toml": ".toml", "ini": ".ini",
+            "text": ".txt", "email": ".eml", "xml": ".xml", "toml": ".toml", "ini": ".ini",
         }
         db = SessionLocal()
         try:
@@ -602,7 +629,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
                         name = f"{base}-{i}" + ("" if "." in base else ext)
                         i += 1
                     used.add(name)
-                    zf.writestr(name, doc.current_content or "")
+                    content = doc.current_content or ""
+                    if (doc.language or "").lower() == "email":
+                        content = re.sub(r"\r?\n---\r?\n", "\r\n\r\n", content, count=1)
+                    zf.writestr(name, content)
                     wrote += 1
             if not wrote:
                 raise HTTPException(404, "No documents found")

routes/editor_draft_routes.py

@@ -26,6 +26,7 @@ from pydantic import BaseModel
 
 from core.database import EditorDraft, SessionLocal
 from src.auth_helpers import get_current_user
+from src.upload_limits import EDITOR_DRAFT_MAX_BYTES
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +76,16 @@ def _load_payload(raw: Optional[str]) -> Dict[str, Any]:
     return payload if isinstance(payload, dict) else {}
 
 
+def _dump_payload(payload: Dict[str, Any]) -> str:
+    raw = json.dumps(payload or {}, separators=(",", ":"))
+    if len(raw.encode("utf-8")) > EDITOR_DRAFT_MAX_BYTES:
+        raise HTTPException(
+            413,
+            f"Editor draft exceeds the {EDITOR_DRAFT_MAX_BYTES // (1024 * 1024)} MB safety limit",
+        )
+    return raw
+
+
 def setup_editor_draft_routes() -> APIRouter:
     router = APIRouter(tags=["editor-drafts"])
 
@@ -120,13 +131,15 @@ def setup_editor_draft_routes() -> APIRouter:
                 source_image_id=body.source_image_id,
                 width=body.width,
                 height=body.height,
-                payload=json.dumps(body.payload or {}),
+                payload=_dump_payload(body.payload),
                 thumbnail=body.thumbnail,
             )
             db.add(d)
             db.commit()
             db.refresh(d)
             return _summary(d)
+        except HTTPException:
+            raise
         except Exception as e:
             db.rollback()
             logger.warning(f"editor-draft create failed: {e}")
@@ -151,7 +164,7 @@ def setup_editor_draft_routes() -> APIRouter:
             if body.height is not None:
                 d.height = body.height
             if body.payload is not None:
-                d.payload = json.dumps(body.payload)
+                d.payload = _dump_payload(body.payload)
             if body.thumbnail is not None:
                 d.thumbnail = body.thumbnail
             db.commit()

routes/email_helpers.py

@@ -886,10 +886,16 @@ def _init_scheduled_db():
             size INTEGER DEFAULT 0,
             flags TEXT DEFAULT '',
             has_attachments INTEGER DEFAULT 0,
+            attachment_names TEXT DEFAULT '',
             updated_at TEXT NOT NULL,
             PRIMARY KEY (owner, account_key, folder, uid)
         )
     """)
+    _message_index_cols = {
+        row[1] for row in conn.execute("PRAGMA table_info(email_message_index)").fetchall()
+    }
+    if "attachment_names" not in _message_index_cols:
+        conn.execute("ALTER TABLE email_message_index ADD COLUMN attachment_names TEXT DEFAULT ''")
     conn.execute("""
         CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
         ON email_message_index(owner, account_key, folder, date_epoch DESC)
@@ -1667,7 +1673,15 @@ def _extract_text(msg):
         payload = msg.get_payload(decode=True)
         if payload:
             charset = msg.get_content_charset() or "utf-8"
-            return payload.decode(charset, errors="replace")
+            text = payload.decode(charset, errors="replace")
+            if msg.get_content_type() == "text/html":
+                text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
+                text = re.sub(r"</(?:p|div|li|tr|h[1-6])\s*>", "\n", text, flags=re.I)
+                text = re.sub(r"<[^>]+>", "", text)
+                text = html.unescape(text)
+                text = re.sub(r"[ \t]+\n", "\n", text)
+                text = re.sub(r"\n{3,}", "\n\n", text)
+            return text.strip()
     return ""
 
 
@@ -1998,6 +2012,9 @@ class SendEmailRequest(BaseModel):
     # answered after successful delivery so it leaves undone/reply-soon views.
     source_uid: Optional[str] = None
     source_folder: Optional[str] = None
+    # Exact IMAP draft to remove after successful delivery.
+    draft_uid: Optional[str] = None
+    draft_folder: Optional[str] = None
     # Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
     odysseus_kind: Optional[str] = None
     # If true, /send waits for SMTP + Sent append and returns the sent UID.

routes/email_pollers.py

@@ -228,6 +228,9 @@ def _ensure_away_reply_table():
 
 
 def _sender_is_automated(msg, sender_addr: str) -> bool:
+    subject = str(msg.get("Subject") or "").lower()
+    if re.search(r"automatic\s+reply|auto(?:matic)?[- ]?reply|out\s+of\s+office|\booo\b|r[ée]ponse\s+automatique", subject):
+        return True
     auto_submitted = (msg.get("Auto-Submitted") or "").strip().lower()
     if auto_submitted and auto_submitted != "no":
         return True
@@ -244,6 +247,31 @@ def _sender_is_automated(msg, sender_addr: str) -> bool:
     }
 
 
+def _remove_urgent_tag_from_cache(message_id: str, owner: str, account_id: str) -> None:
+    """Remove stale urgent tags from messages identified as automated."""
+    import sqlite3 as _sql3
+    conn = _sql3.connect(SCHEDULED_DB)
+    try:
+        owner_clause, owner_params = _email_cache_owner_clause(owner)
+        rows = conn.execute(
+            f"SELECT rowid, tags FROM email_tags WHERE message_id=? AND {owner_clause} "
+            "AND (account_id=? OR account_id='' OR account_id IS NULL)",
+            (message_id, *owner_params, account_id or ""),
+        ).fetchall()
+        for rowid, raw_tags in rows:
+            try:
+                tags = json.loads(raw_tags or "[]")
+            except Exception:
+                tags = []
+            if not isinstance(tags, list) or "urgent" not in tags:
+                continue
+            cleaned = [tag for tag in tags if str(tag).strip().lower() != "urgent"]
+            conn.execute("UPDATE email_tags SET tags=? WHERE rowid=?", (json.dumps(cleaned), rowid))
+        conn.commit()
+    finally:
+        conn.close()
+
+
 def _away_reply_already_sent(settings: dict, account_owner: str, account_id: str | None,
                              message_id: str, sender_addr: str) -> bool:
     import sqlite3 as _sql3
@@ -712,6 +740,9 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                     _, _from_addr_only = email.utils.parseaddr(_from_raw)
                 except Exception:
                     _from_addr_only = ""
+                _is_automated = _sender_is_automated(msg, _from_addr_only)
+                if _is_automated and auto_tag:
+                    _remove_urgent_tag_from_cache(message_id, account_owner or "", account_id or "")
                 _is_self_mail = bool(_self_self_addr) and _from_addr_only.lower() == _self_self_addr
                 need_sum = auto_sum and message_id not in _sum_existing
                 need_reply = auto_reply_draft and message_id not in _reply_existing
@@ -1286,6 +1317,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
                             tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
                             tags = ["marketing" if t == "promo" else t for t in tags]
                             tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
+                            if _is_automated:
+                                tags = [t for t in tags if t != "urgent"]
                             is_spam = bool(parsed.get("spam"))
                             spam_reason = str(parsed.get("reason") or "")[:200]
 

routes/history/history_routes.py

@@ -1,6 +1,7 @@
 """History routes — session history, truncation, fork, conversation topics."""
 
 import json
+import os
 import uuid
 import logging
 import re
@@ -19,6 +20,7 @@ from routes.session_routes import (
     _reject_compact_during_active_run,
     _verify_session_owner,
 )
+from routes.chat_helpers import strip_tui_local_context
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +28,105 @@ _HISTORY_INLINE_MEDIA_THRESHOLD = 200_000
 _DATA_IMAGE_RE = re.compile(r"data:image/[^;,\"]+;base64,[A-Za-z0-9+/=\s]+")
 
 
+def _sft_trace_file_for_owner(owner: str | None) -> str | None:
+    if not str(owner or "").startswith("sft_"):
+        return None
+    flag = os.getenv("ODYSSEUS_SFT_TRACE_CAPTURE", "1").strip().lower()
+    if flag in {"0", "false", "no", "off"}:
+        return None
+    try:
+        from src.constants import DATA_DIR
+        trace_dir = os.getenv("ODYSSEUS_SFT_TRACE_DIR") or os.path.join(DATA_DIR, "sft_traces")
+        return os.path.join(trace_dir, f"{owner}.jsonl")
+    except Exception:
+        return None
+
+
+def _remove_deleted_sft_trace_rows(
+    *,
+    owner: str | None,
+    session_id: str,
+    deleted_pairs: list[dict[str, str]],
+) -> None:
+    """Keep the training JSONL aligned with user-deleted chat attempts."""
+    path = _sft_trace_file_for_owner(owner)
+    if not path or not deleted_pairs or not os.path.exists(path):
+        return
+    try:
+        kept: list[str] = []
+        removed: list[str] = []
+        with open(path, "r", encoding="utf-8") as f:
+            for line in f:
+                raw = line.rstrip("\n")
+                if not raw.strip():
+                    continue
+                try:
+                    row = json.loads(raw)
+                except json.JSONDecodeError:
+                    kept.append(raw)
+                    continue
+                if row.get("session_id") != session_id:
+                    kept.append(raw)
+                    continue
+                row_user = str(row.get("user") or "").strip()
+                row_assistant = str(row.get("assistant") or "").strip()
+                should_remove = any(
+                    row_user == pair.get("user", "").strip()
+                    and row_assistant == pair.get("assistant", "").strip()
+                    for pair in deleted_pairs
+                )
+                if should_remove:
+                    tombstone = dict(row)
+                    tombstone["deleted_from_training"] = True
+                    removed.append(json.dumps(tombstone, ensure_ascii=False))
+                else:
+                    kept.append(raw)
+        with open(path, "w", encoding="utf-8") as f:
+            for raw in kept:
+                f.write(raw + "\n")
+        if removed:
+            trash_path = path + ".trash"
+            with open(trash_path, "a", encoding="utf-8") as f:
+                for raw in removed:
+                    f.write(raw + "\n")
+            logger.info(
+                "Removed %d SFT trace row(s) for deleted messages in session %s",
+                len(removed),
+                session_id,
+            )
+    except Exception as exc:
+        logger.warning("Failed to prune SFT trace rows for %s: %s", session_id, exc)
+
+
+def _deleted_sft_pairs_from_db_rows(rows: list[DbChatMessage]) -> list[dict[str, str]]:
+    """Build user/assistant pairs affected by deleted messages.
+
+    The SFT trace row is one assistant turn paired with the nearest preceding
+    user turn. If the user deletes either side of a failed attempt before
+    retrying, remove that pair from the training JSONL.
+    """
+    pairs: list[dict[str, str]] = []
+    last_user = ""
+    pending_deleted_user = ""
+    for row in rows:
+        role = str(getattr(row, "role", "") or "")
+        content = str(getattr(row, "content", "") or "").strip()
+        will_delete = bool(getattr(row, "_will_delete_for_sft", False))
+        if role == "user":
+            last_user = content
+            if will_delete:
+                pending_deleted_user = content
+            continue
+        if role != "assistant":
+            continue
+        if will_delete and last_user:
+            pairs.append({"user": last_user, "assistant": content})
+        elif pending_deleted_user:
+            pairs.append({"user": pending_deleted_user, "assistant": content})
+            pending_deleted_user = ""
+    return pairs
+
+
 def _history_display_content(content: Any) -> Any:
     """Return a lightweight browser-display copy of stored message content.
 
@@ -100,6 +201,41 @@ def _merge_continue_rows_to_delete(db_messages, db1, db2):
     return to_delete
 
 
+def _is_continue_interruption_message(message: Any) -> bool:
+    if isinstance(message, ChatMessage):
+        role = message.role
+        content = message.content
+    elif isinstance(message, dict):
+        role = message.get("role", "")
+        content = message.get("content", "")
+    else:
+        role = getattr(message, "role", "")
+        content = getattr(message, "content", "")
+    normalized = " ".join(str(content or "").strip().lower().split())
+    return role == "user" and (
+        "previous response was interrupted" in normalized
+        or normalized in {
+            "continue from where you left off.",
+            "continue from where you left off",
+        }
+    )
+
+
+def _has_immediate_continue_marker(messages: list[Any], idx1: int, idx2: int) -> bool:
+    return idx2 - idx1 == 2 and _is_continue_interruption_message(messages[idx1 + 1])
+
+
+def _keep_count_before_message(db_messages, before_msg_id: str | None) -> int | None:
+    """Return the durable-history keep count before a DB message id."""
+    wanted = str(before_msg_id or "").strip()
+    if not wanted:
+        return None
+    for pos, row in enumerate(db_messages):
+        if str(getattr(row, "id", "")) == wanted:
+            return pos
+    return None
+
+
 def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
     router = APIRouter(tags=["history"])
 
@@ -124,13 +260,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             )
 
     def _db_history_entry(m: DbChatMessage) -> Dict[str, Any]:
-        entry = {"role": m.role, "content": _history_display_content(m.content)}
+        entry = {"role": m.role, "content": strip_tui_local_context(_history_display_content(m.content))}
         meta = {}
         if m.meta_data:
             try:
                 meta = json.loads(m.meta_data) or {}
             except (json.JSONDecodeError, ValueError):
                 meta = {}
+        meta["_db_id"] = m.id
         if m.timestamp and "timestamp" not in meta:
             meta["timestamp"] = m.timestamp.isoformat() + "Z"
         if meta:
@@ -199,7 +336,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Skip hidden messages (e.g. compaction summaries for AI context)
                 if msg.metadata and msg.metadata.get("hidden"):
                     continue
-                entry = {"role": msg.role, "content": _history_display_content(msg.content)}
+                entry = {"role": msg.role, "content": strip_tui_local_context(_history_display_content(msg.content))}
                 if msg.metadata:
                     entry["metadata"] = msg.metadata
                 history_dict.append(entry)
@@ -208,7 +345,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     continue
                 entry = {
                     "role": msg.get("role", ""),
-                    "content": _history_display_content(msg.get("content", "")),
+                    "content": strip_tui_local_context(_history_display_content(msg.get("content", ""))),
                 }
                 if msg.get("metadata"):
                     entry["metadata"] = msg["metadata"]
@@ -249,11 +386,36 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
         _verify_session_owner(request, session_id)
         try:
             body = await request.json()
-            keep_count = body.get("keep_count", 0)
+            keep_count = int(body.get("keep_count", 0))
+            before_msg_id = str(body.get("before_msg_id") or body.get("message_id") or "").strip()
+            deleted_sft_pairs: list[dict[str, str]] = []
+            if keep_count >= 0:
+                db = SessionLocal()
+                try:
+                    all_db_messages = db.query(DbChatMessage).filter(
+                        DbChatMessage.session_id == session_id
+                    ).order_by(DbChatMessage.timestamp).all()
+                    if before_msg_id:
+                        resolved_keep_count = _keep_count_before_message(all_db_messages, before_msg_id)
+                        if resolved_keep_count is None:
+                            raise HTTPException(404, "Message not found")
+                        keep_count = resolved_keep_count
+                    for pos, row in enumerate(all_db_messages):
+                        row._will_delete_for_sft = pos >= keep_count
+                    deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+                finally:
+                    db.close()
             result = session_manager.truncate_messages(session_id, keep_count)
+            _remove_deleted_sft_trace_rows(
+                owner=effective_user(request),
+                session_id=session_id,
+                deleted_pairs=deleted_sft_pairs,
+            )
             return {"status": "ok", "kept": keep_count, "truncated": result}
         except KeyError:
             raise HTTPException(404, "Session not found")
+        except HTTPException:
+            raise
         except Exception as e:
             logger.error(f"Truncate error {session_id}: {e}")
             raise HTTPException(500, str(e))
@@ -288,6 +450,18 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             session = session_manager.get_session(session_id)
             db = SessionLocal()
             try:
+                all_db_messages = db.query(DbChatMessage).filter(
+                    DbChatMessage.session_id == session_id
+                ).order_by(DbChatMessage.timestamp).all()
+                delete_id_set = set(msg_ids or [])
+                delete_index_set = set(indices or [])
+                for pos, row in enumerate(all_db_messages):
+                    row._will_delete_for_sft = (
+                        (bool(delete_id_set) and row.id in delete_id_set)
+                        or (not delete_id_set and bool(delete_index_set) and pos in delete_index_set)
+                    )
+                deleted_sft_pairs = _deleted_sft_pairs_from_db_rows(all_db_messages)
+
                 if msg_ids:
                     # New ID-based delete
                     deleted = 0
@@ -330,6 +504,11 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                     db_session.updated_at = datetime.now(timezone.utc)
 
                 db.commit()
+                _remove_deleted_sft_trace_rows(
+                    owner=effective_user(request),
+                    session_id=session_id,
+                    deleted_pairs=deleted_sft_pairs,
+                )
                 return {"status": "ok", "deleted": deleted}
             finally:
                 db.close()
@@ -520,6 +699,9 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 return {"status": "ok", "merged": False}
 
             idx1, idx2 = ai_indices[-2], ai_indices[-1]
+            if not _has_immediate_continue_marker(session.history, idx1, idx2):
+                return {"status": "ok", "merged": False, "reason": "no_continue_marker"}
+
             msg1, msg2 = session.history[idx1], session.history[idx2]
 
             content1 = msg1.content if isinstance(msg1, ChatMessage) else msg1.get('content', '')
@@ -530,7 +712,14 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             meta1 = (msg1.metadata if isinstance(msg1, ChatMessage) else msg1.get('metadata')) or {}
             meta2 = (msg2.metadata if isinstance(msg2, ChatMessage) else msg2.get('metadata')) or {}
             merged_meta = {**meta1, **meta2}
+            thinking1 = str(meta1.get('thinking') or '').strip()
+            thinking2 = str(meta2.get('thinking') or '').strip()
+            if thinking1 and thinking2:
+                merged_meta['thinking'] = thinking1 + "\n\n(continued)\n\n" + thinking2
+            elif thinking1:
+                merged_meta['thinking'] = thinking1
             merged_meta.pop('stopped', None)  # no longer stopped after continue
+            merged_meta.pop('thinking_interrupted', None)
 
             # Update first message, remove second
             if isinstance(msg1, ChatMessage):
@@ -542,13 +731,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
 
             # Also remove the hidden "continue" user message between them if present
             # It's the message at idx2-1 if it's a user message with continue text
-            remove_indices = [idx2]
-            if idx2 - 1 > idx1:
-                between = session.history[idx2 - 1]
-                between_role = between.role if isinstance(between, ChatMessage) else between.get('role', '')
-                between_content = between.content if isinstance(between, ChatMessage) else between.get('content', '')
-                if between_role == 'user' and 'previous response was interrupted' in between_content:
-                    remove_indices.insert(0, idx2 - 1)
+            remove_indices = [idx2, idx1 + 1]
 
             for ri in sorted(remove_indices, reverse=True):
                 session.history.pop(ri)
@@ -566,19 +749,20 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 # Find last two assistant messages in DB
                 ai_db = [(i, m) for i, m in enumerate(db_messages) if m.role == 'assistant']
                 if len(ai_db) >= 2:
-                    (_, db1), (_, db2) = ai_db[-2], ai_db[-1]
-                    db1.content = merged_content
-                    db1.meta_data = _json.dumps(merged_meta)
+                    (db_idx1, db1), (db_idx2, db2) = ai_db[-2], ai_db[-1]
+                    if _has_immediate_continue_marker(db_messages, db_idx1, db_idx2):
+                        db1.content = merged_content
+                        db1.meta_data = _json.dumps(merged_meta)
 
-                    # Mirror the in-memory deletion: remove the second assistant
-                    # message and ONLY the "continue" user message between them
-                    # (not arbitrary tool/system/user rows). The old
-                    # range-delete destroyed every row between the two assistant
-                    # messages, desyncing the DB from the in-memory history.
-                    for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
-                        db.delete(_row)
+                        # Mirror the in-memory deletion: remove the second assistant
+                        # message and ONLY the "continue" user message between them
+                        # (not arbitrary tool/system/user rows). The old
+                        # range-delete destroyed every row between the two assistant
+                        # messages, desyncing the DB from the in-memory history.
+                        for _row in _merge_continue_rows_to_delete(db_messages, db1, db2):
+                            db.delete(_row)
 
-                    db.commit()
+                        db.commit()
             finally:
                 db.close()
             session_manager.save_sessions()
@@ -672,6 +856,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             raise HTTPException(404, "Session not found")
 
         try:
+            from src.context_compactor import auto_compact_threshold_percent
             from src.model_context import estimate_tokens, get_context_length
 
             messages = session.get_context_messages()
@@ -679,6 +864,7 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
             ctx_len = int(get_context_length(session.endpoint_url, session.model) or 0)
             pct = round((used / ctx_len) * 100, 1) if ctx_len else 0.0
             pct = max(0.0, min(100.0, pct))
+            auto_threshold = auto_compact_threshold_percent()
             visible_messages = sum(
                 1 for m in session.history
                 if not (getattr(m, "metadata", None) or {}).get("hidden")
@@ -699,13 +885,119 @@ def setup_history_routes(session_manager, upload_handler=None) -> APIRouter:
                 "context_messages": len(messages),
                 "compacted_messages": compacted_messages,
                 "can_compact": can_compact,
-                "should_compact": pct >= 70,
-                "auto_compact_threshold": 85,
+                "should_compact": pct >= auto_threshold,
+                "auto_compact_threshold": auto_threshold,
+                "memory_extraction_enabled": getattr(session, "memory_extraction_enabled", True) is not False,
+                "skill_injection_enabled": getattr(session, "skill_injection_enabled", True) is not False,
+                "thinking_mode": getattr(session, "thinking_mode", "") or "off",
+                "temperature_override": getattr(session, "temperature_override", None),
+                "max_tokens_override": getattr(session, "max_tokens_override", None),
             }
         except Exception as e:
             logger.error(f"Context usage error {session_id}: {e}")
             raise HTTPException(500, str(e))
 
+    @router.post("/api/session/{session_id}/memory-extraction")
+    async def set_session_memory_extraction(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle automatic memory extraction for one chat session."""
+        _verify_session_owner(request, session_id)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                raise HTTPException(404, "Session not found")
+            db_session.memory_extraction_enabled = enabled
+            db.commit()
+            session.memory_extraction_enabled = enabled
+            return {"status": "success", "memory_extraction_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Memory extraction toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update memory extraction")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/skill-injection")
+    async def set_session_skill_injection(request: Request, session_id: str) -> Dict[str, Any]:
+        """Toggle skill injection for one chat session."""
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        if "enabled" not in body:
+            raise HTTPException(400, "Missing enabled")
+        enabled = bool(body.get("enabled"))
+
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not db_session:
+                # Some active chats exist only in the in-memory manager until
+                # their first persisted write. Keep the toggle usable there.
+                session.skill_injection_enabled = enabled
+                session_manager.save_sessions()
+                return {"status": "success", "skill_injection_enabled": enabled}
+            db_session.skill_injection_enabled = enabled
+            db.commit()
+            session.skill_injection_enabled = enabled
+            return {"status": "success", "skill_injection_enabled": enabled}
+        except HTTPException:
+            raise
+        except Exception as e:
+            db.rollback()
+            logger.error(f"Skill injection toggle error {session_id}: {e}")
+            raise HTTPException(500, "Failed to update skill injection")
+        finally:
+            db.close()
+
+    @router.post("/api/session/{session_id}/generation-settings")
+    async def set_session_generation_settings(request: Request, session_id: str) -> Dict[str, Any]:
+        _verify_session_owner(request, session_id, session_manager)
+        try:
+            session = session_manager.get_session(session_id)
+            body = await request.json()
+        except KeyError:
+            raise HTTPException(404, "Session not found")
+        mode = str(body.get("thinking_mode") or "").lower()
+        if mode not in {"", "on", "off"}:
+            raise HTTPException(400, "Invalid thinking mode")
+        temperature = body.get("temperature_override")
+        temperature = None if temperature in (None, "") else max(0.0, min(2.0, float(temperature)))
+        max_tokens = body.get("max_tokens_override")
+        max_tokens = None if max_tokens in (None, "", 0) else max(256, min(32768, int(max_tokens)))
+        db = SessionLocal()
+        try:
+            row = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if not row:
+                raise HTTPException(404, "Session not found")
+            row.thinking_mode, row.temperature_override, row.max_tokens_override = mode, temperature, max_tokens
+            db.commit()
+            session.thinking_mode, session.temperature_override, session.max_tokens_override = mode, temperature, max_tokens
+            return {"status": "success", "thinking_mode": mode, "temperature_override": temperature, "max_tokens_override": max_tokens}
+        finally:
+            db.close()
+
     @router.post("/api/session/{session_id}/compact")
     async def compact_session(request: Request, session_id: str):
         """Manually trigger context compaction for a session."""

routes/hwfit_routes.py

@@ -16,6 +16,24 @@ from routes._validators import validate_remote_host, validate_ssh_port
 # "metal" routes through the Apple-Silicon path (GGUF-only, llama.cpp/Ollama),
 # the CPU backends through the RAM/offload path, cuda/rocm through vLLM.
 _MANUAL_BACKENDS = {"cuda", "rocm", "metal", "cpu_x86", "cpu_arm"}
+_OFFICIAL_NAMESPACES = {
+    "apple", "allenai", "black-forest-labs", "cohere", "deepseek-ai",
+    "google", "ibm", "lightricks", "meta-llama", "microsoft", "mistralai",
+    "nvidia", "openai", "qwen", "stabilityai", "tencent", "tiiuae",
+    "upstage", "zai-org", "runwayml",
+}
+
+
+def _is_official_model(model: dict) -> bool:
+    """Recognize first-party namespaces without maintaining model-name lists."""
+    # Image rows expose a friendly `name` without its namespace, while regular
+    # rows may use `name`. Prefer whichever field still contains `owner/repo`.
+    model_id = str(model.get("id") or model.get("name") or "")
+    namespace = model_id.split("/", 1)[0].strip().lower() if "/" in model_id else ""
+    # `provider` is a display label for image rows (for example, "Stability AI")
+    # and is not a stable repository namespace. The model id is the canonical
+    # source for this filter, so a recognized namespace is sufficient.
+    return namespace in _OFFICIAL_NAMESPACES
 
 
 def _validate_detection_target(host: str = "", ssh_port: str = "") -> tuple[str, str]:
@@ -191,7 +209,7 @@ def setup_hwfit_routes():
         return detect_system(host=host, ssh_port=ssh_port, platform=platform, fresh=fresh)
 
     @router.get("/models")
-    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False):
+    def get_models(use_case: str = "", sort: str = "newest", limit: int = 50, search: str = "", host: str = "", quant: str = "", ctx: str = "", gpu_count: str = "", gpu_group: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, refresh_catalog: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, fit_only: bool = False, official_only: bool = False):
         """Rank LLM models against detected hardware and return scored results.
         gpu_count: override GPU count (0 = CPU only, 1-N = simulate N GPUs of the
             active group). gpu_group: index into system.gpu_groups (the homogeneous
@@ -310,6 +328,8 @@ def setup_hwfit_routes():
             rank_kwargs.pop("target_context", None)
             rank_kwargs.pop("fit_only", None)
         results = rank_models(system, **rank_kwargs)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         payload = {"system": system, "models": results}
         if catalog_refresh is not None:
             payload["catalog_refresh"] = catalog_refresh
@@ -410,7 +430,7 @@ def setup_hwfit_routes():
         }
 
     @router.get("/image-models")
-    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False):
+    def get_image_models(sort: str = "fit", search: str = "", host: str = "", gpu_count: str = "", ssh_port: str = "", platform: str = "", fresh: bool = False, manual_mode: str = "", manual_gpu_count: str = "", manual_vram_gb: str = "", manual_ram_gb: str = "", manual_backend: str = "", ignore_detected_gpu: bool = False, ignore_detected_ram: bool = False, official_only: bool = False):
         """Rank image generation models against detected hardware."""
         from services.hwfit.hardware import detect_system
         from services.hwfit.image_models import rank_image_models
@@ -451,6 +471,8 @@ def setup_hwfit_routes():
             system["gpu_count"] = 1 if single_vram > 0 else 0
             system["gpu_only"] = True if single_vram > 0 else False
         results = rank_image_models(system, search=search or None, sort=sort)
+        if official_only:
+            results = [m for m in results if _is_official_model(m)]
         return {"system": system, "models": results}
 
     return router

routes/model_routes.py

@@ -17,7 +17,20 @@ from fastapi import APIRouter, HTTPException, Form, Query, Body, Request, Respon
 from pydantic import BaseModel
 from fastapi.responses import StreamingResponse
 from core.database import SessionLocal, ModelEndpoint, Session as DbSession
-from core.log_safety import redact_url as _redact_url_for_log
+try:
+    from core.log_safety import redact_url as _redact_url_for_log
+except ModuleNotFoundError:
+    def _redact_url_for_log(url: str) -> str:
+        try:
+            parsed = urlparse(url or "")
+            host = parsed.hostname or ""
+            if ":" in host:
+                host = f"[{host}]"
+            if parsed.port:
+                host = f"{host}:{parsed.port}"
+            return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
+        except Exception:
+            return "<endpoint>"
 from core.middleware import require_admin
 from src.constants import COOKBOOK_STATE_FILE
 from src.llm_core import _detect_provider, _host_match, ANTHROPIC_MODELS
@@ -455,6 +468,7 @@ def _truthy(value: str | None) -> bool:
 
 _ENDPOINT_KINDS = {"auto", "local", "api", "proxy"}
 _REFRESH_MODES = {"auto", "manual", "disabled"}
+_MODEL_TOOL_MODES = {"none", "compact", "full"}
 
 
 def _normalize_endpoint_kind(value: Any) -> str:
@@ -462,6 +476,30 @@ def _normalize_endpoint_kind(value: Any) -> str:
     return kind if kind in _ENDPOINT_KINDS else "auto"
 
 
+def _normalize_model_tool_mode(value: Any) -> str:
+    mode = str(value or "").strip().lower()
+    return mode if mode in _MODEL_TOOL_MODES else ""
+
+
+def _model_tool_modes(ep: Any) -> Dict[str, str]:
+    raw = getattr(ep, "model_tool_modes", None)
+    if not raw:
+        return {}
+    try:
+        data = json.loads(raw) if isinstance(raw, str) else raw
+    except Exception:
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    modes: Dict[str, str] = {}
+    for key, value in data.items():
+        model_id = str(key or "").strip()
+        mode = _normalize_model_tool_mode(value)
+        if model_id and mode:
+            modes[model_id] = mode
+    return modes
+
+
 def _normalize_refresh_mode(value: Any, endpoint_kind: str = "auto") -> str:
     mode = str(value or "").strip().lower()
     kind = _normalize_endpoint_kind(endpoint_kind)
@@ -1973,6 +2011,7 @@ def setup_model_routes(model_discovery):
                     "ping_error": (ping or {}).get("error") if ping else None,
                     "model_type": getattr(r, "model_type", None) or "llm",
                     "supports_tools": getattr(r, "supports_tools", None),
+                    "model_tool_modes": _model_tool_modes(r),
                     "endpoint_kind": kind,
                     "category": _classify_endpoint(base, kind),
                     "model_refresh_mode": _endpoint_refresh_mode(r, kind),
@@ -2344,6 +2383,7 @@ def setup_model_routes(model_discovery):
                     response.headers["X-Model-Refresh-Warning"] = "Model refresh failed or returned no models; kept cached models."
             _, pinned = _picker_models_for_endpoint(ep, base, kind)
             pinned_set = set(pinned)
+            tool_modes = _model_tool_modes(ep)
             return [
                 {
                     "id": m,
@@ -2351,6 +2391,7 @@ def setup_model_routes(model_discovery):
                     "is_hidden": m in hidden,
                     "is_pinned": m in pinned_set,
                     "picker_requires_pinning": picker_requires_pinning,
+                    "tool_mode": tool_modes.get(m, ""),
                 }
                 for m in _merge_model_ids(all_models, pinned)
             ]
@@ -2401,11 +2442,31 @@ def setup_model_routes(model_discovery):
                     ep.hidden_models = None
                 else:
                     ep.pinned_models = json.dumps(pinned) if pinned else None
+            if "model_tool_modes" in body:
+                raw_modes = body.get("model_tool_modes")
+                if not isinstance(raw_modes, dict):
+                    raise HTTPException(400, "model_tool_modes must be an object")
+                modes = _model_tool_modes(ep)
+                for model_id, mode in raw_modes.items():
+                    model_id = str(model_id or "").strip()
+                    if not model_id:
+                        continue
+                    normalized = _normalize_model_tool_mode(mode)
+                    if normalized:
+                        modes[model_id] = normalized
+                    else:
+                        modes.pop(model_id, None)
+                ep.model_tool_modes = json.dumps(modes) if modes else None
             db.commit()
             _invalidate_models_cache()
             hidden_count = len(json.loads(ep.hidden_models)) if ep.hidden_models else 0
             pinned_count = len(json.loads(ep.pinned_models)) if ep.pinned_models else 0
-            return {"id": ep_id, "hidden_count": hidden_count, "pinned_count": pinned_count}
+            return {
+                "id": ep_id,
+                "hidden_count": hidden_count,
+                "pinned_count": pinned_count,
+                "model_tool_modes": _model_tool_modes(ep),
+            }
         finally:
             db.close()
 
@@ -2572,6 +2633,7 @@ def setup_model_routes(model_discovery):
                 "model_type": ep.model_type,
                 "base_url": ep.base_url,
                 "pinned_models": _normalize_model_ids(getattr(ep, "pinned_models", None)),
+                "model_tool_modes": _model_tool_modes(ep),
                 "endpoint_kind": getattr(ep, "endpoint_kind", None) or "auto",
                 "model_refresh_mode": getattr(ep, "model_refresh_mode", None) or "auto",
                 "model_refresh_interval": getattr(ep, "model_refresh_interval", None),

routes/note/note_routes.py

@@ -35,6 +35,7 @@ class NoteCreate(BaseModel):
     source: str = "user"
     session_id: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = "none"
     sort_order: Optional[int] = None
 
@@ -50,6 +51,7 @@ class NoteUpdate(BaseModel):
     archived: Optional[bool] = None
     due_date: Optional[str] = None
     image_url: Optional[str] = None
+    gallery_id: Optional[str] = None
     repeat: Optional[str] = None
     sort_order: Optional[int] = None
     agent_session_id: Optional[str] = None
@@ -89,6 +91,7 @@ def _note_to_dict(note: Note) -> Dict[str, Any]:
         "session_id": note.session_id,
         "sort_order": note.sort_order or 0,
         "image_url": note.image_url,
+        "gallery_id": getattr(note, "gallery_id", None),
         "repeat": note.repeat or "none",
         "ai_classification": ai_cls,
         "ai_content_hash": getattr(note, "ai_content_hash", None),
@@ -674,6 +677,7 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 source=body.source,
                 session_id=body.session_id,
                 image_url=body.image_url,
+                gallery_id=body.gallery_id,
                 repeat=body.repeat or "none",
                 sort_order=body.sort_order if body.sort_order is not None else 0,
             )
@@ -743,6 +747,8 @@ def setup_note_routes(task_scheduler=None, upload_handler=None):
                 note.due_date = body.due_date
             if body.image_url is not None:
                 note.image_url = body.image_url
+            if body.gallery_id is not None:
+                note.gallery_id = body.gallery_id
             if body.repeat is not None:
                 note.repeat = body.repeat
             if body.sort_order is not None:

routes/preset_routes.py

@@ -21,6 +21,8 @@ class UserTemplateRequest(BaseModel):
     system_prompt: str = Field("", max_length=10000)
     temperature: float = Field(1.0, ge=0.0, le=2.0)
     max_tokens: int = Field(0, ge=0, le=65536)
+    persona_memory: str = Field("", max_length=6000)
+    persona_memory_schema: str = Field("general", pattern="^(general|health)$")
 
 
 def setup_preset_routes(preset_manager) -> APIRouter:
@@ -41,6 +43,10 @@ def setup_preset_routes(preset_manager) -> APIRouter:
                 preset_update.enabled,
                 preset_update.inject_prefix,
                 preset_update.inject_suffix,
+                preset_update.persona_memory,
+                preset_update.persona_memory_schema,
+                preset_update.thinking_mode,
+                preset_update.show_persona_name,
             )
             if success:
                 return {"success": True, "message": "Custom preset updated"}

routes/research/research_routes.py

@@ -7,7 +7,7 @@ import re
 import uuid
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Literal, Optional
 
 from fastapi import APIRouter, HTTPException, Query, Request
 from fastapi.responses import HTMLResponse, StreamingResponse
@@ -271,7 +271,13 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "query": entry.get("query", ""),
                     "status": "running",
                     "progress": entry.get("progress", {}),
+                    "source_state": research_handler.get_source_state(sid),
+                    "source_coverage": research_handler.get_source_coverage(sid),
+                    "navigation_trace": research_handler.get_navigation_trace(sid),
+                    "action_trace": research_handler.get_action_trace(sid),
                     "started_at": entry.get("started_at", 0),
+                    "category": research_handler.get_category(sid),
+                    "mode": research_handler.get_mode(sid),
                 })
         return {"active": active}
 
@@ -284,6 +290,24 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         status = research_handler.get_status(session_id)
         if status is None:
             raise HTTPException(404, "No research found for this session")
+        try:
+            source_state = research_handler.get_source_state(session_id)
+            if isinstance(source_state, str) and source_state:
+                status["source_state"] = source_state
+            source_coverage = research_handler.get_source_coverage(session_id)
+            if isinstance(source_coverage, dict) and source_coverage:
+                status["source_coverage"] = source_coverage
+        except Exception:
+            pass
+        try:
+            navigation_trace = research_handler.get_navigation_trace(session_id)
+            if isinstance(navigation_trace, list) and navigation_trace:
+                status["navigation_trace"] = navigation_trace
+            action_trace = research_handler.get_action_trace(session_id)
+            if isinstance(action_trace, list) and action_trace:
+                status["action_trace"] = action_trace
+        except Exception:
+            pass
         return status
 
     @router.post("/api/research/cancel/{session_id}")
@@ -306,8 +330,26 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        category = research_handler.get_category(session_id)
+        mode = research_handler.get_mode(session_id)
         research_handler.clear_result(session_id)
-        return {"result": result, "sources": sources, "raw_findings": raw_findings}
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": category,
+            "mode": mode,
+        }
 
     def _assert_owns_research(session_id: str, user: str) -> None:
         """404-not-403 ownership gate for a research session's on-disk JSON.
@@ -394,6 +436,8 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "id": p.stem,
                     "query": query,
                     "category": d.get("category") or "",
+                    "mode": d.get("mode") or "research",
+                    "mode": d.get("mode") or "research",
                     "source_count": len(sources),
                     "status": d.get("status", "done"),
                     "duration": d.get("stats", {}).get("Duration", ""),
@@ -479,6 +523,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
 
     class ResearchStartRequest(BaseModel):
         query: str
+        origin_chat_id: Optional[str] = None
         # max_rounds=0 means "Auto" — let the AI decide when to stop, capped at 20.
         max_rounds: int = Field(default=0, ge=0, le=20)
         search_provider: Optional[str] = None
@@ -487,7 +532,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         max_time: int = Field(default=300, ge=60, le=1800)
         extraction_timeout: Optional[int] = Field(default=None, ge=15, le=3600)
         extraction_concurrency: Optional[int] = Field(default=None, ge=1, le=12)
-        category: Optional[str] = None
+        category: Optional[Literal["product", "comparison", "howto", "factcheck"]] = None
 
     @router.post("/api/research/start")
     async def research_start(body: ResearchStartRequest, request: Request):
@@ -509,6 +554,15 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                         pass
                 user = tool_owner
         session_id = f"rp-{uuid.uuid4().hex[:12]}"
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        if body.origin_chat_id:
+            from core.database import SessionLocal, Session as DbSession
+            with SessionLocal() as db:
+                origin = db.get(DbSession, body.origin_chat_id)
+                if origin is None or origin.owner != user:
+                    raise HTTPException(404, 'Origin chat not found')
+            if delivery is None:
+                raise HTTPException(503, 'Background chat delivery is unavailable')
 
         if body.endpoint_id:
             from src.database import SessionLocal
@@ -558,8 +612,12 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             if body.model:
                 ep_model = body.model
 
-        # max_rounds=0 → "Auto", let AI decide; pass 20 as the safety cap.
-        effective_max_rounds = body.max_rounds if body.max_rounds > 0 else 20
+        # 0 = auto research capped at 20.
+        effective_max_rounds = body.max_rounds if body.max_rounds != 0 else 20
+        if body.origin_chat_id and 'max_rounds' not in body.model_fields_set:
+            effective_max_rounds = 2
+        if body.origin_chat_id:
+            delivery.register(session_id, body.origin_chat_id, user, 'research', body.query, effective_max_rounds)
         research_handler.start_research(
             session_id=session_id,
             query=body.query,
@@ -573,8 +631,22 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
             extraction_timeout=body.extraction_timeout,
             extraction_concurrency=body.extraction_concurrency,
             owner=user,
+            on_complete=(lambda sid, result, sources, findings: delivery.complete(sid, result, sources))
+                if body.origin_chat_id else None,
         )
-        return {"session_id": session_id, "status": "running", "query": body.query}
+        return {
+            "session_id": session_id,
+            "status": "running",
+            "query": body.query,
+            "category": body.category or "",
+            "mode": "research",
+        }
+
+    @router.get('/api/research/chat-jobs/{chat_id}')
+    async def chat_research_jobs(chat_id: str, request: Request):
+        user = _require_user(request)
+        delivery = getattr(request.app.state, 'background_tool_jobs', None)
+        return {'jobs': delivery.list_for_chat(chat_id, user) if delivery else []}
 
     @router.get("/api/research/stream/{session_id}")
     async def research_stream(session_id: str, request: Request):
@@ -584,7 +656,7 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
         if not _owns_in_memory(session_id, user):
             raise HTTPException(404, "No research found for this session")
         async def _generate():
-            last_progress = None
+            last_payload = None
             while True:
                 status = research_handler.get_status(session_id)
                 if status is None:
@@ -592,9 +664,30 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     return
                 st = status.get("status", "")
                 progress = status.get("progress", {})
-                if progress != last_progress:
-                    last_progress = progress
-                    yield f"data: {json.dumps({**progress, 'status': st})}\n\n"
+                payload = {
+                    **progress,
+                    'status': st,
+                    'category': research_handler.get_category(session_id),
+                    'mode': research_handler.get_mode(session_id),
+                }
+                try:
+                    source_state = research_handler.get_source_state(session_id)
+                    if source_state:
+                        payload["source_state"] = source_state
+                    source_coverage = research_handler.get_source_coverage(session_id)
+                    if source_coverage:
+                        payload["source_coverage"] = source_coverage
+                    navigation_trace = research_handler.get_navigation_trace(session_id)
+                    if navigation_trace:
+                        payload["navigation_trace"] = navigation_trace
+                    action_trace = research_handler.get_action_trace(session_id)
+                    if action_trace:
+                        payload["action_trace"] = action_trace
+                except Exception:
+                    pass
+                if payload != last_payload:
+                    last_payload = payload
+                    yield f"data: {json.dumps(payload)}\n\n"
                 if st != "running":
                     final = {'status': st, 'final': True}
                     task = research_handler._active_tasks.get(session_id, {})
@@ -625,12 +718,33 @@ def setup_research_routes(research_handler, session_manager=None) -> APIRouter:
                     "result": d.get("result", ""),
                     "sources": d.get("sources", []),
                     "raw_findings": d.get("raw_findings", []),
+                    "analyzed_urls": d.get("analyzed_urls", []),
+                    "source_state": d.get("source_state", ""),
+                    "source_coverage": d.get("source_coverage", {}),
+                    "navigation_trace": d.get("navigation_trace", []),
+                    "action_trace": d.get("action_trace", []),
                     "category": d.get("category") or "",
                 }
             raise HTTPException(404, "No research result available")
         sources = research_handler.get_sources(session_id) or []
         raw_findings = research_handler.get_raw_findings(session_id) or []
-        return {"result": result, "sources": sources, "raw_findings": raw_findings, "category": ""}
+        analyzed_urls = research_handler.get_analyzed_urls(session_id) or []
+        source_state = research_handler.get_source_state(session_id)
+        source_coverage = research_handler.get_source_coverage(session_id)
+        navigation_trace = research_handler.get_navigation_trace(session_id)
+        action_trace = research_handler.get_action_trace(session_id)
+        return {
+            "result": result,
+            "sources": sources,
+            "raw_findings": raw_findings,
+            "analyzed_urls": analyzed_urls,
+            "source_state": source_state,
+            "source_coverage": source_coverage,
+            "navigation_trace": navigation_trace,
+            "action_trace": action_trace,
+            "category": research_handler.get_category(session_id),
+            "mode": research_handler.get_mode(session_id),
+        }
 
     @router.post("/api/research/spinoff/{session_id}")
     async def research_spinoff(session_id: str, request: Request):

routes/search/search_routes.py

@@ -1,9 +1,12 @@
 """Search routes — /api/search/config GET, /api/search POST."""
 
+import html
+import json
 import logging
 from typing import Dict, Any
 
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, Query, Request
+from fastapi.responses import HTMLResponse
 
 import time
 
@@ -39,6 +42,91 @@ async def _request_values(request: Request) -> Dict[str, Any]:
 def setup_search_routes(config) -> APIRouter:
     router = APIRouter(tags=["search"])
 
+    @router.get("/search/web", response_class=HTMLResponse)
+    async def web_search_page(q: str = Query("", min_length=0)) -> HTMLResponse:
+        """Browser-facing search results page for clickable agent web_search rows."""
+        safe_q = str(q or "").strip()
+        title = html.escape(safe_q or "Web search")
+        q_json = json.dumps(safe_q)
+        page = f"""<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>{title} - Odysseus Search</title>
+  <style>
+    :root {{ color-scheme: dark; --bg:#111; --fg:#eee; --muted:#999; --border:#333; --accent:#e05252; }}
+    body {{ margin:0; font:14px/1.45 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; background:var(--bg); color:var(--fg); }}
+    main {{ max-width:900px; margin:0 auto; padding:22px 18px 40px; }}
+    form {{ display:flex; gap:8px; margin:0 0 16px; }}
+    input {{ flex:1; min-width:0; height:34px; padding:0 10px; border:1px solid var(--border); border-radius:7px; background:#181818; color:var(--fg); font:inherit; }}
+    button {{ height:34px; padding:0 13px; border:1px solid color-mix(in srgb,var(--accent) 45%,var(--border)); border-radius:7px; background:color-mix(in srgb,var(--accent) 14%,transparent); color:var(--fg); font:inherit; cursor:pointer; }}
+    h1 {{ margin:0 0 14px; font-size:16px; font-weight:650; }}
+    .status {{ color:var(--muted); font-size:12px; margin:8px 0 14px; }}
+    .result {{ display:block; padding:11px 0; border-top:1px solid var(--border); text-decoration:none; color:inherit; }}
+    .result-title {{ color:var(--fg); font-weight:650; }}
+    .result-url {{ margin-top:3px; color:color-mix(in srgb,var(--accent) 78%,var(--fg)); font-size:12px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }}
+    .result-snippet {{ margin-top:5px; color:color-mix(in srgb,var(--fg) 72%,transparent); font-size:13px; }}
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Web Search</h1>
+    <form id="search-form">
+      <input id="query" value="{html.escape(safe_q, quote=True)}" autocomplete="off">
+      <button type="submit">Search</button>
+    </form>
+    <div class="status" id="status">Loading...</div>
+    <div id="results"></div>
+  </main>
+  <script>
+    const initialQuery = {q_json};
+    const input = document.getElementById('query');
+    const statusEl = document.getElementById('status');
+    const resultsEl = document.getElementById('results');
+    function esc(value) {{
+      return String(value || '').replace(/[&<>"']/g, ch => ({{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}}[ch]));
+    }}
+    async function runSearch(query) {{
+      query = String(query || '').trim();
+      if (!query) {{ statusEl.textContent = 'Enter a search query.'; resultsEl.innerHTML = ''; return; }}
+      statusEl.textContent = 'Searching...';
+      resultsEl.innerHTML = '';
+      const fd = new FormData();
+      fd.append('query', query);
+      const res = await fetch('/api/search', {{ method: 'POST', credentials: 'same-origin', body: fd }});
+      const data = await res.json().catch(() => ({{}}));
+      const sources = Array.isArray(data.sources) ? data.sources : [];
+      if (!res.ok || data.error) {{
+        statusEl.textContent = data.error || `Search failed (${{res.status}})`;
+        return;
+      }}
+      statusEl.textContent = sources.length ? `${{sources.length}} results` : 'No results';
+      resultsEl.innerHTML = sources.map(s => {{
+        const url = s.url || s.link || '';
+        const title = s.title || url || 'Untitled';
+        const snippet = s.snippet || s.content || '';
+        return `<a class="result" href="${{esc(url)}}" target="_blank" rel="noopener noreferrer">
+          <div class="result-title">${{esc(title)}}</div>
+          <div class="result-url">${{esc(url)}}</div>
+          <div class="result-snippet">${{esc(snippet)}}</div>
+        </a>`;
+      }}).join('');
+    }}
+    document.getElementById('search-form').addEventListener('submit', ev => {{
+      ev.preventDefault();
+      const q = input.value.trim();
+      const url = new URL(window.location.href);
+      url.searchParams.set('q', q);
+      history.replaceState(null, '', url);
+      runSearch(q);
+    }});
+    runSearch(initialQuery);
+  </script>
+</body>
+</html>"""
+        return HTMLResponse(page)
+
     @router.get("/api/search/config")
     async def get_search_settings() -> Dict[str, Any]:
         return get_search_config()

routes/session_routes.py

@@ -3,8 +3,10 @@ import re
 import html
 import json
 import uuid
+import time
+from pathlib import Path
 from datetime import datetime
-from fastapi import APIRouter, Form, HTTPException, Response, Request
+from fastapi import APIRouter, Form, HTTPException, Response, Request, Query
 import logging
 
 from core.session_manager import SessionManager
@@ -60,6 +62,114 @@ def _content_to_text(content) -> str:
     return ""
 
 
+def _context_info_skill_inventory(
+    skills_manager, owner: str | None, limit: int = 80
+) -> list[dict]:
+    """Compact skill metadata for TUI context/status/autocomplete.
+
+    This intentionally exposes only the skill index fields. Full SKILL.md
+    bodies remain behind manage_skills/view so context_info cannot become a
+    prompt/body dump path.
+    """
+    if not skills_manager:
+        return []
+    try:
+        indexed = skills_manager.index_for(owner=owner, active_toolsets=None)
+    except Exception:
+        return []
+    try:
+        loaded = skills_manager.load(owner=owner)
+    except Exception:
+        loaded = []
+    paths_by_name = {
+        str(skill.get("name") or ""): str(skill.get("path") or "").strip()
+        for skill in loaded
+        if isinstance(skill, dict)
+    }
+    out: list[dict] = []
+    seen: set[str] = set()
+    for row in indexed:
+        if not isinstance(row, dict):
+            continue
+        name = str(row.get("name") or "").strip()
+        if not name or name in seen:
+            continue
+        item = {"name": name}
+        description = str(row.get("description") or "").strip()
+        if description:
+            item["description"] = description
+        path = paths_by_name.get(name, "")
+        if path:
+            item["source"] = f"file: {path}"
+        out.append(item)
+        seen.add(name)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_tool_inventory(limit: int = 80) -> list[dict]:
+    """Compact built-in tool metadata for TUI context/status/autocomplete."""
+    try:
+        from src.tool_index import BUILTIN_TOOL_DESCRIPTIONS
+    except Exception:
+        return []
+    out: list[dict] = []
+    for name, description in BUILTIN_TOOL_DESCRIPTIONS.items():
+        clean_name = str(name or "").strip()
+        if not clean_name:
+            continue
+        item = {"name": clean_name, "source": "backend"}
+        clean_description = re.sub(r"\s+", " ", str(description or "")).strip()
+        if clean_description:
+            item["description"] = clean_description[:280]
+        out.append(item)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def _context_info_agents_md_inventory(workspace: str | None, limit: int = 8) -> list[dict]:
+    """Compact AGENTS.md path metadata for the active workspace.
+
+    Bodies intentionally stay on disk. The TUI can read a selected file only
+    when the user asks for `/agent <path> --show`.
+    """
+    try:
+        from src.tool_execution import vet_workspace
+        root = vet_workspace(workspace or "")
+    except Exception:
+        root = None
+    if not root:
+        return []
+
+    start = Path(root).resolve()
+    candidates = []
+    current = start
+    while True:
+        candidate = current / "AGENTS.md"
+        if candidate.is_file():
+            candidates.append(candidate)
+        if current.parent == current:
+            break
+        current = current.parent
+        if len(candidates) >= limit:
+            break
+
+    # Codex-style precedence reads parent instructions before child overrides.
+    out: list[dict] = []
+    seen: set[str] = set()
+    for candidate in reversed(candidates):
+        path = str(candidate)
+        if path in seen:
+            continue
+        out.append({"path": path, "source": "workspace"})
+        seen.add(path)
+        if len(out) >= limit:
+            break
+    return out
+
+
 def _message_role(message) -> str:
     if isinstance(message, ChatMessage):
         return message.role or ""
@@ -157,20 +267,36 @@ def _reject_raw_endpoint_url_for_non_admin(
         raise HTTPException(403, "Choose a registered model endpoint")
 
 
-def _persist_session_headers(session_id: str, headers: dict | None) -> None:
+def _persist_session_headers(session_id: str, headers: dict | None) -> bool:
     """Persist endpoint auth headers for DB-backed session metadata."""
-    db = SessionLocal()
-    try:
-        db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
-        if db_session:
-            db_session.headers = headers or {}
-            db_session.updated_at = utcnow_naive()
-            db.commit()
-    except Exception:
-        db.rollback()
-        raise
-    finally:
-        db.close()
+    delays = (0.05, 0.15, 0.35)
+    last_exc: Exception | None = None
+    for attempt in range(len(delays) + 1):
+        db = SessionLocal()
+        try:
+            db_session = db.query(DbSession).filter(DbSession.id == session_id).first()
+            if db_session:
+                db_session.headers = headers or {}
+                db_session.updated_at = utcnow_naive()
+                db.commit()
+            return True
+        except Exception as exc:
+            db.rollback()
+            last_exc = exc
+            if attempt >= len(delays):
+                break
+            if "database is locked" not in str(exc).lower():
+                break
+            time.sleep(delays[attempt])
+        finally:
+            db.close()
+
+    logger.warning(
+        "Failed to persist headers for session %s; continuing with in-memory headers: %s",
+        session_id,
+        last_exc,
+    )
+    return False
 
 
 _HIDDEN_SYSTEM_SESSION_NAMES = {
@@ -184,6 +310,16 @@ _HIDDEN_SYSTEM_SESSION_NAMES = {
 }
 
 
+def _is_hidden_session_name(name: str | None) -> bool:
+    """Return whether a session should be omitted from the sidebar list."""
+    clean = (name or "").strip()
+    return (
+        clean in ("Nobody", "Incognito")
+        or clean in _HIDDEN_SYSTEM_SESSION_NAMES
+        or clean.startswith("SFT trace batch ")
+    )
+
+
 def _pick_endpoint_for_sort(owner=None):
     """Pick model endpoint for auto-sort LLM call — uses utility endpoint setting, falls back to default."""
     from src.endpoint_resolver import resolve_endpoint
@@ -210,6 +346,7 @@ def setup_session_routes(
     config: dict,
     webhook_manager=None,
     upload_handler=None,
+    skills_manager=None,
 ):
     """Setup session routes with the provided manager and config"""
 
@@ -258,35 +395,22 @@ def setup_session_routes(
         except Exception:
             pass
         user_sessions = session_manager.get_sessions_for_user(user)
-        # Fetch folder info from DB for each session
+        # The sidebar must be backed by persisted DB rows. SessionManager only
+        # hydrates a bounded recent cache at startup, so older-but-valid
+        # conversations can disappear after refresh if this endpoint trusts
+        # memory as the source of truth.
         db = SessionLocal()
         try:
-            folder_map = {}
-            token_map = {}
-            important_map = {}
-            created_map = {}
-            updated_map = {}
-            last_msg_map = {}
-            mode_map = {}
-            msg_count_map = {}
-            q = db.query(DbSession.id, DbSession.folder, DbSession.total_input_tokens, DbSession.total_output_tokens, DbSession.is_important, DbSession.created_at, DbSession.updated_at, DbSession.last_message_at, DbSession.mode, DbSession.message_count).filter(DbSession.archived == False)
+            q = (
+                db.query(DbSession)
+                .filter(DbSession.archived == False)
+                .order_by(DbSession.is_important.desc(), DbSession.updated_at.desc())
+            )
             q = owner_filter(q, DbSession, user)
-            rows = q.all()
-            for row in rows:
-                folder_map[row.id] = row.folder
-                token_map[row.id] = (row.total_input_tokens or 0) + (row.total_output_tokens or 0)
-                important_map[row.id] = row.is_important or False
-                created_map[row.id] = row.created_at.isoformat() if row.created_at else None
-                updated_map[row.id] = row.updated_at.isoformat() if row.updated_at else None
-                # Fall back to updated_at then created_at so sessions that
-                # predate the column (or have no messages) still sort sanely.
-                last_msg_map[row.id] = (
-                    row.last_message_at.isoformat() if row.last_message_at
-                    else (row.updated_at.isoformat() if row.updated_at
-                          else (row.created_at.isoformat() if row.created_at else None))
-                )
-                mode_map[row.id] = row.mode
-                msg_count_map[row.id] = row.message_count or 0
+            rows = [
+                row for row in q.all()
+                if not _is_hidden_session_name(row.name)
+            ]
             # Sessions with active documents that have content
             from sqlalchemy import func
             doc_session_ids = set(
@@ -305,26 +429,58 @@ def setup_session_routes(
                     GalleryImage, user)
                 .distinct().all()
             )
+
+            # Resolve saved routes without waiting for the frontend model catalog.
+            from core.database import ModelEndpoint
+            from src.endpoint_resolver import build_chat_url, normalize_base
+            endpoint_routes = {}
+            endpoint_query = owner_filter(db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True), ModelEndpoint, user)
+            for endpoint in endpoint_query.all():
+                route_url = build_chat_url(normalize_base(endpoint.base_url or '')).rstrip('/')
+                endpoint_routes.setdefault(route_url, []).append(endpoint)
+            sessions = []
+            for s in rows:
+                if (
+                    (s.message_count or 0) <= 0
+                    and s.id not in doc_session_ids
+                    and s.id not in img_session_ids
+                    and s.id not in user_sessions
+                ):
+                    continue
+                # Fall back to updated_at then created_at so sessions that
+                # predate the column (or have no messages) still sort sanely.
+                last_message_at = (
+                    s.last_message_at.isoformat() if s.last_message_at
+                    else (s.updated_at.isoformat() if s.updated_at
+                          else (s.created_at.isoformat() if s.created_at else None))
+                )
+                matches = endpoint_routes.get((s.endpoint_url or '').rstrip('/'), [])
+                selected_endpoint = matches[0] if len(matches) == 1 else None
+                sessions.append({
+                    "id": s.id,
+                    "name": s.name,
+                    "model": _public_model(s.name, s.model),
+                    "endpoint_url": s.endpoint_url,
+                    "endpoint_id": selected_endpoint.id if selected_endpoint else None,
+                    "endpoint_name": selected_endpoint.name if selected_endpoint else None,
+                    "rag": s.rag,
+                    "archived": s.archived,
+                    "folder": s.folder,
+                    "cwd": s.cwd,
+                    "total_tokens": (s.total_input_tokens or 0) + (s.total_output_tokens or 0),
+                    "total_cost_usd": s.total_cost_usd or 0.0,
+                    "is_important": s.is_important or False,
+                    "created_at": s.created_at.isoformat() if s.created_at else None,
+                    "updated_at": s.updated_at.isoformat() if s.updated_at else None,
+                    "last_message_at": last_message_at,
+                    "has_documents": s.id in doc_session_ids,
+                    "has_images": s.id in img_session_ids,
+                    "mode": s.mode,
+                    "message_count": s.message_count or 0,
+                })
         finally:
             db.close()
 
-        sessions = [{"id": s.id, "name": s.name, "model": _public_model(s.name, s.model),
-                     "endpoint_url": s.endpoint_url, "rag": s.rag,
-                     "archived": s.archived, "folder": folder_map.get(s.id),
-                     "total_tokens": token_map.get(s.id, 0),
-                     "is_important": important_map.get(s.id, False),
-                     "created_at": created_map.get(s.id),
-                     "updated_at": updated_map.get(s.id),
-                     "last_message_at": last_msg_map.get(s.id),
-                     "has_documents": s.id in doc_session_ids,
-                     "has_images": s.id in img_session_ids,
-                     "mode": mode_map.get(s.id),
-                     "message_count": msg_count_map.get(s.id, 0)}
-                    for s in user_sessions.values()
-                    if not s.archived
-                    and (s.name or "").strip() not in ("Nobody", "Incognito")
-                    and (s.name or "").strip() not in _HIDDEN_SYSTEM_SESSION_NAMES]
-
         return sessions
     
     @router.post("/session", response_model=SessionResponse)
@@ -337,6 +493,7 @@ def setup_session_routes(
         skip_validation: str = Form(None),
         api_key: str = Form(""),
         endpoint_id: str = Form(""),
+        cwd: str = Form(None),
     ):
         skip_val = str(skip_validation).lower() == "true"
         user = effective_user(request)
@@ -432,6 +589,7 @@ def setup_session_routes(
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
             owner=user,
+            cwd=cwd or None,
         )
         # Set auth headers for custom API-key endpoints
         resolved_key = request_api_key
@@ -456,7 +614,8 @@ def setup_session_routes(
             name=session.name,
             model=model_to_use,
             rag=str(rag).lower() == "true" if rag else False,
-            archived=False
+            archived=False,
+            cwd=session.cwd,
         )    
     @router.patch("/session/{sid}")
     def rename_session(
@@ -464,6 +623,7 @@ def setup_session_routes(
         name: str = Form(None), folder: str = Form(None),
         model: str = Form(None), endpoint_url: str = Form(None),
         endpoint_id: str = Form(None),
+        cwd: str = Form(None),
     ):
         _verify_session_owner(request, sid)
         try:
@@ -486,6 +646,19 @@ def setup_session_routes(
                     result["folder"] = folder if folder else None
             finally:
                 db.close()
+        if cwd is not None:
+            clean_cwd = cwd.strip() or None
+            db = SessionLocal()
+            try:
+                db_session = db.query(DbSession).filter(DbSession.id == sid).first()
+                if db_session:
+                    db_session.cwd = clean_cwd
+                    db_session.updated_at = utcnow_naive()
+                    db.commit()
+                    session.cwd = clean_cwd
+                    result["cwd"] = clean_cwd
+            finally:
+                db.close()
         # Switch model/endpoint mid-session
         if model is not None and endpoint_url is not None:
             user = effective_user(request)
@@ -597,6 +770,8 @@ def setup_session_routes(
                     db.close()
 
                 if session_manager.delete_session(sid):
+                    from routes.chat_helpers import remove_session_sft_trace_rows
+                    remove_session_sft_trace_rows(effective_user(request), sid)
                     deleted_count += 1
             except Exception:
                 pass
@@ -621,6 +796,8 @@ def setup_session_routes(
 
             # Delete the session and all its messages
             if session_manager.delete_session(sid):
+                from routes.chat_helpers import remove_session_sft_trace_rows
+                remove_session_sft_trace_rows(effective_user(request), sid)
                 return {"status": "deleted"}
             else:
                 raise HTTPException(404, "Session not found")
@@ -1034,11 +1211,19 @@ def setup_session_routes(
         if not session_manager.replace_messages(session_id, new_history):
             raise HTTPException(500, "Failed to save compacted history")
 
+        # Rough token estimate of the compacted history so clients can
+        # refresh their context-pressure display without waiting for the
+        # next turn's metrics event.
+        context_tokens_estimate = sum(
+            len(_message_text(m) or "") // 4 + 8 for m in new_history
+        )
+
         return {
             "ok": True,
             "summarized": len(older),
             "kept": len(recent),
             "message_count": len(new_history),
+            "context_tokens_estimate": context_tokens_estimate,
         }
 
     @router.post("/sessions/auto-sort")
@@ -1328,19 +1513,74 @@ def setup_session_routes(
         }
 
     @router.get("/session/{session_id}/context_info")
-    async def get_context_info(request: Request, session_id: str):
+    async def get_context_info(
+        request: Request,
+        session_id: str,
+        cwd: str | None = Query(default=None),
+    ):
         """Get the real context length for a session's model from the endpoint."""
         _verify_session_owner(request, session_id)
+        owner = effective_user(request)
         session = session_manager.get_session(session_id)
         if not session:
             raise HTTPException(404, "Session not found")
+        skills = _context_info_skill_inventory(skills_manager, owner=owner)
+        tools = _context_info_tool_inventory()
+        agents_md = _context_info_agents_md_inventory(cwd)
+        # Workspace visibility: lets the TUI answer "can the backend actually
+        # see this directory?" (mounted vs bridge-only) without probing.
+        from src.workspace_paths import backend_workspace_path, workspace_mount_pairs
+
+        _raw_cwd = str(cwd or getattr(session, "cwd", "") or "").strip()
+        _backend_cwd = backend_workspace_path(_raw_cwd)[:400] if _raw_cwd else ""
+        # Server-side tool policy: non-admin owners silently lose the computer
+        # tools (src/tool_security); surface that so the TUI can show it.
+        try:
+            from src.tool_security import blocked_tools_for_owner
+
+            _blocked = blocked_tools_for_owner(owner)
+        except Exception:
+            _blocked = set()
+        _computer = {"bash", "python", "read_file", "write_file", "host_shell"}
+        _policy = {
+            "computer_tools": "restricted" if _computer & _blocked else "full",
+            "reason": "non-admin owner" if _blocked else "single-user or admin",
+        }
+        _workspace = {
+            "backend_path": _backend_cwd,
+            "exists_in_backend": bool(_backend_cwd) and Path(_backend_cwd).is_dir(),
+            "mount_configured": bool(workspace_mount_pairs()),
+            "via_mount": bool(_raw_cwd) and backend_workspace_path(_raw_cwd) != _raw_cwd,
+        }
         if not session.endpoint_url or not session.model:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         try:
             from src.model_context import get_context_length
             ctx = get_context_length(session.endpoint_url, session.model)
-            return {"context_length": ctx, "model": session.model}
+            return {
+                "context_length": ctx,
+                "model": session.model,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
         except Exception:
-            return {"context_length": None}
+            return {
+                "context_length": None,
+                "skills": skills,
+                "tools": tools,
+                "agents_md": agents_md,
+                "workspace": _workspace,
+                "tool_policy": _policy,
+            }
 
     return router

routes/shell_routes.py

@@ -11,6 +11,7 @@ import shutil
 import subprocess
 import uuid
 import tempfile
+import time
 from collections import namedtuple
 from pathlib import Path
 from typing import Dict, Any
@@ -22,6 +23,7 @@ from src.host_docker_access import (
     running_in_container as _running_in_container,
 )
 from src.optional_deps import prepare_optional_dependency_import
+from src.auth_helpers import _auth_disabled
 
 # POSIX-only: `pty`/`fcntl` transitively import `termios`, which does NOT exist
 # on Windows, so importing them unconditionally crashed app startup there
@@ -53,6 +55,11 @@ from core.platform_compat import (
 def _require_admin(request: Request):
     """Reject non-admin callers. Shell exec is admin-only — never expose to
     regular users; that's RCE-after-signup."""
+    # In the explicitly single-user, auth-disabled deployment the middleware
+    # does not attach a current user. AuthManager is still instantiated by the
+    # app, so checking only for its presence incorrectly returns 403 here.
+    if _auth_disabled():
+        return
     auth_manager = getattr(request.app.state, "auth_manager", None)
     if not auth_manager:
         # No auth at all — only safe in fully-trusted localhost dev mode
@@ -78,6 +85,13 @@ def _reject_cross_site(request: Request):
 _SSH_PORT_RE = re.compile(r"^\d{1,5}$")
 _SAFE_VENV_RE = re.compile(r"^[A-Za-z0-9_./~-]+$")
 
+# Dependency probes can involve several SSH/import checks. Keep the result
+# briefly so the Dependencies tab and a pre-launch check arriving together do
+# not repeat the same expensive work. Installation clears this cache.
+_PACKAGE_STATUS_CACHE: dict[tuple[str, ...], tuple[float, dict[str, Any]]] = {}
+_PACKAGE_STATUS_CACHE_TTL = 3.0
+_PACKAGE_STATUS_CACHE_MAX = 64
+
 
 def _ssh_base_argv(host: str, ssh_port: str | None) -> list[str]:
     """Build an ssh argv prefix for remote probes without local-shell parsing."""
@@ -204,6 +218,19 @@ def _package_installed_from_probe(name: str, probe: dict) -> bool:
             (dists.get("transformers") or modules.get("transformers", {}).get("real_module"))
             and (dists.get("torch") or modules.get("torch", {}).get("real_module"))
         )
+    if name == "office_docs":
+        return bool(
+            dists.get("markitdown")
+            or modules.get("markitdown", {}).get("real_module")
+            or dists.get("python-docx")
+            or modules.get("docx", {}).get("real_module")
+        )
+    if name == "psd_tools":
+        return bool(dists.get("psd-tools") or modules.get("psd_tools", {}).get("real_module"))
+    if name == "pymupdf":
+        return bool(dists.get("PyMuPDF") or modules.get("fitz", {}).get("real_module"))
+    if name == "libreoffice":
+        return bool(binaries.get("soffice") or binaries.get("libreoffice"))
     if name == "hf_transfer":
         return bool(
             dists.get("hf-transfer")
@@ -254,6 +281,28 @@ def _package_status_note(name: str, probe: dict) -> str:
         if _package_installed_from_probe(name, probe):
             return f"SAM object masks: transformers {dists.get('transformers', 'available')} with torch {dists.get('torch', 'available')}"
         return "SAM click/object mask selection needs transformers and torch."
+    if name == "office_docs":
+        if _package_installed_from_probe(name, probe):
+            if dists.get("markitdown"):
+                return f"Office document extraction: markitdown {dists['markitdown']}"
+            if dists.get("python-docx"):
+                return f"Word document extraction: python-docx {dists['python-docx']}"
+            return "Office document extraction available"
+        return "Office attachments need MarkItDown for full fidelity; DOCX has a basic built-in fallback."
+    if name == "psd_tools":
+        if _package_installed_from_probe(name, probe):
+            return f"PSD support: psd-tools {dists.get('psd-tools', 'available')}"
+        return "PSD files need psd-tools for layer/image parsing."
+    if name == "pymupdf":
+        if _package_installed_from_probe(name, probe):
+            return f"PDF forms/rendering: PyMuPDF {dists.get('PyMuPDF', 'available')}"
+        return "Advanced PDF open/render/form features need PyMuPDF."
+    if name == "libreoffice":
+        if binaries.get("soffice"):
+            return f"DOCX signable preview converter: {binaries['soffice']}"
+        if binaries.get("libreoffice"):
+            return f"DOCX signable preview converter: {binaries['libreoffice']}"
+        return "DOCX signing preview needs LibreOffice/soffice to convert Word files to PDF."
     if name == "mlx_lm":
         if _package_installed_from_probe(name, probe):
             return f"MLX LM {dists.get('mlx-lm', 'available')}"
@@ -399,16 +448,21 @@ dist_names={{
     'diffusers':['diffusers','torch'],
     'krea_diffusers':['diffusers','torch'],
     'sam_mask':['transformers','torch'],
-    'hf_transfer':['hf-transfer','hf_transfer'],
-}}
-bin_names={{
+    'office_docs':['markitdown','python-docx'],
+	    'psd_tools':['psd-tools'],
+	    'pymupdf':['PyMuPDF'],
+	    'libreoffice':[],
+	    'hf_transfer':['hf-transfer','hf_transfer'],
+	}}
+	bin_names={{
     'vllm':['vllm'],
     'llama_cpp':['llama-server'],
     'mflux':['mflux-generate-qwen', 'mflux-generate'],
-    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
-    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
-    'tmux':['tmux'],
-}}
+	    'mlx_lama_swift':['odysseus-mlx-inpaint', 'mlx-lama-serve'],
+	    'mlx_ddcolor_swift':['odysseus-mlx-colorize', 'mlx-ddcolor-serve'],
+	    'libreoffice':['soffice', 'libreoffice'],
+	    'tmux':['tmux'],
+	}}
 
 def add_user_install_bins_to_path():
     candidates = []
@@ -457,6 +511,13 @@ def probe(n):
     mods = {{n: mod_status(n)}}
     if n == 'diffusers':
         mods['torch'] = mod_status('torch')
+    if n == 'office_docs':
+        mods['markitdown'] = mod_status('markitdown')
+        mods['docx'] = mod_status('docx')
+    if n == 'psd_tools':
+        mods['psd_tools'] = mod_status('psd_tools')
+    if n == 'pymupdf':
+        mods['fitz'] = mod_status('fitz')
     dists = dist_status(dist_names.get(n, [n]))
     bins = {{b: shutil.which(b) for b in bin_names.get(n, [])}}
     files = {{}}
@@ -1145,6 +1206,7 @@ def setup_shell_routes() -> APIRouter:
         "make":            {"debian": ["make"], "arch": ["make"], "fedora": ["make"], "alpine": ["make"], "suse": ["make"], "macos": []},
         "git":             {"debian": ["git"], "arch": ["git"], "fedora": ["git"], "alpine": ["git"], "suse": ["git"], "macos": ["git"]},
         "tmux":            {"debian": ["tmux"], "arch": ["tmux"], "fedora": ["tmux"], "alpine": ["tmux"], "suse": ["tmux"], "macos": ["tmux"]},
+        "libreoffice":     {"debian": ["libreoffice"], "arch": ["libreoffice-fresh"], "fedora": ["libreoffice"], "alpine": ["libreoffice"], "suse": ["libreoffice"], "macos": ["--cask", "libreoffice"]},
     }
     _BACKEND_EXTRAS = {
         "cuda":   {"debian": ["nvidia-cuda-toolkit"], "arch": ["cuda"], "fedora": ["cuda-toolkit"], "alpine": [], "suse": ["cuda"], "macos": []},
@@ -1206,13 +1268,16 @@ def setup_shell_routes() -> APIRouter:
         import sys
 
         platform_l = (platform or "").strip().lower()
-        model_hint_l = (model_hint or "").strip().lower()
-        has_krea_model = "krea" in model_hint_l
-        has_lama_mlx_model = any(
-            key in model_hint_l
-            for key in ("lama", "mi-gan", "migan", "inpainting-mlx")
+        package_cache_key = (
+            (host or "").strip(),
+            (ssh_port or "").strip(),
+            (venv or "").strip(),
+            (backend or "").strip().lower(),
+            platform_l,
         )
-        has_ddcolor_mlx_model = "ddcolor" in model_hint_l
+        cached_status = _PACKAGE_STATUS_CACHE.get(package_cache_key)
+        if cached_status and time.monotonic() - cached_status[0] < _PACKAGE_STATUS_CACHE_TTL:
+            return cached_status[1]
         _prepend_user_install_bins_to_path()
         importlib.invalidate_caches()
         try:
@@ -1396,6 +1461,13 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Image",
                 "target": "local",
             },
+            {
+                "name": "psd_tools",
+                "pip": "psd-tools",
+                "desc": "Open Photoshop PSD files and inspect flattened/layered image data",
+                "category": "Image",
+                "target": "local",
+            },
             # ── Tools ──
             {
                 "name": "playwright",
@@ -1404,6 +1476,31 @@ def setup_shell_routes() -> APIRouter:
                 "category": "Tools",
                 "target": "local",
             },
+            {
+                "name": "office_docs",
+                "pip": "markitdown[docx,pptx,xlsx,xls]",
+                "desc": "Open Office attachments and documents (.docx, .pptx, .xlsx, .xls) as readable Markdown",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "pymupdf",
+                "pip": "PyMuPDF",
+                "desc": "Advanced PDF opening, rendering, forms, annotations, and signatures",
+                "category": "Tools",
+                "target": "local",
+            },
+            {
+                "name": "libreoffice",
+                "pip": "",
+                "desc": "Convert DOCX attachments to signable PDF previews",
+                "category": "Tools",
+                "target": "local",
+                "kind": "system",
+                "system_prereqs": ["libreoffice"],
+                "install_cmd": "sudo apt install -y libreoffice || brew install --cask libreoffice",
+                "install_hint": "Install LibreOffice/soffice where Odysseus runs to open DOCX attachments as signable PDF previews. Without it, DOCX opens as readable Markdown.",
+            },
         ]
 
         # Most packages should not be installed through external means. Hence, set the default of the
@@ -1411,21 +1508,10 @@ def setup_shell_routes() -> APIRouter:
         for pkg in packages:
             pkg.setdefault("install_cmd", None)
             pkg.setdefault("update_cmd", None)
-        if not has_krea_model:
-            packages = [
-                p for p in packages
-                if p.get("name") not in {"krea_diffusers", "transformers"}
-            ]
-        if not has_lama_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_lama_swift"
-            ]
-        if not has_ddcolor_mlx_model:
-            packages = [
-                p for p in packages
-                if p.get("name") != "mlx_ddcolor_swift"
-            ]
+        # Keep the Image section complete. Dependency visibility is a product
+        # capability decision, not a substring test against a model id. Model
+        # catalogs may declare an explicit runtime package, while the generic
+        # backend preflight handles ordinary models.
         # Remote check: for remote-target packages, probe the selected server's
         # venv over SSH so a remote `pip install` actually reflects here.
         remote_status: dict = {}
@@ -1596,6 +1682,14 @@ def setup_shell_routes() -> APIRouter:
                         if IS_APPLE_SILICON
                         else "Requires a native Apple Silicon Mac with Apple Foundational Models support."
                     )
+                elif pkg["name"] == "libreoffice":
+                    soffice_path = shutil.which("soffice") or shutil.which("libreoffice")
+                    pkg["installed"] = soffice_path is not None
+                    pkg["status_note"] = (
+                        f"DOCX signable preview converter: {soffice_path}"
+                        if soffice_path
+                        else "DOCX signing preview needs LibreOffice/soffice."
+                    )
                 else:
                     pkg["installed"] = shutil.which(pkg["name"]) is not None
             elif pkg["name"] == "llama_cpp" and shutil.which("llama-server"):
@@ -1757,7 +1851,12 @@ def setup_shell_routes() -> APIRouter:
                 )
                 pkg["applicable"] = status.applicable
                 pkg["install_hint"] = status.install_hint
-        return {"packages": packages}
+        result = {"packages": packages}
+        if len(_PACKAGE_STATUS_CACHE) >= _PACKAGE_STATUS_CACHE_MAX:
+            oldest_key = min(_PACKAGE_STATUS_CACHE, key=lambda key: _PACKAGE_STATUS_CACHE[key][0])
+            _PACKAGE_STATUS_CACHE.pop(oldest_key, None)
+        _PACKAGE_STATUS_CACHE[package_cache_key] = (time.monotonic(), result)
+        return result
 
     @router.post("/api/cookbook/packages/install")
     async def install_package(request: Request):
@@ -1802,6 +1901,7 @@ def setup_shell_routes() -> APIRouter:
             *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
         )
         stdout, stderr = await proc.communicate()
+        _PACKAGE_STATUS_CACHE.clear()
         if proc.returncode == 0:
             return {"ok": True, "output": stdout.decode()[-200:]}
         return {"ok": False, "error": stderr.decode()[-300:]}
@@ -1824,7 +1924,7 @@ def setup_shell_routes() -> APIRouter:
         ssh_port = body.get("ssh_port")
         # Names users can request — must match canonical names used in the
         # deps catalog's `system_prereqs` field and on the System rows.
-        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make"}
+        ALLOWED = {"cmake", "build-essential", "g++", "gcc", "git", "tmux", "make", "libreoffice"}
         pkgs = [str(p).strip() for p in raw if str(p).strip() in ALLOWED]
         if not pkgs:
             return {"ok": False, "error": "no installable packages requested (allowlist: " + ", ".join(sorted(ALLOWED)) + ")"}
@@ -1854,7 +1954,15 @@ def setup_shell_routes() -> APIRouter:
                 else: out.append(n)
             return out
         def _brew(names):
-            return [n for n in names if n not in ("build-essential", "g++", "gcc", "make")]
+            out = []
+            for n in names:
+                if n in ("build-essential", "g++", "gcc", "make"):
+                    continue
+                if n == "libreoffice":
+                    out += ["--cask", "libreoffice"]
+                else:
+                    out.append(n)
+            return out
         # Build a single shell snippet that detects the package manager and
         # runs the right install. Non-interactive sudo (-n) only — if sudo
         # asks for a password the script reports it instead of hanging.
@@ -1920,6 +2028,7 @@ def setup_shell_routes() -> APIRouter:
             combined = err_txt or tail_out or f"exit code {proc.returncode}"
         else:
             combined = None
+        _PACKAGE_STATUS_CACHE.clear()
         return {
             "ok": ok,
             "exit_code": proc.returncode,

routes/task/task_routes.py

@@ -10,7 +10,7 @@ from typing import Optional, Dict, Any
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
-from core.database import SessionLocal, ScheduledTask, TaskRun
+from core.database import SessionLocal, ScheduledTask, TaskRun, NotificationLog
 from core.constants import internal_api_base
 from src.auth_helpers import get_current_user
 from src.constants import DATA_DIR, EMAIL_URGENCY_CACHE_DIR
@@ -569,6 +569,57 @@ def setup_task_routes(task_scheduler) -> APIRouter:
         notes = task_scheduler.pop_notifications(owner=user)
         return {"notifications": notes}
 
+    @router.get("/notification-logs")
+    async def get_notification_logs(request: Request, limit: int = 200):
+        """Return persisted task notifications without consuming them."""
+        user = _owner(request)
+        if not user:
+            return {"notifications": []}
+        limit = max(1, min(int(limit or 200), 1000))
+        db = SessionLocal()
+        try:
+            rows = (db.query(NotificationLog)
+                    .filter(NotificationLog.owner == user)
+                    .order_by(NotificationLog.timestamp.desc())
+                    .limit(limit)
+                    .all())
+            return {"notifications": [
+                {
+                    "id": row.id,
+                    "task_name": row.task_name,
+                    "task_id": row.task_id,
+                    "status": row.status,
+                    "body": row.body,
+                    "timestamp": row.timestamp.isoformat() + "Z" if row.timestamp else None,
+                }
+                for row in rows
+            ]}
+        finally:
+            db.close()
+
+    @router.post("/notification-logs")
+    async def create_notification_log(request: Request):
+        """Persist an in-app toast so Settings can show notification history."""
+        user = _owner(request)
+        if not user:
+            raise HTTPException(401, "Authentication required")
+        body = await request.json()
+        message = str(body.get("body") or "").strip()[:2000]
+        if not message:
+            raise HTTPException(400, "Notification body required")
+        row = NotificationLog(
+            id=str(uuid.uuid4()), owner=user,
+            task_name=str(body.get("title") or "Odysseus")[:200],
+            status="error" if body.get("status") == "error" else "success",
+            body=message,
+        )
+        db = SessionLocal()
+        try:
+            db.add(row); db.commit()
+            return {"success": True}
+        finally:
+            db.close()
+
     @router.post("/{task_id}/clear-cache")
     async def clear_task_cache(request: Request, task_id: str):
         """Clear derived cache for one built-in task."""

routes/upload_routes.py

@@ -190,7 +190,8 @@ def setup_upload_routes(upload_handler):
             return None
         return session_id
 
-    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None) -> str | None:
+    def _promote_chat_image_to_gallery(meta: dict, owner: str | None, session_id: str | None = None,
+                                       gallery_id: str | None = None) -> str | None:
         """Make chat-uploaded images visible in Gallery without changing chat storage."""
         is_image_file = getattr(upload_handler, "is_image_file", None)
         if not callable(is_image_file):
@@ -205,6 +206,21 @@ def setup_upload_routes(upload_handler):
         db = SessionLocal()
         try:
             file_hash = meta.get("hash")
+            if gallery_id:
+                existing = db.query(GalleryImage).filter(
+                    GalleryImage.id == gallery_id,
+                    GalleryImage.is_active == True,  # noqa: E712
+                ).first()
+                if existing and (not owner or existing.owner == owner):
+                    image_dir = Path(GENERATED_IMAGES_DIR)
+                    image_dir.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(source_path, image_dir / existing.filename)
+                    existing.file_hash = file_hash
+                    existing.file_size = meta.get("size")
+                    existing.width = meta.get("width")
+                    existing.height = meta.get("height")
+                    db.commit()
+                    return existing.id
             if file_hash:
                 q = db.query(GalleryImage).filter(
                     GalleryImage.file_hash == file_hash,
@@ -259,6 +275,7 @@ def setup_upload_routes(upload_handler):
         request: Request,
         files: List[UploadFile] = File(...),
         session_id: Optional[str] = Form(None),
+        gallery_id: Optional[str] = Form(None),
     ):
         """Upload files with enhanced security and organization."""
         if not isinstance(session_id, str):
@@ -289,7 +306,7 @@ def setup_upload_routes(upload_handler):
             try:
                 owner = effective_user(request)
                 meta = upload_handler.save_upload(u, client_ip, owner=owner)
-                gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id)
+                promoted_gallery_id = _promote_chat_image_to_gallery(meta, owner, session_id, gallery_id)
                 item = {
                     "id": meta["id"],
                     "name": meta["name"],
@@ -303,8 +320,8 @@ def setup_upload_routes(upload_handler):
                     "height": meta.get("height"),
                     "is_duplicate": meta.get("is_duplicate", False)
                 }
-                if gallery_id:
-                    item["gallery_id"] = gallery_id
+                if promoted_gallery_id:
+                    item["gallery_id"] = promoted_gallery_id
                 out.append(item)
             except HTTPException:
                 raise

routes/workspace_routes.py

@@ -82,4 +82,26 @@ def setup_workspace_routes():
         resolved = vet_workspace(path)
         return {"ok": resolved is not None, "path": resolved}
 
+    @router.get("/default")
+    def default_workspace(request: Request):
+        """Return the explicitly configured backend workspace, if usable.
+
+        WebUI has no local launch directory: it runs against this backend's
+        filesystem. An explicit default gives it the same zero-setup behavior
+        as TUI while keeping workspace access opt-in and server-vetted.
+        """
+        owner = get_current_user(request)
+        if not owner_is_admin_or_single_user(owner):
+            raise HTTPException(status_code=403, detail="Workspace default is admin-only")
+
+        configured = os.environ.get("ODYSSEUS_WORKSPACE_DEFAULT", "").strip()
+        if not configured:
+            return {"ok": False, "path": None}
+
+        from src.tool_execution import vet_workspace
+        from src.workspace_paths import backend_workspace_path
+
+        resolved = vet_workspace(backend_workspace_path(configured))
+        return {"ok": resolved is not None, "path": resolved}
+
     return router