Merge current dev and refresh PR #6738 from the 7011 integration

services/memory/memory_extractor.py

@@ -15,6 +15,7 @@ import json
 import logging
 import os
 import re
+import time
 from typing import Optional
 
 from src.memory import MemoryStoreUnreadable
@@ -78,11 +79,13 @@ EXTRACT_SYSTEM_PROMPT = (
     "things the assistant said, one-off tasks, opinions on the current topic.\n\n"
     "Rules:\n"
     "- MAX 2 facts per conversation — only the most important\n"
-    "- Only extract facts the USER stated or clearly implied\n"
+    "- Only extract facts the USER explicitly stated about themselves\n"
+    "- Questions, requests to inspect records, and tool/assistant output are not evidence\n"
     "- Each fact must be a single short sentence (under 15 words)\n"
     "- If a fact is similar to something likely already known, skip it\n"
     "- If nothing durable was revealed, return []\n\n"
-    "Return a JSON array of objects with 'text' and 'category' fields.\n"
+    "Return a JSON array of objects with 'text', 'category', and 'evidence' fields. "
+    "Evidence must be an exact quote from a user's own factual statement.\n"
     "Categories: 'identity', 'preference', 'fact', 'contact', 'project', 'goal'\n\n"
     "Return ONLY valid JSON, no markdown fences."
 )
@@ -213,7 +216,10 @@ def _fallback_memory_candidates(messages) -> list[dict]:
         if not text:
             continue
 
-        m = re.search(r"\bmy name is\s+([A-Za-z][A-Za-z0-9 .'\-]{1,50})\b", text, re.I)
+        m = re.search(
+            r"\bmy name is\s+([A-Za-z][A-Za-z0-9 '\-]{0,49}?)"
+            r"(?=\s+(?:and|but|please)\b|[.!?;,\n]|$)", text, re.I,
+        )
         if m:
             name = _clean_memory_value(m.group(1), 50)
             if name:
@@ -259,22 +265,183 @@ def _fallback_memory_candidates(messages) -> list[dict]:
     return candidates[:2]
 
 
+_USER_FACT_ASSERTION = re.compile(
+    r"\b(?:i(?:'m| am| live| work| use| have| prefer| like| love| hate| own| study| grew up)\b"
+    r"|my\s+[a-z][a-z -]{0,40}\s+(?:is|are|was|works|lives)\b"
+    r"|we\s+(?:are|live|work|have|own)\b"
+    r"|our\s+[a-z][a-z -]{0,40}\s+(?:is|are|was)\b)",
+    re.IGNORECASE,
+)
+
+
+def _has_user_fact_evidence(fact: dict, messages) -> bool:
+    """Require an exact user assertion before persisting an LLM-inferred memory."""
+    evidence = fact.get("evidence")
+    if not isinstance(evidence, str):
+        return False
+    evidence = re.sub(r"\s+", " ", evidence).strip()
+    if len(evidence.split()) < 3 or not _USER_FACT_ASSERTION.match(evidence):
+        return False
+    for message in messages:
+        if _message_role(message) != "user":
+            continue
+        utterance = re.sub(r"\s+", " ", _message_text(message)).strip()
+        position = utterance.casefold().find(evidence.casefold())
+        if position < 0:
+            continue
+        # Judge the clause containing the quote, so a later question does not
+        # erase an earlier factual statement in the same user turn.
+        clause_start = max(
+            utterance.rfind(mark, 0, position) for mark in (".", "?", "!", ";")
+        ) + 1
+        clause_end = min(
+            [end for mark in (".", "?", "!", ";")
+             if (end := utterance.find(mark, position)) >= 0] or [len(utterance)]
+        )
+        clause = utterance[clause_start:clause_end].strip()
+        # A question about one's records is not a statement of the answer.
+        if utterance[clause_end:clause_end + 1] == "?" or re.match(
+            r"^(?:who|what|where|when|why|how|do|does|did|can|could|would|"
+            r"show|list|find|search|tell|summarize|open|read)\b",
+            clause, re.IGNORECASE,
+        ):
+            continue
+        return True
+    return False
+
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+# Function words carry no distinguishing content. Dropping them before
+# comparing two facts keeps punctuation and harmless rephrasing from defeating
+# the duplicate check.
+_STOPWORDS = frozenset(
+    "a an the and or but if then of to in on at for with by from as is are was "
+    "were be been being do does did has have had it its this that these those "
+    "there here user users someone somebody they them their he she his her him i "
+    "we you your my our me us who whom whose what when where why how also "
+    "currently now still about into over under again more most very can could "
+    "may might must will would should".split()
+)
+
+# Words that frame a name/username statement; excluded when reading the value
+# so the actual name tokens can be compared.
+_NAME_TEMPLATE_WORDS = frozenset(
+    "name names named called call appears appear seems seem the user users his "
+    "her their it is are was were be been being username usernames login "
+    "account handle".split()
+)
+
+_NAME_KEYWORDS = frozenset({"name", "named", "called"})
+_USERNAME_KEYWORDS = frozenset({"username", "login"})
+
+
+def _content_tokens(text: str) -> set:
+    """Lowercased content tokens with punctuation and stopwords removed."""
+    toks = _TOKEN_RE.findall((text or "").lower())
+    return {t for t in toks if t not in _STOPWORDS and len(t) >= 2}
+
+
+def _name_slot(text: str):
+    """Return (slot, value_tokens) when text states a name/username, else None.
+
+    Tokens are scanned so "username" is never mistaken for "name", and the
+    first keyword decides the slot (a sentence about a server *named* X that
+    also mentions a *username* Y is a 'name' fact, not a username fact)."""
+    words = _TOKEN_RE.findall((text or "").lower())
+    slot = None
+    for w in words:
+        if w in _NAME_KEYWORDS:
+            slot = "name"
+            break
+        if w in _USERNAME_KEYWORDS:
+            slot = "username"
+            break
+    if not slot:
+        return None
+    value = {
+        w for w in words
+        if w not in _NAME_TEMPLATE_WORDS and w not in _STOPWORDS and len(w) >= 2
+    }
+    return (slot, value) if value else None
+
+
+def _name_slot_tokens(text: str):
+    slot = _name_slot(text)
+    return slot[1] if slot else None
+
+
+def _name_vals_close(a: set, b: set) -> bool:
+    """True when two name values state the same name: identical, or one is a
+    strict refinement of the other where the larger value is a clean name
+    (<=4 alphabetic tokens). Guards against merging a username into a fact
+    like 'a server named Odysseus ... with username pewds'."""
+    if a == b:
+        return True
+    if not a or not b:
+        return False
+    bigger = a if len(a) >= len(b) else b
+    clean = len(bigger) <= 4 and all(t.isalpha() for t in bigger)
+    return clean and (a <= b or b <= a)
+
+
 def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) -> bool:
-    """Check if new_text is too similar to any existing memory (Jaccard similarity)."""
-    new_tokens = set(new_text.lower().split())
+    """Check if new_text states the same fact as an existing memory.
+
+    Three tests, cheapest first: symmetric Jaccard over content tokens,
+    containment (one fact is a strict refinement of the other), and a
+    name/username-slot match. The naive whole-string comparison this replaces
+    kept punctuation (so "Alex." != "alex") and missed short-vs-full names
+    ("Felix" vs "FELIX ARVID ULF KJELLBERG" scored 0.43).
+    """
+    new_tokens = _content_tokens(new_text)
     if not new_tokens:
         return False
+    new_slot = _name_slot(new_text)
     for entry in _memory_dicts(existing):
-        old_tokens = set(entry.get("text", "").lower().split())
+        old_text = entry.get("text", "")
+        old_tokens = _content_tokens(old_text)
         if not old_tokens:
             continue
         intersection = new_tokens & old_tokens
+        if not intersection:
+            continue
         union = new_tokens | old_tokens
         if len(intersection) / len(union) >= threshold:
             return True
+        # Containment: one fact is a strict refinement of the other. Guard on
+        # the shorter side so two-token facts ("Likes Python") are not merged
+        # into longer sentences that merely start the same way.
+        if len(intersection) >= 3 and len(intersection) / min(len(new_tokens), len(old_tokens)) >= 0.85:
+            return True
+        # Name slot: both state the same kind of name and the values match
+        # (short vs full name, or the same name behind different phrasing).
+        old_slot = _name_slot(old_text)
+        if new_slot and old_slot and new_slot[0] == old_slot[0]:
+            if _name_vals_close(new_slot[1], old_slot[1]):
+                return True
     return False
 
 
+def _name_enrichment_target(new_text: str, existing: list):
+    """Return an existing entry whose name value is a strict subset of the
+    new fact's, so the fuller name replaces it in place; else None."""
+    ns = _name_slot(new_text)
+    if not ns:
+        return None
+    slot, new_val = ns
+    best = None
+    for entry in _memory_dicts(existing):
+        old_slot = _name_slot(entry.get("text", ""))
+        if not old_slot or old_slot[0] != slot:
+            continue
+        old_val = old_slot[1]
+        if old_val < new_val and _name_vals_close(old_val, new_val) and len(new_val) <= 4 and all(t.isalpha() for t in new_val):
+            if best is None or len(entry.get("text", "")) < len(best.get("text", "")):
+                best = entry
+    return best
+
+
 def _parse_extraction_json(raw: str) -> list:
     """Parse the extraction LLM's reply into a list of facts, tolerating
     reasoning-model noise.
@@ -331,6 +498,7 @@ async def extract_and_store(
 
     try:
         from src.llm_core import llm_call_async
+        from src.utility_effort import effort_for_call
 
         # Get last N messages from session
         messages = session.get_context_messages()
@@ -404,6 +572,9 @@ async def extract_and_store(
                 # enough once thinking has room.
                 max_tokens=4096,
                 headers=headers,
+                reasoning_effort=effort_for_call(
+                    endpoint_url, model, getattr(session, "owner", None), session
+                ),
             )
 
             # Parse JSON, tolerating reasoning-model noise (<think> blocks, a
@@ -416,8 +587,13 @@ async def extract_and_store(
         if not isinstance(facts, list):
             facts = []
 
+        # The extraction model sees assistant/tool text and can mistake a
+        # failed lookup or a one-off request for a durable personal fact.
+        # Only its claims with a quoted user assertion may enter memory.
+        facts = [fact for fact in facts if isinstance(fact, dict)
+                 and _has_user_fact_evidence(fact, stripped_recent)]
         if fallback_facts:
-            facts = list(facts) + fallback_facts
+            facts.extend(fallback_facts)
 
         if not facts:
             logger.info("Auto memory extraction ran: 0 candidates")
@@ -479,6 +655,25 @@ async def extract_and_store(
 
             # Text dedup fallback: exact match + fuzzy similarity
             user_existing = [e for e in existing if e.get("owner") == _owner or e.get("owner") is None] if _owner else existing
+
+            # Name enrichment: if an existing name fact is a strict prefix of
+            # this more complete one, upgrade it in place instead of adding a
+            # near-duplicate ("name is Felix" -> "name is Felix Arvid Ulf
+            # Kjellberg"). The vector store upserts by id, so re-adding is safe.
+            _enrich = _name_enrichment_target(fact_text, user_existing)
+            if _enrich is not None:
+                _enrich["text"] = fact_text
+                _enrich["timestamp"] = int(time.time())
+                if category:
+                    _enrich["category"] = category
+                if memory_vector and memory_vector.healthy:
+                    try:
+                        memory_vector.add(_enrich["id"], fact_text)
+                    except Exception as e:
+                        logger.warning(f"Memory vector update failed for {_enrich['id']}: {e}")
+                added += 1
+                continue
+
             if memory_manager.find_duplicates(fact_text, user_existing):
                 continue
             # Fuzzy text similarity check (catches rephrased duplicates when vector index is unavailable)
@@ -559,6 +754,7 @@ async def update_persona_memory(
 
     try:
         from src.llm_core import llm_call_async
+        from src.utility_effort import effort_for_call
         from src.text_helpers import strip_think
 
         custom = {}
@@ -607,10 +803,15 @@ async def update_persona_memory(
             temperature=0.1,
             max_tokens=1200,
             headers=headers,
+            reasoning_effort=effort_for_call(
+                endpoint_url, model, getattr(session, "owner", None), session
+            ),
         )
 
         updated = strip_think(str(raw or ""), prose=True, prompt_echo=True).strip()
-        updated = re.sub(r"^```(?:text|markdown)?\s*|\s*```$", "", updated, flags=re.I | re.S).strip()
+        # No leading `\s*` before the closing fence: .strip() drops that
+        # whitespace anyway, and scanning it from every offset was quadratic.
+        updated = re.sub(r"^```(?:text|markdown)?\s*|```$", "", updated, flags=re.I | re.S).strip()
         if len(updated) > 6000:
             updated = updated[:6000].rstrip()
         if updated == existing_memory:
@@ -641,6 +842,7 @@ async def audit_memories(
     """
     try:
         from src.llm_core import llm_call_async
+        from src.utility_effort import effort_for_call
 
         existing = memory_manager.load(owner=owner)
         if not existing:
@@ -689,13 +891,15 @@ async def audit_memories(
             # Bound the call so the Tidy whirlpool can't spin indefinitely on a
             # slow/large generation.
             timeout=120,
+            reasoning_effort=effort_for_call(endpoint_url, model, owner),
         )
 
         # Parse the JSON list, tolerating reasoning-model noise: <think> blocks,
         # markdown fences, leading prose, and trailing commas.
         import re as _re
+        from src.text_helpers import strip_closed_think_blocks
         text = (raw or "").strip()
-        text = _re.sub(r'<think(?:ing)?>[\s\S]*?</think(?:ing)?>', '', text, flags=_re.I).strip()
+        text = strip_closed_think_blocks(text).strip()
 
         def _loads_list(s):
             if not s:
@@ -711,7 +915,9 @@ async def audit_memories(
 
         cleaned = _loads_list(text)
         if cleaned is None:
-            _m = _re.search(r'```(?:json)?\s*\n?([\s\S]*?)```', text)
+            # Possessive `\s*+`: handing fence whitespace back to the body on a
+            # missing closing fence only rescanned the same tail (ReDoS).
+            _m = _re.search(r'```(?:json)?\s*+\n?([\s\S]*?)```', text)
             if _m:
                 cleaned = _loads_list(_m.group(1).strip())
         if cleaned is None:

services/memory/skill_extractor.py

@@ -149,6 +149,7 @@ async def maybe_extract_skill(
 
     try:
         from src.llm_core import llm_call_async
+        from src.utility_effort import effort_for_call
 
         # Get recent messages
         history = session.get_context_messages()
@@ -204,6 +205,7 @@ async def maybe_extract_skill(
             ],
             headers=headers,
             timeout=30,
+            reasoning_effort=effort_for_call(endpoint_url, model, owner, session),
         )
         logger.debug(
             "[skill-extract] LLM returned in %.1fs (len=%d, head=%r)",

services/memory/skills.py

@@ -674,6 +674,7 @@ class SkillsManager:
         *,
         active_toolsets: Optional[List[str]] = None,
         platform: Optional[str] = None,
+        local_model: Optional[bool] = None,
     ) -> List[Dict]:
         """Return the `[{name, description, category, status}]` list the
         agent sees in its system prompt.
@@ -710,6 +711,11 @@ class SkillsManager:
             # Platform gating
             if platform and s.get("platforms") and platform not in s["platforms"]:
                 continue
+            # remote-only tag: hide from local/private-endpoint models. Big
+            # on-demand reference lookups overwhelm small local contexts.
+            # local_model=None (API listings, UI) never gates.
+            if local_model and "remote-only" in (s.get("tags") or []):
+                continue
             # requires_toolsets: hide unless every required toolset is active.
             # active_toolsets=None means the caller doesn't know the active
             # set (API listings, chat preface) — don't gate in that case;
@@ -745,6 +751,7 @@ class SkillsManager:
         min_confidence: float = 0.0,
         available_toolsets: Optional[Iterable[str]] = None,
         platform: Optional[str] = None,
+        published_only: bool = False,
     ) -> List[Dict]:
         if skills is None:
             skills = self.load_all()
@@ -775,9 +782,15 @@ class SkillsManager:
                 if not skill.get("platforms") or platform in skill.get("platforms", [])
             ]
         # Prompt injection is fail-closed for user skills. Built-ins are
-        # shipped procedures; every other skill needs a passing audit and a
-        # confidence score at the user's current threshold.
-        if min_confidence > 0:
+        # shipped procedures. With Auto-approve OFF (published_only), the
+        # user's publish is the approval; otherwise a skill needs a passing
+        # audit and a confidence score at the user's current threshold.
+        if published_only:
+            skills = [
+                s for s in skills
+                if s.get("source") == "builtin" or s.get("status") == "published"
+            ]
+        elif min_confidence > 0:
             def _passes(s):
                 if s.get("source") == "builtin":
                     return True