jpmschweitzer/odysseus
1
0
Fork 0
mirror of https://github.com/pewdiepie-archdaemon/odysseus.git synced 2026-09-26 10:02:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Squash Odysseus development history

Browse Source
This commit is contained in:
pewdiepie-archdaemon
2026-09-11 06:04:19 +00:00
parent e5c99a5eee
commit 6ee6502010
2050 changed files with 538359 additions and 57745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/memory/__init__.py
+2 -1
View File
@@ -2,7 +2,7 @@
"""Memory service — persistent memory storage and retrieval."""
from .service import MemoryService, Memory, MemorySearchResult
from .memory import MemoryManager
from .memory import MemoryManager, MemoryStoreUnreadable
from .memory_vector import MemoryVectorStore
__all__ = [
@@ -10,5 +10,6 @@ __all__ = [
"Memory",
"MemorySearchResult",
"MemoryManager",
"MemoryStoreUnreadable",
"MemoryVectorStore",
]
services/memory/builtin_skills.py
+74
View File
@@ -0,0 +1,74 @@
"""Install tracked built-in skills into the shared immutable skill catalog."""
from __future__ import annotations
from pathlib import Path
from typing import Iterable
from .skill_format import Skill
from .skills import SkillsManager
_BUILTIN_ROOT = Path(__file__).resolve().parents[2] / "resources" / "skills"
_SYNC_FIELDS = (
"name",
"description",
"version",
"category",
"tags",
"status",
"confidence",
"source",
"owner",
"when_to_use",
"procedure",
"pitfalls",
"verification",
"platforms",
"requires_toolsets",
"fallback_for_toolsets",
"body_extra",
)
def install_builtin_skills(manager: SkillsManager, owners: Iterable[str]) -> int:
"""Copy missing built-in skills into the ownerless shared catalog.
Built-ins are explicitly marked and remain ownerless because the on-disk
skill path is not owner-qualified. ``SkillsManager.load(owner=...)``
exposes only these immutable built-ins in addition to that owner's files.
Installation is safe before first-user setup because no owner identity is
assigned and unauthenticated requests still cannot access skill routes.
"""
existing = {row.get("name") for row in manager.load_all()}
installed = 0
paths = sorted(_BUILTIN_ROOT.rglob("SKILL.md")) if _BUILTIN_ROOT.is_dir() else []
for path in paths:
try:
skill = Skill.from_markdown(path.read_text(encoding="utf-8"))
except Exception:
continue
# Tracked procedures ship as trusted application behavior. They are
# available immediately and never enter the user's audit queue.
skill.status = "published"
skill.confidence = 1.0
existing_rows = [row for row in manager.load_all() if row.get("name") == skill.name]
if existing_rows:
row = existing_rows[0]
# Built-ins are immutable tracked assets. Synchronize updated
# versions/procedures on startup while leaving usage counters in
# their sidecar untouched. Older startup code could also stamp the
# first admin onto one; normalize that migration at the same time.
if row.get("source") == "builtin":
skill.owner = ""
skill.source = "builtin"
desired = skill.to_dict()
if any(row.get(field) != desired.get(field) for field in _SYNC_FIELDS):
manager._write_skill(skill)
continue
skill.owner = ""
skill.source = "builtin"
manager._write_skill(skill)
existing.add(skill.name)
installed += 1
return installed
services/memory/memory.py
+17 -356
View File
@@ -1,359 +1,20 @@
"""Compatibility import for the canonical memory manager.
import json
import logging
import os
import time
import uuid
import re
from typing import List, Dict, Tuple
from datetime import datetime
Historically this package carried a second copy of ``MemoryManager``. The
application runtime instantiates ``src.memory.MemoryManager``, so keeping a
parallel implementation here risks silent drift between import paths.
"""
logger = logging.getLogger(__name__)
from src.memory import (
MemoryManager,
MemoryStoreUnreadable,
get_text_similarity,
tokenize,
)
def tokenize(text: str) -> List[str]:
"""Simple tokenizer that splits on whitespace and removes punctuation."""
return [word.strip('.,!?";') for word in text.split()]
def get_text_similarity(text1: str, text2: str) -> float:
"""Calculate Jaccard similarity between two texts."""
if not text1 or not text2:
return 0.0
tokens1 = set(tokenize(text1.lower()))
tokens2 = set(tokenize(text2.lower()))
if not tokens1 and not tokens2:
return 1.0
if not tokens1 or not tokens2:
return 0.0
intersection = tokens1.intersection(tokens2)
union = tokens1.union(tokens2)
return len(intersection) / len(union)
class MemoryManager:
def __init__(self, data_dir: str):
self.memory_file = os.path.join(data_dir, "memory.json")
self.ensure_file_exists()
def extract_memory_from_chat(self, chat_history: List[Dict], session_id: str = None) -> List[Dict]:
"""
Extract memory entries from chat history as a fallback when LLM fails.
Args:
chat_history: List of chat messages with 'role' and 'content' keys
session_id: Optional session ID to associate with extracted memories
Returns:
List of memory entries with text, timestamp, and optional session_id
"""
memories = []
for msg in chat_history:
if msg.get("role") == "assistant":
content = str(msg.get("content", ""))
lines = content.split('\n')
for line in lines:
line = line.strip()
# Look for bullet points or numbered lists that might contain memories
if re.match(r'^[-*•]|\d+\.', line):
# Extract the text after the bullet/number
text_match = re.match(r'^[-*•]|\d+\.\s*(.*)', line)
if text_match:
text = text_match.group(1).strip()
if text:
memories.append({
"text": text,
"timestamp": int(datetime.now().timestamp()),
"session_id": session_id
})
# If we see a heading that suggests memories
elif re.search(r'memory|fact|note|remember', line, re.I):
pass
# If we see a clear separator or end
elif re.match(r'^={3,}|-{3,}|_{3,}', line):
pass
return memories
def process_inline_memory_command(self, message: str) -> Tuple[bool, str]:
"""
Check if a message is an inline memory command (e.g. "remember: X").
Args:
message: The user message to check
Returns:
Tuple of (is_command, extracted_text) where is_command is True if
the message matches the memory command pattern
"""
# Pattern for memory commands: "remember: X", "memorize: X", "save: X", etc.
pattern = r'^(?:remember|memorize|save|note|store)[:\-]?\s+(.+)$'
match = re.match(pattern, message.strip(), re.IGNORECASE)
if match:
return True, match.group(1).strip()
else:
return False, ""
def ensure_file_exists(self):
"""Create memory file if it doesn't exist."""
if not os.path.exists(self.memory_file):
with open(self.memory_file, 'w', encoding='utf-8') as f:
json.dump([], f, ensure_ascii=False, indent=2)
def load_all(self) -> List[Dict]:
"""Load all memory entries from JSON file (unfiltered)."""
if not os.path.exists(self.memory_file):
return []
try:
with open(self.memory_file, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
return self._validate_entries(data)
except (json.JSONDecodeError, PermissionError) as e:
logger.error("Error loading memory.json: %s", e)
return self._migrate_from_legacy()
return []
def load(self, owner: str = None) -> List[Dict]:
"""Load memory entries, filtered by owner."""
entries = self.load_all()
if owner is None:
return entries
return [e for e in entries if e.get("owner") == owner]
def claim_ownerless(self, owner: str):
"""Assign all ownerless memory entries to the given owner. Run once to migrate."""
entries = self.load_all()
changed = False
for e in entries:
if not e.get("owner"):
e["owner"] = owner
changed = True
if changed:
self.save(entries)
logger.info("Claimed %d ownerless memories for %s", sum(1 for e in entries if e.get("owner") == owner), owner)
def _validate_entries(self, entries: List[Dict]) -> List[Dict]:
"""Ensure all entries have required fields."""
validated = []
for entry in entries:
if "id" not in entry:
entry["id"] = str(uuid.uuid4())
if "timestamp" not in entry:
entry["timestamp"] = int(time.time())
if "source" not in entry:
entry["source"] = "unknown"
if "category" not in entry:
entry["category"] = "fact"
validated.append(entry)
return validated
def _migrate_from_legacy(self) -> List[Dict]:
"""Migrate from old text format to JSON if needed."""
legacy_path = os.path.join(os.path.dirname(self.memory_file), "memory.txt")
if not os.path.exists(legacy_path):
return []
logger.info("Converting legacy memory.txt to new JSON format")
try:
with open(legacy_path, "r", encoding="utf-8") as f:
lines = [ln.strip() for ln in f.readlines() if ln.strip()]
entries = []
for line in lines:
entries.append({
"id": str(uuid.uuid4()),
"text": line,
"timestamp": int(time.time()),
"source": "user",
"category": "fact"
})
self.save(entries)
return entries
except Exception as e:
logger.error("Failed to convert legacy memory: %s", e)
return []
def save(self, entries: List[Dict]):
"""Save memory entries to JSON file."""
# Validate entries before saving
for entry in entries:
if "id" not in entry:
entry["id"] = str(uuid.uuid4())
if "timestamp" not in entry:
entry["timestamp"] = int(time.time())
if "source" not in entry:
entry["source"] = "user"
if "category" not in entry:
entry["category"] = "fact"
# Use atomic write
tmp_file = self.memory_file + ".tmp"
with open(tmp_file, "w", encoding="utf-8") as f:
json.dump(entries, f, ensure_ascii=False, indent=2)
os.replace(tmp_file, self.memory_file)
def add_entry(self, text: str, source: str = "user", category: str = "fact", owner: str = None) -> Dict:
"""Add a new memory entry."""
if not text.strip():
raise ValueError("Memory text cannot be empty")
entry = {
"id": str(uuid.uuid4()),
"text": text.strip(),
"timestamp": int(time.time()),
"source": source,
"category": category
}
if owner:
entry["owner"] = owner
return entry
def find_duplicates(self, text: str, entries: List[Dict] = None) -> List[Dict]:
"""Find duplicate memory entries based on text content."""
if entries is None:
entries = self.load()
text_lower = text.strip().lower()
return [entry for entry in entries if entry["text"].lower() == text_lower]
def categorize_memory_by_relevance(self, message: str, memories: list):
"""Categorize memories by type and relevance"""
categories = {
"contacts": [],
"preferences": [],
"facts": [],
"tasks": []
}
msg_lower = message.lower()
for mem in memories:
text_lower = mem["text"].lower()
# Contact info
if any(word in text_lower for word in ["phone", "email", "address", "lives", "works"]):
if any(word in msg_lower for word in ["contact", "phone", "address", "email"]):
categories["contacts"].append(mem)
# Personal preferences
elif any(word in text_lower for word in ["likes", "dislikes", "prefers", "favorite"]):
if any(word in msg_lower for word in ["like", "prefer", "favorite", "want"]):
categories["preferences"].append(mem)
# Tasks and todos
elif any(word in text_lower for word in ["todo", "task", "remind", "meeting"]):
if any(word in msg_lower for word in ["todo", "task", "schedule", "remind"]):
categories["tasks"].append(mem)
# General facts - only if very relevant
else:
if get_text_similarity(message, mem["text"]) > 0.4:
categories["facts"].append(mem)
return categories
def get_relevant_memories(self, query: str, memories: list, threshold: float = 0.05, max_items: int = 8):
"""Get memories that are relevant to the query based on text similarity and semantic keyword matching."""
if not memories or not query.strip():
return []
# Define keyword categories for semantic matching
identity_words = ["name", "who", "i", "am", "called", "identity", "myself", "me", "my"]
contact_words = ["phone", "email", "address", "contact", "number", "where", "located", "reach"]
preference_words = ["like", "prefer", "favorite", "want", "love", "hate", "dislike", "enjoy", "interested"]
task_words = ["todo", "task", "remind", "meeting", "appointment", "schedule", "deadline"]
fact_words = ["what", "when", "where", "how", "why", "explain", "describe", "information", "know"]
query_lower = query.lower()
# Determine query type based on keywords
query_type = None
if any(word in query_lower for word in identity_words):
query_type = "identity"
elif any(word in query_lower for word in contact_words):
query_type = "contact"
elif any(word in query_lower for word in preference_words):
query_type = "preference"
elif any(word in query_lower for word in task_words):
query_type = "task"
elif any(word in query_lower for word in fact_words):
query_type = "fact"
relevant = []
identity_memories = []
other_memories = []
# Separate identity memories from others
for memory in memories:
memory_text = memory["text"].lower()
# Check if this is an identity memory (contains name patterns or identity indicators)
is_identity = any([
re.search(r'\b[A-Z][a-z]+ [A-Z][a-z]+\b', memory["text"]),
any(word in memory_text for word in ["name is", "i'm", "i am", "called", "my name", "named", "call me"])
])
if is_identity:
identity_memories.append(memory)
else:
other_memories.append(memory)
# For identity queries, include all identity memories regardless of similarity
if query_type == "identity" and identity_memories:
# Give them high scores to ensure they're included first
for memory in identity_memories:
relevant.append((0.9, memory)) # High score for identity memories in identity queries
# Process other memories with similarity scoring
for memory in other_memories:
memory_text = memory["text"].lower()
memory_tokens = set(tokenize(memory_text))
query_tokens = set(tokenize(query_lower))
# Calculate base Jaccard similarity
if not query_tokens or not memory_tokens:
continue
base_similarity = len(query_tokens & memory_tokens) / len(query_tokens | memory_tokens)
final_score = base_similarity
# Apply boosts based on semantic matching
if query_type == "contact":
# Boost memories with contact information
has_contact_info = any(word in memory_text for word in ["@gmail.com", "@", ".com",
"phone", "number", "address",
"http", "www", "tel:"])
if has_contact_info:
final_score *= 1.4 # 40% boost for contact-related memories
elif query_type == "preference":
# Boost memories with preference indicators
has_preference = any(word in memory_text for word in ["like", "love", "hate", "dislike",
"prefer", "favorite", "enjoy", "interested"])
if has_preference:
final_score *= 1.3 # 30% boost for preference-related memories
elif query_type == "task":
# Boost memories with task indicators
has_task = any(word in memory_text for word in ["todo", "task", "remind", "meeting",
"appointment", "schedule", "deadline", "need to"])
if has_task:
final_score *= 1.3 # 30% boost for task-related memories
# Always consider exact phrase matches as highly relevant
if query.lower() in memory["text"].lower():
final_score = max(final_score, 0.8) # Ensure high relevance for exact matches
# Include memory if it meets threshold after boosts
if final_score >= threshold:
relevant.append((final_score, memory))
# Sort by final score (descending) and return top matches
relevant.sort(key=lambda x: x[0], reverse=True)
return [mem for _, mem in relevant[:max_items]]
__all__ = [
"MemoryManager",
"MemoryStoreUnreadable",
"get_text_similarity",
"tokenize",
]
services/memory/memory_extractor.py
+308 -34
View File
@@ -17,6 +17,8 @@ import os
import re
from typing import Optional
from src.memory import MemoryStoreUnreadable
logger = logging.getLogger(__name__)
@@ -34,7 +36,7 @@ def _fingerprint_entries(entries) -> str:
only on id+text+category. Any add/edit/delete invalidates it."""
items = sorted(
(str(e.get("id", "")), e.get("text", ""), e.get("category", ""))
for e in entries
for e in _memory_dicts(entries)
)
h = hashlib.sha256()
for triple in items:
@@ -42,10 +44,16 @@ def _fingerprint_entries(entries) -> str:
return h.hexdigest()
def _memory_dicts(entries):
for entry in entries or []:
if isinstance(entry, dict):
yield entry
def _load_tidy_state(memory_manager) -> dict:
path = _tidy_state_path(memory_manager)
try:
with open(path, "r") as f:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
return data if isinstance(data, dict) else {}
except (FileNotFoundError, json.JSONDecodeError):
@@ -57,7 +65,7 @@ def _save_tidy_state(memory_manager, owner: Optional[str], fingerprint: str) ->
state = _load_tidy_state(memory_manager)
state[owner or ""] = {"fingerprint": fingerprint}
try:
with open(path, "w") as f:
with open(path, "w", encoding="utf-8") as f:
json.dump(state, f, indent=2)
except OSError as e:
logger.warning(f"Could not persist tidy fingerprint: {e}")
@@ -82,6 +90,29 @@ EXTRACT_SYSTEM_PROMPT = (
# How many recent messages to include for extraction
CONTEXT_WINDOW = 6
PERSONA_MEMORY_SYSTEM_PROMPT = (
"You maintain concise continuity notes for one active chat persona. "
"Update the existing notes using only durable details established in the transcript. "
"Keep details that help the same persona stay consistent in future conversations: "
"relationship context, names, preferences, recurring story details, boundaries, and unresolved threads. "
"Do not store generic chat events, temporary wording, assistant reasoning, or one-off requests. "
"Never invent details. Return only the updated notes as short bullet points, max 12 bullets. "
"If there is nothing worth keeping, return the existing notes unchanged or an empty string."
)
HEALTH_PERSONA_MEMORY_SYSTEM_PROMPT = (
"You maintain a cautious health-record brief for a medical reasoning persona. "
"Update the existing brief using only medically durable information from the transcript. "
"Keep facts that may matter in future health conversations: confirmed diagnoses, chronic conditions, "
"surgeries/procedures, allergies, regular medications/supplements, important test results, clinicians/hospitals, "
"ongoing symptoms or care plans, and the user's preferences for medical explanations. "
"Use uncertainty labels when needed: 'reported', 'possible', 'asked about', 'unclear'. "
"Do not turn guesses into diagnoses. Do not store casual one-off symptoms unless they are recurring, severe, "
"or tied to an ongoing episode. Never invent facts. Return only the updated brief with these headings when useful: "
"Medical profile, Medications/allergies, Episodes/open questions, Preferences. Max 16 concise bullets total. "
"If nothing medically durable changed, return the existing brief unchanged or an empty string."
)
AUDIT_SYSTEM_PROMPT = (
"You are a memory database curator. Be CONSERVATIVE: remove only TRUE "
"duplicates and clearly useless entries. Every distinct fact must survive. "
@@ -104,6 +135,20 @@ AUDIT_SYSTEM_PROMPT = (
)
AUDIT_INTERVAL = 5 # audit every N new memories added
AUTO_PINNED_IDENTITY_LIMIT = 5
def _is_owner_memory(entry, owner):
if owner:
return entry.get("owner") == owner or entry.get("owner") is None
return True
def _is_auto_pinned_identity(entry):
return (
bool(entry.get("pinned"))
and (entry.get("category") or "").lower() in {"identity", "contact"}
)
_extractions_since_audit = 0
@@ -186,11 +231,19 @@ def _fallback_memory_candidates(messages) -> list[dict]:
if place:
add(f"User lives in {place}.", "identity")
m = re.search(r"\bi (?:prefer|like|love|hate|do not like|don't like)\s+([^.!?\n]{4,100})", text, re.I)
m = re.search(r"\bi (prefer|like|love|hate|do not like|don't like)\s+([^.!?\n]{4,100})", text, re.I)
if m:
preference = _clean_memory_value(m.group(1), 100)
preference = _clean_memory_value(m.group(2), 100)
if preference:
add(f"User prefers {preference}.", "preference")
# The same pattern catches likes and dislikes; keep the stored
# sentiment faithful instead of recording every match as a
# preference ("I hate cilantro" must not become "User prefers
# cilantro").
verb = m.group(1).lower()
if verb in ("hate", "do not like", "don't like"):
add(f"User dislikes {preference}.", "preference")
else:
add(f"User prefers {preference}.", "preference")
m = re.search(
r"\bi (?:(?:want|would like|plan|hope) to|wanna) "
@@ -211,7 +264,7 @@ def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) ->
new_tokens = set(new_text.lower().split())
if not new_tokens:
return False
for entry in existing:
for entry in _memory_dicts(existing):
old_tokens = set(entry.get("text", "").lower().split())
if not old_tokens:
continue
@@ -222,6 +275,43 @@ def _is_text_duplicate(new_text: str, existing: list, threshold: float = 0.6) ->
return False
def _parse_extraction_json(raw: str) -> list:
"""Parse the extraction LLM's reply into a list of facts, tolerating
reasoning-model noise.
The model emits <think>…</think> (and sometimes a prose preamble or a
```json fence) AROUND the JSON array; without stripping it, json.loads
bombs and the run silently yields "0 candidates". Pure str -> list (no
LLM/network); returns [] on any parse failure instead of raising.
"""
text = (raw or "").strip()
try:
from src.text_helpers import strip_think as _strip_think
text = _strip_think(text, prose=True, prompt_echo=True).strip()
except Exception:
pass
if text.startswith("```"):
text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
# JSON may still be embedded in surrounding commentary (leading prose or
# trailing remarks like "[...] Done!") — slice from the first '[' to the
# last ']' whenever both exist. Slice unconditionally: a reply that starts
# with '[' can still carry trailing commentary that breaks json.loads.
_start = text.find("[")
_end = text.rfind("]")
if 0 <= _start < _end:
text = text[_start : _end + 1]
try:
facts = json.loads(text)
except json.JSONDecodeError:
logger.debug("Memory extraction returned non-JSON: %r", (raw or "")[:120])
return []
except Exception:
logger.debug("Memory extraction returned non-JSON: %r", (raw or "")[:120])
return []
return facts if isinstance(facts, list) else []
async def extract_and_store(
session,
memory_manager,
@@ -235,6 +325,10 @@ async def extract_and_store(
Designed to run as a background task (asyncio.create_task).
Errors are logged, never raised.
"""
if not endpoint_url or not model:
logger.debug("[memory-extract] No model or URL provided, skipping")
return
try:
from src.llm_core import llm_call_async
@@ -245,11 +339,55 @@ async def extract_and_store(
if len(recent) < 2:
return # Need at least a user message and assistant response
fallback_facts = _fallback_memory_candidates(recent)
# Strip media (images/audio) from messages — background memory extraction
# only needs the text. The VL-generated descriptions are already in the
# text content of the messages. This avoids sending image tokens to
# non-vision models and prevents accidental "vision grounding" triggers.
stripped_recent = []
for msg in recent:
role = msg.get("role")
content = msg.get("content", "")
if isinstance(content, list):
# Filter out multimodal blocks that aren't text
text_only = [b for b in content if isinstance(b, dict) and b.get("type") == "text"]
if not text_only and content:
continue
content = text_only
stripped_recent.append({"role": role, "content": content})
if not stripped_recent:
return
fallback_facts = _fallback_memory_candidates(stripped_recent)
# Flatten the window into a SINGLE user message instead of appending the
# raw alternating role messages. Passed as raw chat messages, the model
# treats the window as a conversation to CONTINUE rather than a transcript
# to ANALYZE, so it reliably extracts nothing — typically returning `[]`
# (and, depending on the input, sometimes an empty or <think>-only
# completion when the window ends on an assistant turn). This was the real
# cause of auto-memory logging "0 candidates" on every run. Reframing it as
# one "analyze this transcript, return the JSON array" user message makes
# the model actually extract. Controlled repro on this model: 0/6 trials
# with the old structure vs 6/6 with this one. The skill extractor flattens
# for the same reason.
def _flatten_msg(m):
c = m.get("content", "")
if isinstance(c, list):
c = " ".join(
b.get("text", "") for b in c
if isinstance(b, dict) and b.get("type") == "text"
)
return f"{m.get('role', '?')}: {c}"
transcript = "\n\n".join(_flatten_msg(m) for m in stripped_recent)
extraction_messages = [
{"role": "system", "content": EXTRACT_SYSTEM_PROMPT},
] + recent
{"role": "user", "content": (
"Conversation to analyze:\n\n" + transcript
+ "\n\nReturn the JSON array of durable facts now (or [] if none)."
)},
]
facts = []
try:
@@ -258,19 +396,20 @@ async def extract_and_store(
model,
extraction_messages,
temperature=0.1,
max_tokens=500,
# A reasoning model spends most of its budget on <think> tokens
# BEFORE emitting the JSON, so the old 500 truncated the response
# before any JSON appeared → every run logged "0 candidates". The
# audit path hit the same wall and raised to 16384; extraction's
# output (a short facts list) is small, so an ample ceiling is
# enough once thinking has room.
max_tokens=4096,
headers=headers,
)
# Parse JSON from response (handle markdown fences if model wraps them)
text = raw.strip()
if text.startswith("```"):
text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
try:
facts = json.loads(text)
except json.JSONDecodeError:
logger.debug("Memory extraction returned non-JSON")
# Parse JSON, tolerating reasoning-model noise (<think> blocks, a
# ```json fence, and leading/trailing commentary). See
# _parse_extraction_json — returns [] rather than raising.
facts = _parse_extraction_json(raw)
except Exception as e:
logger.warning(f"LLM memory extraction failed; using fallback candidates if available: {e}")
@@ -287,8 +426,18 @@ async def extract_and_store(
# Get owner from session
_owner = getattr(session, 'owner', None)
existing = memory_manager.load_all()
# Strict load: this is a read-modify-write. Degrading to [] here would
# save only the newly extracted facts and drop the entire store.
try:
existing = memory_manager.load_all_for_update()
except MemoryStoreUnreadable as e:
logger.error("Skipping auto memory extraction, store unreadable: %s", e)
return
added = 0
auto_pinned_identity_count = sum(
1 for entry in existing
if _is_owner_memory(entry, _owner) and _is_auto_pinned_identity(entry)
)
for fact in facts:
if isinstance(fact, str):
@@ -296,19 +445,37 @@ async def extract_and_store(
category = "fact"
elif isinstance(fact, dict):
fact_text = fact.get("text", "").strip()
category = fact.get("category", "fact")
category = str(fact.get("category", "fact") or "fact")
else:
continue
if not fact_text or len(fact_text) < 5:
continue
# Dedup: check vector similarity first (fast), then exact text match
# Dedup: check vector similarity first (fast), then exact text match.
# A runtime embedding/ChromaDB failure (backend OOM, model evicted,
# remote endpoint down) must not abort the whole batch — fall through
# to the text/fuzzy dedup below instead of losing every validated
# fact extracted this session. (`.healthy` is only set at init, so
# it does not catch failures that develop later.)
if memory_vector and memory_vector.healthy:
existing_id = memory_vector.find_similar(fact_text, threshold=0.72)
try:
existing_id = memory_vector.find_similar(fact_text, threshold=0.72)
except Exception as e:
logger.warning(f"Memory dedup (vector) unavailable, using text fallback: {e}")
existing_id = None
if existing_id:
logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}")
continue
# The vector store is a single shared collection with no
# owner metadata, so find_similar can return ANOTHER
# tenant's memory. Only treat it as a duplicate when the
# match is this user's own (or a legacy unowned) memory —
# otherwise the user's freshly-extracted fact would be
# silently dropped. Mirror the owner predicate used by the
# text dedup below; cross-tenant/stale matches fall through.
_match = next((e for e in existing if e.get("id") == existing_id), None)
if _match is not None and (_match.get("owner") == _owner or _match.get("owner") is None):
logger.debug(f"Memory dedup (vector): '{fact_text[:50]}' matches {existing_id}")
continue
# Text dedup fallback: exact match + fuzzy similarity
user_existing = [e for e in existing if e.get("owner") == _owner or e.get("owner") is None] if _owner else existing
@@ -320,9 +487,15 @@ async def extract_and_store(
continue
entry = memory_manager.add_entry(fact_text, source="auto", category=category, owner=_owner)
# Auto-pin identity facts (name, job, location) — core context
if category == "identity":
# Auto-pin only the first few identity/contact facts. Extra identity
# memories are still saved, but they must be recalled by relevance
# instead of riding along in every prompt forever.
if (
category.lower() in {"identity", "contact"}
and auto_pinned_identity_count < AUTO_PINNED_IDENTITY_LIMIT
):
entry["pinned"] = True
auto_pinned_identity_count += 1
if hasattr(session, "session_id"):
entry["session_id"] = session.session_id
elif hasattr(session, "name"):
@@ -330,9 +503,14 @@ async def extract_and_store(
existing.append(entry)
# Add to vector index
# Add to vector index. The JSON store (saved below) is the source of
# truth and the keyword path can still retrieve this entry, so a vector
# write failure must not drop the fact or abort the remaining batch.
if memory_vector and memory_vector.healthy:
memory_vector.add(entry["id"], fact_text)
try:
memory_vector.add(entry["id"], fact_text)
except Exception as e:
logger.warning(f"Memory vector add failed for {entry['id']}: {e}")
added += 1
@@ -361,6 +539,88 @@ async def extract_and_store(
logger.error(f"Memory extraction failed: {e}")
async def update_persona_memory(
session,
preset_manager,
character_name: str,
endpoint_url: str,
model: str,
headers: Optional[dict] = None,
schema: str = "general",
):
"""Update the active persona's continuity notes from recent conversation.
Persona memory is stored with the persona/template data, not in the global
memory DB, so deleting a saved persona also deletes its notes.
"""
character_name = (character_name or "").strip()
if not character_name or not endpoint_url or not model or preset_manager is None:
return
try:
from src.llm_core import llm_call_async
from src.text_helpers import strip_think
custom = {}
try:
custom = preset_manager.presets.get("custom", {}) if isinstance(preset_manager.presets, dict) else {}
except Exception:
custom = {}
existing_memory = ""
if isinstance(custom, dict) and custom.get("character_name") == character_name:
existing_memory = custom.get("persona_memory", "") or ""
messages = session.get_context_messages()
recent = messages[-CONTEXT_WINDOW:] if len(messages) > CONTEXT_WINDOW else messages
if len(recent) < 2:
return
lines = []
for msg in recent:
role = msg.get("role")
content = msg.get("content", "")
if isinstance(content, list):
content = " ".join(
b.get("text", "") for b in content
if isinstance(b, dict) and b.get("type") == "text"
)
content = str(content or "").strip()
if content:
lines.append(f"{role}: {content}")
if not lines:
return
system_prompt = HEALTH_PERSONA_MEMORY_SYSTEM_PROMPT if schema == "health" else PERSONA_MEMORY_SYSTEM_PROMPT
raw = await llm_call_async(
endpoint_url,
model,
[
{"role": "system", "content": system_prompt},
{"role": "user", "content": (
f"Persona name: {character_name}\n\n"
f"Existing continuity notes:\n{existing_memory or '(none)'}\n\n"
"Recent transcript:\n"
+ "\n\n".join(lines)
+ "\n\nReturn only the updated continuity notes."
)},
],
temperature=0.1,
max_tokens=1200,
headers=headers,
)
updated = strip_think(str(raw or ""), prose=True, prompt_echo=True).strip()
updated = re.sub(r"^```(?:text|markdown)?\s*|\s*```$", "", updated, flags=re.I | re.S).strip()
if len(updated) > 6000:
updated = updated[:6000].rstrip()
if updated == existing_memory:
return
if preset_manager.update_persona_memory(character_name, updated):
logger.info("Updated persona memory for %s", character_name)
except Exception as e:
logger.warning("Persona memory update failed: %s", e)
async def audit_memories(
memory_manager,
memory_vector,
@@ -503,24 +763,38 @@ async def audit_memories(
# Merge audited entries back with other users' entries
if owner:
all_entries = memory_manager.load_all()
# Strict load: the merge below reconstructs the whole file. If this
# degraded to [] we would save only this owner's audited slice and
# destroy every other tenant's memories.
try:
all_entries = memory_manager.load_all_for_update()
except MemoryStoreUnreadable as e:
logger.error("Aborting memory audit save, store unreadable: %s", e)
return {
"before": before_count,
"after": before_count,
"error": "store_unreadable",
}
audited_ids = {e["id"] for e in final_entries}
other_entries = [e for e in all_entries if e.get("owner") != owner and (e.get("owner") is not None)]
# Also keep legacy entries that weren't part of this audit
for e in all_entries:
if e.get("owner") is None and e["id"] not in audited_ids and e["id"] not in {o["id"] for o in other_entries}:
other_entries.append(e)
memory_manager.save(final_entries + other_entries)
saved_entries = final_entries + other_entries
else:
memory_manager.save(final_entries)
saved_entries = final_entries
memory_manager.save(saved_entries)
logger.info(
f"Memory audit complete: {before_count} -> {after_count} entries "
f"({before_count - after_count} removed/merged)"
)
# Rebuild vector index
# Rebuild vector index from the full saved set, not just this owner's
# slice — otherwise the shared collection is wiped of every other
# owner's entries until they happen to run their own audit.
if memory_vector and memory_vector.healthy:
memory_vector.rebuild(final_entries)
memory_vector.rebuild(saved_entries)
# Persist the post-tidy fingerprint so the next call short-circuits
# if nothing has changed in the meantime.
services/memory/memory_vector.py
+3 -173
View File
@@ -1,175 +1,5 @@
"""
memory_vector.py
"""Compatibility import for the canonical memory vector store."""
ChromaDB-backed vector store for memory entries.
Shares the EmbeddingClient with RAG to save memory.
Stores pre-computed embeddings (ChromaDB does not manage embedding).
"""
from src.memory_vector import MemoryVectorStore
import logging
from typing import List, Dict, Optional
logger = logging.getLogger(__name__)
class MemoryVectorStore:
"""Vector index over memory entries for semantic retrieval."""
COLLECTION_NAME = "odysseus_memories"
def __init__(self, data_dir: str, embedding_model=None):
self._model = embedding_model
self._collection = None
self._healthy = False
self._initialize()
def _initialize(self):
try:
from src.chroma_client import get_chroma_client
if self._model is None:
from src.embeddings import get_embedding_client
self._model = get_embedding_client()
if self._model is None:
raise RuntimeError("No embedding backend available")
logger.info(f"MemoryVectorStore using embeddings: {self._model.url}")
client = get_chroma_client()
self._collection = client.get_or_create_collection(
name=self.COLLECTION_NAME,
metadata={"hnsw:space": "cosine"},
)
self._healthy = True
count = self._collection.count()
logger.info(f"MemoryVectorStore ready (entries={count})")
except Exception as e:
logger.error(f"MemoryVectorStore init failed: {e}")
@property
def healthy(self) -> bool:
return self._healthy
def _embed(self, texts: List[str]) -> List[List[float]]:
vecs = self._model.encode(texts, normalize_embeddings=True)
return vecs.tolist()
def count(self) -> int:
"""Return the number of stored vectors."""
if not self._healthy:
return 0
return self._collection.count()
def add(self, memory_id: str, text: str):
"""Add a single memory entry to the vector index."""
if not self._healthy:
return
# Skip if already exists
existing = self._collection.get(ids=[memory_id])
if existing["ids"]:
return
embeddings = self._embed([text])
self._collection.add(
ids=[memory_id],
embeddings=embeddings,
documents=[text],
metadatas=[{"source": "memory"}],
)
def remove(self, memory_id: str):
"""Remove a memory entry. O(1) — no rebuild needed."""
if not self._healthy:
return
try:
self._collection.delete(ids=[memory_id])
except Exception as e:
logger.warning(f"memory remove {memory_id}: {e}")
def search(self, query: str, k: int = 8) -> List[Dict]:
"""Search for the most relevant memory IDs by semantic similarity.
Returns list of {"memory_id": str, "score": float}.
ChromaDB cosine distance = 1 - cosine_similarity.
We convert back: similarity = 1.0 - distance.
"""
if not self._healthy or self._collection.count() == 0:
return []
embeddings = self._embed([query])
actual_k = min(k, self._collection.count())
results = self._collection.query(
query_embeddings=embeddings,
n_results=actual_k,
)
out = []
for idx, mid in enumerate(results["ids"][0]):
distance = results["distances"][0][idx]
out.append({
"memory_id": mid,
"score": round(1.0 - distance, 4),
})
return out
def find_similar(self, text: str, threshold: float = 0.92) -> Optional[str]:
"""Check if a near-duplicate exists. Returns memory_id if found, else None."""
if not self._healthy or self._collection.count() == 0:
return None
embeddings = self._embed([text])
results = self._collection.query(
query_embeddings=embeddings,
n_results=1,
)
if results["ids"][0]:
distance = results["distances"][0][0]
similarity = 1.0 - distance
if similarity >= threshold:
return results["ids"][0][0]
return None
def rebuild(self, memories: List[Dict]):
"""Rebuild the entire index from a list of memory entries.
Each entry must have 'id' and 'text' keys."""
if not self._healthy:
return
from src.chroma_client import get_chroma_client
# Delete and recreate collection for a clean rebuild
client = get_chroma_client()
try:
client.delete_collection(self.COLLECTION_NAME)
except Exception:
pass
self._collection = client.get_or_create_collection(
name=self.COLLECTION_NAME,
metadata={"hnsw:space": "cosine"},
)
texts = []
ids = []
for mem in memories:
text = mem.get("text", "").strip()
mid = mem.get("id", "")
if text and mid:
texts.append(text)
ids.append(mid)
if texts:
# Batch in chunks of 100 to avoid oversized requests
for i in range(0, len(texts), 100):
batch_texts = texts[i:i + 100]
batch_ids = ids[i:i + 100]
embeddings = self._embed(batch_texts)
self._collection.add(
ids=batch_ids,
embeddings=embeddings,
documents=batch_texts,
metadatas=[{"source": "memory"}] * len(batch_ids),
)
logger.info(f"MemoryVectorStore rebuilt with {len(ids)} entries")
__all__ = ["MemoryVectorStore"]
services/memory/service.py
+50 -61
View File
@@ -7,6 +7,8 @@ import os
from .memory import MemoryManager
from .memory_vector import MemoryVectorStore
from src.memory_provider import MemoryRecord, NativeMemoryProvider
from src.constants import DATA_DIR
@dataclass
@@ -37,11 +39,38 @@ class MemoryService:
results = await service.recall("preferences")
"""
def __init__(self, data_dir: str = "data"):
def __init__(self, data_dir: str = DATA_DIR):
self.manager = MemoryManager(data_dir)
self.vector_store = MemoryVectorStore(data_dir) if os.path.exists(
os.path.join(data_dir, "memory_vectors")
) else None
self.provider = NativeMemoryProvider(self.manager, self.vector_store)
def _sync_provider(self) -> None:
self.provider.memory_vector = self.vector_store
@staticmethod
def _to_memory(entry: Dict[str, Any], metadata: Optional[Dict[str, Any]] = None) -> Memory:
return Memory(
id=entry.get("id", ""),
text=entry.get("text", ""),
timestamp=entry.get("timestamp", 0),
session_id=entry.get("session_id"),
metadata=metadata or {},
)
@staticmethod
def _record_to_memory(record: MemoryRecord, metadata: Optional[Dict[str, Any]] = None) -> Memory:
merged_metadata = dict(record.metadata)
if metadata:
merged_metadata.update(metadata)
return Memory(
id=record.id,
text=record.text,
timestamp=record.timestamp,
session_id=record.session_id,
metadata=merged_metadata,
)
async def remember(self, text: str, session_id: Optional[str] = None) -> Memory:
"""
@@ -54,31 +83,9 @@ class MemoryService:
Returns:
Created Memory object
"""
import uuid
import time
memory_id = str(uuid.uuid4())[:8]
timestamp = int(time.time())
entry = {
"id": memory_id,
"text": text,
"timestamp": timestamp,
"session_id": session_id,
}
self.manager.add_memory(entry)
# Also add to vector store if available
if self.vector_store:
self.vector_store.add(text, {"id": memory_id, "session_id": session_id})
return Memory(
id=memory_id,
text=text,
timestamp=timestamp,
session_id=session_id,
)
self._sync_provider()
record = await self.provider.remember(text, session_id=session_id)
return self._record_to_memory(record)
async def recall(self, query: str, top_k: int = 5) -> MemorySearchResult:
"""
@@ -91,47 +98,29 @@ class MemoryService:
Returns:
MemorySearchResult with matching memories
"""
# Try vector search first
if self.vector_store:
results = self.vector_store.search(query, k=top_k)
memories = [
Memory(
id=r.get("id", ""),
text=r.get("text", ""),
timestamp=r.get("timestamp", 0),
session_id=r.get("session_id"),
metadata=r.get("metadata", {}),
)
for r in results
]
return MemorySearchResult(memories=memories, query=query, total=len(memories))
# Fallback to keyword search
results = self.manager.search_memories(query, limit=top_k)
self._sync_provider()
results = await self.provider.recall(query, top_k=top_k)
memories = [
Memory(
id=m.get("id", ""),
text=m.get("text", ""),
timestamp=m.get("timestamp", 0),
session_id=m.get("session_id"),
)
for m in results
self._record_to_memory(hit.memory, metadata={"score": hit.score})
if hit.score is not None
else self._record_to_memory(hit.memory)
for hit in results
]
return MemorySearchResult(memories=memories, query=query, total=len(memories))
def get_all(self, limit: int = 100) -> List[Memory]:
"""Get all memories."""
memories = self.manager.get_memories(limit=limit)
return [
Memory(
id=m.get("id", ""),
text=m.get("text", ""),
timestamp=m.get("timestamp", 0),
session_id=m.get("session_id"),
)
for m in memories
]
records = self.manager.load_all()[:limit]
return [self._to_memory(m) for m in records]
def delete(self, memory_id: str) -> bool:
"""Delete a memory by ID."""
return self.manager.delete_memory(memory_id)
memories = self.manager.load_all()
remaining = [m for m in memories if m.get("id") != memory_id]
if len(remaining) == len(memories):
return False
self.manager.save(remaining)
if self.vector_store and self.vector_store.healthy:
self.vector_store.remove(memory_id)
return True
services/memory/skill_extractor.py
+110 -20
View File
@@ -28,6 +28,10 @@ SKILL_EXTRACT_PROMPT = (
"(personal errands, a specific person/place/date, casual conversation).\n"
"- A pure question/answer or explanation with no transferable method.\n"
"- The agent failed, gave up, or the approach is not worth repeating.\n\n"
"- Routine use of an existing tool, or a generic checklist with no new discovery.\n"
"Prefer a specific successful workaround, an unexpected pitfall, or a verified "
"sequence that would save rediscovery. Preserve exact useful commands and "
"verification steps, but replace private identifiers and credentials with placeholders.\n\n"
"When (and only when) a genuine reusable procedure exists, return a JSON "
"object with:\n"
'- "title": short name (under 10 words)\n'
@@ -48,6 +52,77 @@ MIN_CONFIDENCE = 0.6
CONTEXT_WINDOW = 12
def _skill_dicts(skills):
for skill in skills or []:
if isinstance(skill, dict):
yield skill
def _has_duplicate_title(skills, title: str) -> bool:
wanted = title.lower()
for skill in _skill_dicts(skills):
existing = skill.get("title", "")
if isinstance(existing, str) and existing.lower() == wanted:
return True
return False
def _extract_json_object(text: str) -> Optional[dict]:
"""Best-effort extraction of a JSON object from an LLM response.
The response may be wrapped in code fences or surrounded by prose. Uses
json.JSONDecoder().raw_decode() to locate the boundaries of complete JSON
objects starting at each '{' position. Nested objects are filtered out to
keep only top-level candidates. If multiple non-overlapping valid JSON
objects are found, it is treated as ambiguous and returns None. Otherwise,
returns the single valid candidate dictionary.
"""
if not text:
return None
s = text.strip()
if s.startswith("```"):
s = s.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
decoder = json.JSONDecoder()
candidates = []
start = s.find("{")
while start != -1:
try:
obj, idx = decoder.raw_decode(s[start:])
end_pos = start + idx
if isinstance(obj, dict):
candidates.append((start, end_pos, obj))
except (json.JSONDecodeError, ValueError):
pass
start = s.find("{", start + 1)
# Filter out nested candidates to identify top-level dictionaries
top_level = []
for c in candidates:
is_nested = False
for other in candidates:
if other == c:
continue
if other[0] <= c[0] and c[1] <= other[1]:
is_nested = True
break
if not is_nested:
top_level.append(c)
if not top_level:
return None
if len(top_level) > 1:
logger.debug(
"[skill-extract] Found multiple non-overlapping JSON objects: %s",
[item[2].get("title") for item in top_level]
)
return None
return top_level[0][2]
async def maybe_extract_skill(
session,
skills_manager,
@@ -59,6 +134,10 @@ async def maybe_extract_skill(
owner: Optional[str] = None,
):
"""Extract a skill if the agent run was complex enough."""
if not model:
logger.debug("[skill-extract] No model provided, skipping")
return None
# Quiet by default; flip to DEBUG when chasing extractor issues.
logger.debug(
"[skill-extract] start: rounds=%d tools=%d model=%s owner=%s",
@@ -78,9 +157,23 @@ async def maybe_extract_skill(
logger.debug("[skill-extract] no recent messages, skipping")
return None
# Strip media (images/audio) from messages
stripped_recent = []
for msg in recent:
content = msg.get("content", "")
if isinstance(content, list):
text_only = [b for b in content if isinstance(b, dict) and b.get("type") == "text"]
if not text_only and content:
continue
content = text_only
stripped_recent.append({"role": msg.get("role"), "content": content})
if not stripped_recent:
return None
# Build conversation summary for extraction
conv_lines = []
for msg in recent:
for msg in stripped_recent:
role = msg.get("role", "?")
content = msg.get("content", "")
if isinstance(content, list):
@@ -136,21 +229,14 @@ async def maybe_extract_skill(
except Exception:
pass
# Parse JSON
text = response.strip()
if text.startswith("```"):
text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
# After strip_think, the JSON may still be embedded inside surrounding
# commentary — slice from the first '{' to the matching last '}'.
if text and text[0] != "{":
_start = text.find("{")
_end = text.rfind("}")
if 0 <= _start < _end:
text = text[_start : _end + 1]
data = json.loads(text)
if not data or not isinstance(data, dict):
logger.debug("[skill-extract] parsed JSON not a dict, dropping")
# Parse JSON. The object may be wrapped in code fences or surrounded by
# commentary (and may contain a stray/invalid brace fragment before
# the real object — including one that makes the response itself look
# like it starts with '{'), so use a tolerant extractor that tries the
# whole string first and then each '{' candidate left-to-right.
data = _extract_json_object(response)
if not data:
logger.debug("[skill-extract] no JSON object found in response, dropping")
return None
title = data.get("title", "").strip()
@@ -173,10 +259,13 @@ async def maybe_extract_skill(
# Check for duplicate skills
existing = skills_manager.load(owner=owner)
for sk in existing:
if sk.get("title", "").lower() == title.lower():
logger.debug("[skill-extract] '%s' already exists — dropped as duplicate", title)
return None
if _has_duplicate_title(existing, title):
logger.debug("[skill-extract] '%s' already exists — dropped as duplicate", title)
return None
# Automatic approval happens only after the audit has passed. A new
# extraction begins as a draft so it cannot enter chat context early.
_initial_status = "draft"
entry = skills_manager.add_skill(
title=title,
@@ -188,6 +277,7 @@ async def maybe_extract_skill(
confidence=data.get("confidence", 0.7),
session_id=getattr(session, "session_id", None),
owner=owner,
status=_initial_status,
)
try:
from src.event_bus import fire_event
services/memory/skill_format.py
+43 -4
View File
@@ -50,7 +50,7 @@ import json
import logging
import re
from dataclasses import dataclass, field
from datetime import datetime
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
logger = logging.getLogger(__name__)
@@ -100,6 +100,18 @@ def _parse_scalar(raw: str) -> Any:
if raw.lower() in ("null", "none", "~"):
return None
if (raw[0] == raw[-1]) and raw[0] in ("'", '"'):
if raw[0] == '"':
# _emit_scalar writes double-quoted scalars with json.dumps, so
# decode the escapes instead of only stripping the quotes. Without
# this, `\"` / `\\` / `\uXXXX` stayed verbatim in the value and the
# next save escaped their backslashes again, doubling them on every
# load/save cycle (issue #5210).
try:
return json.loads(raw)
except ValueError:
# Hand-written file using escapes JSON rejects (e.g. a bare
# Windows path). Keep the previous literal reading.
pass
return raw[1:-1]
# Try number
try:
@@ -171,6 +183,26 @@ def parse_frontmatter(text: str) -> tuple[Dict[str, Any], str]:
return fm, body
# Characters that force a quoted scalar. The punctuation would otherwise change
# how the value reads back; the second row is every character str.splitlines()
# treats as a line break, and parse_frontmatter() reads one scalar per line, so
# emitting one of those bare would split the value across lines.
_FM_MUST_QUOTE = (
":", "#", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@",
"\n", "\r", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029",
)
# json.dumps escapes every C0 control character, but with ensure_ascii=False it
# passes NEL / LINE SEPARATOR / PARAGRAPH SEPARATOR through literally, and
# str.splitlines() still breaks on all three. Re-escape exactly those, which
# json.loads decodes again on the way in, so the pair stays symmetric.
_FM_POST_DUMPS_ESCAPES = (
("\x85", "\\u0085"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
)
def _emit_scalar(v: Any) -> str:
if v is None:
return "null"
@@ -181,8 +213,15 @@ def _emit_scalar(v: Any) -> str:
if isinstance(v, list):
return "[" + ", ".join(_emit_scalar(x) for x in v) + "]"
s = str(v)
if any(c in s for c in (":", "#", "\n", "[", "]", "{", "}", ",", "&", "*", "!", "|", ">", "'", '"', "%", "@")):
return json.dumps(s)
if any(c in s for c in _FM_MUST_QUOTE):
# ensure_ascii=False keeps non-ASCII text as itself. SKILL.md is UTF-8 at
# both ends (skills.py reads it, atomic_write_text writes it), so the
# \uXXXX form bought nothing and leaked into the parsed value (#5210).
out = json.dumps(s, ensure_ascii=False)
for ch, esc in _FM_POST_DUMPS_ESCAPES:
if ch in out:
out = out.replace(ch, esc)
return out
return s
@@ -441,4 +480,4 @@ class Skill:
def _now_iso() -> str:
return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
services/memory/skill_importer.py
+487
View File
@@ -0,0 +1,487 @@
"""Import SKILL.md bundles from public GitHub (or skills.sh → GitHub) URLs."""
from __future__ import annotations
import ipaddress
import logging
import os
import time
from dataclasses import dataclass
from typing import Dict, Iterable, List, Optional, Tuple, cast
from urllib.parse import quote, urljoin, urlparse
import httpcore
import httpx
from src.url_safety import _default_resolver, check_outbound_url
logger = logging.getLogger(__name__)
MAX_FILES = 64
MAX_TOTAL_BYTES = 2_000_000
MAX_FILE_BYTES = 400_000
ALLOWED_SUFFIXES = (
".md", ".txt", ".json", ".yaml", ".yml", ".py", ".sh", ".toml",
".js", ".ts", ".css", ".html", ".xml", ".csv",
)
TEXT_NAMES = {"skill.md", "license", "license.md", "readme.md"}
_GITHUB_HOSTS = frozenset({
"github.com", "www.github.com", "api.github.com", "raw.githubusercontent.com",
})
_SKILLS_SH_HOSTS = frozenset({"skills.sh", "www.skills.sh"})
def _github_host(url: str) -> str:
return (urlparse(str(url)).hostname or "").lower()
def _assert_github_url(url: str, *, context: str = "URL") -> None:
host = _github_host(url)
if host not in _GITHUB_HOSTS:
raise SkillImportError(
f"{context} must stay on GitHub (got {host or 'unknown host'})"
)
@dataclass
class ResolvedSource:
owner: str
repo: str
ref: str
path: str # directory or file path inside repo (no leading slash)
class SkillImportError(ValueError):
pass
def _safe_relpath(rel: str) -> str:
rel = (rel or "").replace("\\", "/").strip().lstrip("/")
if not rel or rel.startswith("..") or "/../" in f"/{rel}/":
raise SkillImportError(f"unsafe path: {rel!r}")
parts = [p for p in rel.split("/") if p and p != "."]
if any(p == ".." for p in parts):
raise SkillImportError(f"unsafe path: {rel!r}")
return "/".join(parts)
def _is_text_file(name: str) -> bool:
low = name.lower()
if low in TEXT_NAMES:
return True
return any(low.endswith(s) for s in ALLOWED_SUFFIXES)
# Max redirect hops to follow manually while re-validating each one.
_MAX_FETCH_REDIRECTS = 5
def _validated_ips(raw_ips: List[str]) -> List[ipaddress._BaseAddress]:
"""Parse and de-duplicate one resolver snapshot in resolver order."""
ips: List[ipaddress._BaseAddress] = []
seen = set()
for raw in raw_ips:
if not isinstance(raw, str):
continue
try:
ip = ipaddress.ip_address(raw.split("%", 1)[0])
except ValueError:
continue
if ip in seen:
continue
seen.add(ip)
ips.append(ip)
return ips
def _resolve_and_check_url(url: str) -> List[ipaddress._BaseAddress]:
"""Return the exact address snapshot approved for one fetch hop."""
resolved_ips: List[str] = []
def _recording_resolver(host: str) -> List[str]:
answers = list(_default_resolver(host))
resolved_ips[:] = answers
return answers
ok, reason = check_outbound_url(
url,
block_private=True,
resolver=_recording_resolver,
)
if not ok:
raise SkillImportError(f"outbound URL blocked: {reason}")
pinned_ips = _validated_ips(resolved_ips)
if not pinned_ips:
raise SkillImportError("outbound URL blocked: host did not resolve to a usable address")
return pinned_ips
# Backward compatibility alias for tests importing _check_fetch_url directly
_check_fetch_url = _resolve_and_check_url
class _PinnedBackend(httpcore.NetworkBackend):
"""Connect only to addresses from one validated DNS snapshot."""
def __init__(self, ips: List[ipaddress._BaseAddress]):
self._ips = [str(ip) for ip in ips]
self._real = httpcore.SyncBackend()
def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options=None,
):
deadline = None if timeout is None else time.monotonic() + timeout
last_exc: Optional[Exception] = None
for ip in self._ips:
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
try:
return self._real.connect_tcp(
ip,
port,
remaining,
local_address,
socket_options,
)
except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc:
last_exc = exc
if deadline is not None and time.monotonic() >= deadline:
break
if last_exc is not None:
raise last_exc
raise httpcore.ConnectError("no validated address available")
def connect_unix_socket(self, path, timeout=None, socket_options=None):
return self._real.connect_unix_socket(path, timeout, socket_options)
def sleep(self, seconds: float) -> None:
return self._real.sleep(seconds)
_HTTPCORE_TO_HTTPX_EXC = {
httpcore.ConnectError: httpx.ConnectError,
httpcore.ConnectTimeout: httpx.ConnectTimeout,
httpcore.LocalProtocolError: httpx.LocalProtocolError,
httpcore.NetworkError: httpx.NetworkError,
httpcore.PoolTimeout: httpx.PoolTimeout,
httpcore.ProtocolError: httpx.ProtocolError,
httpcore.ProxyError: httpx.ProxyError,
httpcore.ReadError: httpx.ReadError,
httpcore.ReadTimeout: httpx.ReadTimeout,
httpcore.RemoteProtocolError: httpx.RemoteProtocolError,
httpcore.TimeoutException: httpx.TimeoutException,
httpcore.UnsupportedProtocol: httpx.UnsupportedProtocol,
httpcore.WriteError: httpx.WriteError,
httpcore.WriteTimeout: httpx.WriteTimeout,
}
class _PinnedTransport(httpx.BaseTransport):
"""Pin socket connects while preserving URL authority, Host, and TLS SNI."""
def __init__(self, ips: List[ipaddress._BaseAddress]):
self._pinned_ips = list(ips)
self._pool = httpcore.ConnectionPool(
ssl_context=httpx.create_ssl_context(),
http1=True,
http2=False,
network_backend=_PinnedBackend(ips),
)
def handle_request(self, request: httpx.Request) -> httpx.Response:
core_request = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
core_response = None
try:
core_response = self._pool.handle_request(core_request)
content = b"".join(cast(Iterable[bytes], core_response.stream))
except Exception as exc:
mapped = _HTTPCORE_TO_HTTPX_EXC.get(type(exc))
if mapped is not None:
raise mapped(str(exc)) from exc
raise
finally:
if core_response is not None:
core_response.close()
return httpx.Response(
status_code=core_response.status,
headers=core_response.headers,
content=content,
extensions=core_response.extensions,
)
def close(self) -> None:
self._pool.close()
def _get_checked(
url: str,
*,
headers: Optional[dict] = None,
timeout: float = 30.0,
) -> httpx.Response:
"""GET that follows redirects manually, re-running the SSRF guard per hop.
``httpx``'s ``follow_redirects=True`` validates only the initial URL, so a
``3xx`` to an internal address (``169.254.169.254``, ``127.0.0.1``, …) would
still be connected to before any post-hoc host check. Following redirects by
hand lets us re-validate every hop, closing that blind-SSRF gap.
"""
current = url
for _ in range(_MAX_FETCH_REDIRECTS + 1):
pinned_ips = _resolve_and_check_url(current)
with httpx.Client(
transport=_PinnedTransport(pinned_ips),
follow_redirects=False,
timeout=timeout,
) as client:
r = client.get(current, headers=headers)
if r.status_code in (301, 302, 303, 307, 308):
location = r.headers.get("location")
if not location:
return r
current = urljoin(str(r.url), location)
continue
return r
raise SkillImportError("too many redirects while fetching skill bundle")
def parse_skill_source(url: str) -> ResolvedSource:
"""Normalize skills.sh / GitHub web URLs into owner/repo/ref/path."""
url = (url or "").strip()
if not url:
raise SkillImportError("URL is required")
# ``urlparse`` only reports an unambiguous scheme when the URL carries the
# ``scheme://`` form. Opaque schemes (``mailto:``, ``javascript:``) and a
# schemeless ``host:port`` both parse a "scheme" that is not one, so they
# fall through to the host check below and are rejected on the host instead.
scheme = urlparse(url).scheme.lower()
if scheme not in ("http", "https"):
if scheme and url.lower().startswith(f"{scheme}://"):
raise SkillImportError(f"unsupported URL scheme: {scheme}")
# Schemeless "github.com/owner/repo" — accept only a supported host.
rough_host = (urlparse("//" + url).hostname or "").lower()
if rough_host not in _GITHUB_HOSTS and rough_host not in _SKILLS_SH_HOSTS:
raise SkillImportError("Only GitHub or skills.sh URLs are supported")
url = "https://" + url
parsed = urlparse(url)
hostname = (parsed.hostname or "").lower()
if hostname not in _GITHUB_HOSTS and hostname not in _SKILLS_SH_HOSTS:
raise SkillImportError("Only GitHub or skills.sh URLs are supported")
# A skills.sh link is only usable if it redirects to an exact supported
# GitHub host. Scraping the page body for a github.com link cannot work:
# skill pages only ever link the repository root, never the skill's
# subdirectory, so the scrape resolves every skill in a repo to the same
# (wrong) bundle. Fail with an actionable message instead.
if hostname in _SKILLS_SH_HOSTS:
r = _get_checked(url, timeout=20.0)
if r.status_code >= 400:
raise _github_response_error(r)
final = str(r.url)
if _github_host(final) not in _GITHUB_HOSTS:
raise SkillImportError(
"skills.sh did not redirect to GitHub — open the skill's "
"repository on GitHub, navigate to the exact skill folder or "
"SKILL.md file, and paste that URL; the repository-root link "
"alone is not sufficient"
)
url = final
# Update parsed and hostname to reflect the new GitHub URL
parsed = urlparse(url)
hostname = (parsed.hostname or "").lower()
_assert_github_url(url)
if hostname == "raw.githubusercontent.com":
# /owner/repo/ref/path/to/file
bits = [p for p in parsed.path.split("/") if p]
if len(bits) < 4:
raise SkillImportError("Invalid raw GitHub URL")
owner, repo, ref = bits[0], bits[1], bits[2]
path = "/".join(bits[3:])
return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path)
bits = [p for p in parsed.path.split("/") if p]
if len(bits) < 2:
raise SkillImportError("Invalid GitHub URL")
owner, repo = bits[0], bits[1]
ref = "main"
path = ""
if len(bits) >= 4 and bits[2] in ("tree", "blob"):
ref = bits[3]
path = "/".join(bits[4:])
elif len(bits) == 2:
path = ""
else:
raise SkillImportError("GitHub URL must include /tree/<branch>/... or /blob/<branch>/...")
return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path)
def _raw_url(src: ResolvedSource, rel_path: str) -> str:
rel = _safe_relpath(rel_path)
return f"https://raw.githubusercontent.com/{src.owner}/{src.repo}/{quote(src.ref, safe='')}/{quote(rel, safe='/')}"
def _api_contents_url(src: ResolvedSource, rel_path: str = "") -> str:
rel = _safe_relpath(rel_path) if rel_path else ""
base = f"https://api.github.com/repos/{src.owner}/{src.repo}/contents"
if rel:
base += f"/{quote(rel, safe='/')}"
return f"{base}?ref={quote(src.ref, safe='')}"
def _github_response_error(response: httpx.Response) -> SkillImportError:
"""Turn a failed GitHub HTTP response into a user-visible import error."""
status = response.status_code
detail = ""
try:
body = response.json()
if isinstance(body, dict):
detail = str(body.get("message") or "").strip()
except Exception:
detail = (response.text or "").strip()[:200]
low = detail.lower()
if status == 403 and "rate limit" in low:
return SkillImportError(
"GitHub API rate limit exceeded — try again in a bit"
+ (f" ({detail})" if detail else "")
)
if status == 404:
return SkillImportError("path not found on GitHub")
if detail:
return SkillImportError(f"GitHub request failed ({status}): {detail}")
return SkillImportError(f"GitHub request failed ({status})")
def _fetch_bytes(url: str) -> bytes:
r = _get_checked(url, headers={"Accept": "application/vnd.github+json"}, timeout=30.0)
if r.status_code >= 400:
raise _github_response_error(r)
_assert_github_url(str(r.url), context="redirect target")
if len(r.content) > MAX_FILE_BYTES:
raise SkillImportError(f"file too large: {url}")
return r.content
def _fetch_text(url: str) -> str:
data = _fetch_bytes(url)
try:
return data.decode("utf-8")
except UnicodeDecodeError as e:
raise SkillImportError(f"non-text file: {url}") from e
def _list_github_dir(src: ResolvedSource, rel_dir: str, out: Dict[str, str], *, depth: int = 0) -> None:
if depth > 4 or len(out) >= MAX_FILES:
return
url = _api_contents_url(src, rel_dir)
r = _get_checked(url, headers={"Accept": "application/vnd.github+json"}, timeout=30.0)
if r.status_code >= 400:
raise _github_response_error(r)
_assert_github_url(str(r.url), context="redirect target")
entries = r.json()
if not isinstance(entries, list):
raise SkillImportError("expected a directory on GitHub")
total = sum(len(v.encode("utf-8")) for v in out.values())
for ent in entries:
if len(out) >= MAX_FILES or total >= MAX_TOTAL_BYTES:
break
if not isinstance(ent, dict):
continue
name = ent.get("name") or ""
ent_type = ent.get("type")
rel = _safe_relpath(f"{rel_dir}/{name}" if rel_dir else name)
if ent_type == "dir":
_list_github_dir(src, rel, out, depth=depth + 1)
total = sum(len(v.encode("utf-8")) for v in out.values())
continue
if ent_type != "file" or not _is_text_file(name):
continue
dl = ent.get("download_url")
if not dl:
continue
_assert_github_url(dl, context="download URL")
text = _fetch_text(dl)
total += len(text.encode("utf-8"))
if total > MAX_TOTAL_BYTES:
raise SkillImportError("skill bundle exceeds size limit")
out[rel] = text
def fetch_skill_bundle(url: str) -> Tuple[Dict[str, str], ResolvedSource]:
"""Download SKILL.md and sibling text assets. Returns relative_path → content."""
src = parse_skill_source(url)
files: Dict[str, str] = {}
path = _safe_relpath(src.path) if src.path else ""
if path.lower().endswith("skill.md"):
files[path] = _fetch_text(_raw_url(src, path))
parent = "/".join(path.split("/")[:-1])
if parent:
try:
_list_github_dir(src, parent, files)
except SkillImportError:
pass
return files, src
if path:
try:
_fetch_text(_raw_url(src, f"{path}/SKILL.md"))
_list_github_dir(src, path, files)
return files, src
except Exception:
pass
try:
text = _fetch_text(_raw_url(src, path))
if path.lower().endswith(".md"):
files[path] = text
return files, src
except Exception:
pass
_list_github_dir(src, path, files)
else:
_list_github_dir(src, "", files)
if not any(p.lower().endswith("skill.md") for p in files):
# Flat repo root with SKILL.md only
try:
files["SKILL.md"] = _fetch_text(_raw_url(src, "SKILL.md"))
except Exception as e:
raise SkillImportError(
"No SKILL.md found — link to a skill folder or SKILL.md on GitHub"
) from e
return files, src
def pick_skill_md(files: Dict[str, str]) -> Tuple[str, str]:
for rel, content in files.items():
if rel.lower().endswith("skill.md"):
return rel, content
raise SkillImportError("bundle has no SKILL.md")
def default_category_from_source(src: ResolvedSource) -> str:
return "imported"
services/memory/skill_lifecycle.py
+20
View File
@@ -0,0 +1,20 @@
"""Bounded automatic review queue for user-owned procedural memory."""
import time
def automatic_audit_candidates(skills, limit=8, now=None):
"""Retry transient checks daily and failed repairs weekly, oldest first."""
now = time.time() if now is None else now
pending = []
for skill in skills:
if not skill.get("name") or skill.get("source") == "builtin" or skill.get("status") == "binned":
continue
verdict = skill.get("audit_verdict")
if verdict in {"pass", "skipped"}:
continue
checked = float(skill.get("audited_at") or 0)
delay = 7 * 86400 if verdict in {"fail", "needs_work"} else 86400
if not verdict or now - checked >= delay:
pending.append(skill)
pending.sort(key=lambda skill: float(skill.get("audited_at") or 0))
return pending[:max(1, limit)]
services/memory/skills.py
+287 -68
View File
@@ -6,8 +6,8 @@ YAML frontmatter and a structured markdown body (When to Use / Procedure /
Pitfalls / Verification). See `skill_format.py` for the format.
Usage counters (`uses`, `last_used`) live in a sidecar
`data/skills/_usage.json` keyed by skill name so the SKILL.md content
doesn't churn on every retrieval.
`data/skills/_usage.json` keyed by owner plus skill name so the SKILL.md
content doesn't churn on every retrieval.
Ownership: skills declare `owner: <username>` in frontmatter. Single-user
deployments can leave that blank.
@@ -54,6 +54,25 @@ def _to_float(x, default: float = 0.0) -> float:
return default
def _approval_policy(owner: Optional[str]) -> tuple[bool, float]:
"""Read the user's automatic skill-approval gate without breaking retrieval."""
try:
from routes.prefs_routes import _load_for_user
prefs = _load_for_user(owner) or {}
except Exception:
prefs = {}
try:
from src.settings import get_setting
default_minimum = float(get_setting("skill_autosave_min_confidence", 0.85))
except Exception:
default_minimum = 0.85
try:
minimum = float(prefs.get("skill_min_confidence", default_minimum))
except (TypeError, ValueError):
minimum = default_minimum
return bool(prefs.get("auto_approve_skills", True)), max(0.0, min(1.0, minimum))
# ---------------------------------------------------------------------------
# SkillsManager
# ---------------------------------------------------------------------------
@@ -89,7 +108,7 @@ class SkillsManager:
if not os.path.exists(self.usage_file):
return {}
try:
with open(self.usage_file) as f:
with open(self.usage_file, encoding="utf-8") as f:
d = json.load(f)
return d if isinstance(d, dict) else {}
except Exception:
@@ -101,33 +120,77 @@ class SkillsManager:
atomic_write_json(self.usage_file, usage, indent=2)
except Exception:
tmp = self.usage_file + ".tmp"
with open(tmp, "w") as f:
with open(tmp, "w", encoding="utf-8") as f:
json.dump(usage, f, indent=2)
os.replace(tmp, self.usage_file)
@staticmethod
def _usage_key(name: str, owner: Optional[str] = None) -> str:
# Skill names are not globally unique once multiple owners are present.
# Keep the usage sidecar keyed the same way the skill file is scoped.
return f"{owner}::{name}" if owner else name
def _usage_entry(self, usage: Dict[str, Dict], name: str, owner: Optional[str] = None) -> Dict:
key = self._usage_key(name, owner)
entry = usage.get(key)
if isinstance(entry, dict):
return entry
return {}
def set_audit(self, name: str, verdict: str, by_teacher: bool = False,
worker_model: str = "", teacher_model: str = "") -> None:
worker_model: str = "", teacher_model: str = "",
owner: Optional[str] = None, saved_turns: Optional[int] = None,
saved_tool_calls: Optional[int] = None,
baseline_verdict: Optional[str] = None,
usefulness: Optional[float] = None,
audit_summary: Optional[str] = None) -> None:
"""Record the last test/audit result for a skill in the usage sidecar
(so it surfaces in load() without touching SKILL.md). Drives the
'verified' check + teacher mark on the card."""
import time as _t
usage = self._load_usage()
e = usage.setdefault(name, {"uses": 0, "last_used": None})
key = self._usage_key(name, owner)
e = usage.setdefault(key, {"uses": 0, "last_used": None})
e["audit_verdict"] = verdict
# Replace, rather than retain, the explanation from a previous run.
e["audit_summary"] = str(audit_summary or "")[:2000]
# Version 2 fixes audit-arm isolation and separates functional success
# from baseline utility. Legacy inconclusive results are not evidence
# under that protocol and should be eligible for a clean re-audit.
e["audit_version"] = 2
e["audit_by_teacher"] = bool(by_teacher)
if worker_model:
e["audit_worker_model"] = worker_model
if teacher_model:
e["audit_teacher_model"] = teacher_model
if saved_turns is not None:
try:
e["saved_turns"] = int(saved_turns)
except (TypeError, ValueError):
e.pop("saved_turns", None)
if saved_tool_calls is not None:
try:
e["saved_tool_calls"] = int(saved_tool_calls)
except (TypeError, ValueError):
e.pop("saved_tool_calls", None)
if baseline_verdict is not None:
e["baseline_verdict"] = str(baseline_verdict or "unknown")
if usefulness is not None:
try:
e["usefulness"] = float(usefulness)
except (TypeError, ValueError):
e.pop("usefulness", None)
e["audited_at"] = _t.time()
self._save_usage(usage)
def set_necessity(self, name: str, necessary: bool,
redundant_with=None, reason: str = "") -> None:
redundant_with=None, reason: str = "",
owner: Optional[str] = None) -> None:
"""Record the advisory 'is this skill necessary?' judgment in the usage
sidecar. Surfaced on the card as a flag; never acts on the skill."""
usage = self._load_usage()
e = usage.setdefault(name, {"uses": 0, "last_used": None})
key = self._usage_key(name, owner)
e = usage.setdefault(key, {"uses": 0, "last_used": None})
e["necessity"] = {
"necessary": bool(necessary),
"redundant_with": list(redundant_with or []),
@@ -148,7 +211,7 @@ class SkillsManager:
def _read_skill(self, path: str) -> Optional[Skill]:
try:
with open(path) as f:
with open(path, encoding="utf-8") as f:
text = f.read()
return Skill.from_markdown(text, path=path)
except Exception as e:
@@ -180,6 +243,8 @@ class SkillsManager:
sk = self._read_skill(path)
if not sk:
continue
if sk.source == "builtin":
continue
owner = (sk.owner or "").strip()
if owner == primary_owner:
continue
@@ -207,21 +272,34 @@ class SkillsManager:
if not sk:
continue
d = sk.to_dict()
u = usage.get(sk.name) or {}
u = self._usage_entry(usage, sk.name, sk.owner)
d["uses"] = int(u.get("uses", 0))
d["last_used"] = u.get("last_used")
d["audit_verdict"] = u.get("audit_verdict")
audit_verdict = u.get("audit_verdict")
try:
audit_version = int(u.get("audit_version") or 0)
except (TypeError, ValueError):
audit_version = 0
if audit_verdict == "inconclusive" and audit_version < 2:
audit_verdict = None
d["audit_verdict"] = audit_verdict
d["audit_summary"] = u.get("audit_summary", "") if audit_verdict else ""
d["audit_version"] = audit_version
d["audit_by_teacher"] = bool(u.get("audit_by_teacher"))
d["audit_worker_model"] = u.get("audit_worker_model")
d["audit_teacher_model"] = u.get("audit_teacher_model")
d["audited_at"] = u.get("audited_at")
d["audited_at"] = u.get("audited_at") if audit_verdict else None
d["saved_turns"] = u.get("saved_turns")
d["saved_tool_calls"] = u.get("saved_tool_calls")
d["baseline_verdict"] = u.get("baseline_verdict")
d["usefulness"] = u.get("usefulness")
d["necessity"] = u.get("necessity")
out.append(d)
seen_names.add(sk.name)
# Legacy JSON entries — surfaced as draft, not editable from new flow
if os.path.exists(self.legacy_file):
try:
with open(self.legacy_file) as f:
with open(self.legacy_file, encoding="utf-8") as f:
legacy = json.load(f)
if isinstance(legacy, list):
for row in legacy:
@@ -267,7 +345,11 @@ class SkillsManager:
# leaked legacy / un-stamped skills to every authenticated user.
# Hide them now; the owner needs to be backfilled on disk if those
# skills should be visible to a specific user.
return [s for s in entries if s.get("owner") == owner]
return [
s for s in entries
if s.get("owner") == owner
or (s.get("source") == "builtin" and not s.get("owner"))
]
# ----------------------------------------------------------------------
# CRUD — disk-backed
@@ -308,6 +390,7 @@ class SkillsManager:
# never auto-skipped — a human asked for it. The every-X AI audit
# handles the fuzzier near-duplicates this cheap check won't catch.
_all = self.load_all()
_dedup_pool = _all if owner is None else [s for s in _all if s.get("owner") == owner]
if source != "user":
cand = _tokenize(" ".join([
nm, (description or title or ""),
@@ -315,7 +398,7 @@ class SkillsManager:
" ".join(procedure if procedure is not None else (steps or [])),
]))
if cand:
for s in _all:
for s in _dedup_pool:
ex = _tokenize(" ".join([
s.get("name", ""), s.get("description", ""),
s.get("when_to_use", ""),
@@ -326,7 +409,7 @@ class SkillsManager:
# existing skill's usage and return it so the caller
# knows it already exists.
try:
self.record_use(s["name"])
self.record_use(s["name"], owner=s.get("owner"))
except Exception:
pass
return {**s, "_deduped": True, "_duplicate_of": s.get("name")}
@@ -363,19 +446,81 @@ class SkillsManager:
return sk.to_dict()
def update_skill(self, skill_id: str, updates: Dict) -> bool:
def import_bundle_from_files(
self,
files: Dict[str, str],
*,
owner: Optional[str] = None,
source_url: str = "",
category: str = "imported",
) -> Dict:
"""Install a fetched skill bundle (relative path → text) under skills/."""
from .skill_importer import SkillImportError, pick_skill_md, _safe_relpath
from core.atomic_io import atomic_write_text
if not files:
raise SkillImportError("empty bundle")
_rel, skill_md = pick_skill_md(files)
sk = Skill.from_markdown(skill_md)
nm = slugify(sk.name or _rel.split("/")[-2] or "skill")
cat = slugify(category or sk.category or "imported", fallback="imported")
existing = {s["name"] for s in self.load_all()}
base = nm
i = 2
while nm in existing:
nm = f"{base}-{i}"
i += 1
skill_dir = self._skill_dir(cat, nm)
os.makedirs(skill_dir, exist_ok=True)
# Preserve bundle layout (templates/, references/, etc.) under the skill dir.
for rel, content in files.items():
safe = _safe_relpath(rel)
dest = os.path.join(skill_dir, safe)
os.makedirs(os.path.dirname(dest), exist_ok=True)
atomic_write_text(dest, content)
sk.name = nm
sk.category = cat
sk.owner = owner
sk.source = "imported"
if source_url:
extra = (sk.body_extra or "").strip()
note = f"Imported from {source_url}"
sk.body_extra = f"{extra}\n\n{note}".strip() if extra else note
atomic_write_text(self._skill_file(cat, nm), sk.to_markdown())
sk.path = self._skill_file(cat, nm)
return sk.to_dict()
def update_skill(self, skill_id: str, updates: Dict, owner: Optional[str] = None) -> bool:
"""`skill_id` is the slug name. Allows updating any field plus
renames if `name` changes (file is moved on disk)."""
renames if `name` changes (file is moved on disk).
The call is owner-scoped: it matches a skill on disk only if
`skill.owner == owner` (string compare; both empty-string and
None mean "ownerless"). When `owner is None` (the default), the
call only matches skills whose own `owner` field is empty —
callers that want to edit an owned skill must pass the matching
owner explicitly. This prevents a caller with one owner from
mutating a file owned by another user that happens to share
the same slug across category directories. The `owner` key in
`updates` is also ignored — ownership is not an editable field
via this path; rename or admin tooling is required for that.
"""
for path in self._iter_skill_files():
sk = self._read_skill(path)
if not sk or sk.name != skill_id:
continue
if (sk.owner or "") != (owner or ""):
continue
old_dir = os.path.dirname(path)
# Apply updates in a Skill-shape friendly way
scalar_keys = (
"description", "version", "category", "status", "confidence",
"source", "teacher_model", "owner", "when_to_use",
"source", "teacher_model", "when_to_use",
"body_extra",
)
for k in scalar_keys:
@@ -414,18 +559,21 @@ class SkillsManager:
os.rename(old_dir, new_dir)
# Also rename usage key
usage = self._load_usage()
if skill_id in usage:
usage[sk.name] = usage.pop(skill_id)
old_usage_key = self._usage_key(skill_id, sk.owner)
if old_usage_key in usage:
usage[self._usage_key(sk.name, sk.owner)] = usage.pop(old_usage_key)
self._save_usage(usage)
self._write_skill(sk)
return True
return False
def delete_skill(self, skill_id: str) -> bool:
def delete_skill(self, skill_id: str, owner: Optional[str] = None) -> bool:
for path in self._iter_skill_files():
sk = self._read_skill(path)
if not sk or sk.name != skill_id:
continue
if (sk.owner or "") != (owner or ""):
continue
skill_dir = os.path.dirname(path)
try:
# Remove the whole skill dir
@@ -439,15 +587,17 @@ class SkillsManager:
logger.warning(f"Failed to remove skill dir {skill_dir}: {e}")
return False
usage = self._load_usage()
if skill_id in usage:
del usage[skill_id]
usage_key = self._usage_key(skill_id, sk.owner)
if usage_key in usage:
del usage[usage_key]
self._save_usage(usage)
return True
return False
def record_use(self, skill_id: str) -> None:
def record_use(self, skill_id: str, owner: Optional[str] = None) -> None:
usage = self._load_usage()
entry = usage.setdefault(skill_id, {"uses": 0, "last_used": None})
key = self._usage_key(skill_id, owner)
entry = usage.setdefault(key, {"uses": 0, "last_used": None})
entry["uses"] = int(entry.get("uses", 0)) + 1
entry["last_used"] = int(time.time())
self._save_usage(usage)
@@ -456,24 +606,40 @@ class SkillsManager:
# Reading a single skill (used by the skill_view tool)
# ----------------------------------------------------------------------
def read_skill_md(self, name: str) -> Optional[str]:
def read_skill_md(self, name: str, owner: Optional[str] = None) -> Optional[str]:
for path in self._iter_skill_files():
sk = self._read_skill(path)
if sk and sk.name == name:
try:
with open(path) as f:
return f.read()
except Exception:
return None
if not sk or sk.name != name:
continue
# Built-in skills are shared, ownerless procedures. ``load``
# exposes them to every owner, so direct progressive-disclosure
# reads must apply the same visibility rule as the index/list
# path. Previously a built-in appeared in `list` but `view`
# returned not-found for authenticated users.
if not (
(sk.owner or "") == (owner or "")
or (sk.source == "builtin" and not (sk.owner or ""))
):
continue
try:
with open(path, encoding="utf-8") as f:
return f.read()
except Exception:
return None
return None
def read_skill_reference(self, name: str, ref_path: str) -> Optional[str]:
def read_skill_reference(self, name: str, ref_path: str, owner: Optional[str] = None) -> Optional[str]:
"""Read a sub-file under the skill's directory (references/, etc).
Refuses path traversal."""
for path in self._iter_skill_files():
sk = self._read_skill(path)
if not sk or sk.name != name:
continue
if not (
(sk.owner or "") == (owner or "")
or (sk.source == "builtin" and not (sk.owner or ""))
):
continue
base = os.path.realpath(os.path.dirname(path))
target = os.path.realpath(os.path.join(base, ref_path))
if os.path.commonpath([base, target]) != base or target == os.path.dirname(path):
@@ -481,7 +647,7 @@ class SkillsManager:
if not os.path.isfile(target):
return None
try:
with open(target) as f:
with open(target, encoding="utf-8") as f:
return f.read()
except Exception:
return None
@@ -501,19 +667,12 @@ class SkillsManager:
"""Return the `[{name, description, category, status}]` list the
agent sees in its system prompt.
Includes:
- All published skills.
- Drafts written by the teacher-escalation loop
(`source == "teacher-escalation"`). The whole point of
the teacher loop is for the student to find the new
procedure on the very next turn — waiting for a manual
publish click defeats the loop.
Excludes user-created drafts (status=draft, source != teacher-
escalation) — those are work-in-progress and pollute the
prompt with half-finished procedures.
Includes built-ins plus user skills that have passed their audit and
meet the owner's current automatic-approval threshold. A persistent
``published`` flag is not sufficient: a changed threshold or a legacy
record must not make an unaudited skill eligible for prompt injection.
"""
active_toolsets = active_toolsets or []
auto_approve, min_confidence = _approval_policy(owner)
out = []
for s in self.load(owner=owner):
status = s.get("status")
@@ -524,16 +683,32 @@ class SkillsManager:
pass # let it through
else:
continue
# A stale published record must not remain injectable after an
# audit has recorded a failure. Inconclusive is not a failure.
audit_verdict = str(s.get("audit_verdict") or "").lower()
if audit_verdict in {"needs_work", "fail"}:
continue
if s.get("source") != "builtin" and auto_approve:
if status != "published" or audit_verdict != "pass":
continue
if _to_float(s.get("confidence"), 0.0) < min_confidence:
continue
necessity = s.get("necessity") or {}
if isinstance(necessity, dict) and necessity.get("necessary") is False:
continue
# Platform gating
if platform and s.get("platforms") and platform not in s["platforms"]:
continue
# requires_toolsets: hide unless every required toolset is active
# requires_toolsets: hide unless every required toolset is active.
# active_toolsets=None means the caller doesn't know the active
# set (API listings, chat preface) — don't gate in that case;
# only an explicit list filters.
req = s.get("requires_toolsets") or []
if req and not all(t in active_toolsets for t in req):
if req and active_toolsets is not None and not all(t in active_toolsets for t in req):
continue
# fallback_for_toolsets: hide when any of those toolsets is active
fb = s.get("fallback_for_toolsets") or []
if fb and any(t in active_toolsets for t in fb):
if fb and active_toolsets and any(t in active_toolsets for t in fb):
continue
out.append({
"name": s["name"],
@@ -557,6 +732,8 @@ class SkillsManager:
threshold: float = 0.3,
max_items: int = 5,
min_confidence: float = 0.0,
available_toolsets: Optional[Iterable[str]] = None,
platform: Optional[str] = None,
) -> List[Dict]:
if skills is None:
skills = self.load_all()
@@ -568,26 +745,62 @@ class SkillsManager:
# without a manual publish click. The UI flags teacher-written
# entries with a 🎓 badge so users can demote / delete bad
# ones when they spot them.
skills = [s for s in skills if s.get("status") in ("published", "draft")]
# Confidence gate (used by prompt-injection, NOT by search): a DRAFT
# skill must clear the bar to be injected. Published skills are already
# vetted, so they always qualify. Missing confidence = treat as 1.0
# (legacy skills shouldn't silently vanish). 0 disables the gate.
skills = [
s for s in skills
if s.get("status") in ("published", "draft")
and str(s.get("audit_verdict") or "").lower()
not in {"needs_work", "fail", "skipped"}
]
available = set(available_toolsets) if available_toolsets is not None else None
if available is not None:
skills = [
skill for skill in skills
if all(tool in available for tool in (skill.get("requires_toolsets") or []))
and not any(tool in available for tool in (skill.get("fallback_for_toolsets") or []))
]
if platform:
skills = [
skill for skill in skills
if not skill.get("platforms") or platform in skill.get("platforms", [])
]
# Prompt injection is fail-closed for user skills. Built-ins are
# shipped procedures; every other skill needs a passing audit and a
# confidence score at the user's current threshold.
if min_confidence > 0:
def _passes(s):
if s.get("status") == "published":
if s.get("source") == "builtin":
return True
c = s.get("confidence")
if c is None:
return True # unset → don't filter (legacy)
return _to_float(c, 1.0) >= min_confidence # unparseable → pass
return (
s.get("status") == "published"
and str(s.get("audit_verdict") or "").lower() == "pass"
and _to_float(s.get("confidence"), 0.0) >= min_confidence
)
skills = [s for s in skills if _passes(s)]
if not skills:
return []
query_tokens = _tokenize(query)
semantic_scores: Dict[int, float] = {}
semantic_enabled = str(
os.environ.get("ODYSSEUS_SKILL_SEMANTIC_RETRIEVAL", "1")
).strip().lower() not in {"0", "false", "no", "off"}
if semantic_enabled:
try:
from src.skill_index import semantic_skill_scores
semantic_scores = semantic_skill_scores(query, skills)
except Exception as exc:
logger.debug("Semantic skill retrieval unavailable: %s", exc)
try:
semantic_threshold = float(
os.environ.get("ODYSSEUS_SKILL_SEMANTIC_THRESHOLD", "0.4")
)
except (TypeError, ValueError):
semantic_threshold = 0.4
semantic_threshold = max(-1.0, min(1.0, semantic_threshold))
scored = []
for sk in skills:
for position, sk in enumerate(skills):
text = " ".join([
sk.get("name", ""),
sk.get("description", ""),
@@ -595,16 +808,22 @@ class SkillsManager:
" ".join(sk.get("tags", []) or []),
" ".join(sk.get("procedure", []) or []),
])
score = _jaccard(query_tokens, _tokenize(text))
lexical_score = _jaccard(query_tokens, _tokenize(text))
for tag in sk.get("tags", []) or []:
if tag and tag in query.lower():
score = max(score, 0.3) * 1.3
# Match tags as whole tokens, not substrings: `tag in query`
# boosted e.g. a "ai" tag for any query containing "email".
tag_tokens = _tokenize(tag)
if tag_tokens and tag_tokens <= query_tokens:
lexical_score = max(lexical_score, 0.3) * 1.3
if query.lower() in (sk.get("description") or "").lower():
score = max(score, 0.6)
lexical_score = max(lexical_score, 0.6)
semantic_score = semantic_scores.get(position, -1.0)
if lexical_score < threshold and semantic_score < semantic_threshold:
continue
score = max(lexical_score, semantic_score)
score *= 1.0 + _to_float(sk.get("confidence"), 0.5) * 0.1
if sk.get("uses", 0) > 0:
score *= 1.05
if score >= threshold:
scored.append((score, sk))
scored.append((score, sk))
scored.sort(key=lambda x: x[0], reverse=True)
return [sk for _, sk in scored[:max_items]]