Squash Odysseus development history

scripts/migrate_faiss_to_chroma.py

@@ -26,20 +26,55 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(mess
 logger = logging.getLogger("migrate")
 
 
+def _load_json(path, default):
+    try:
+        with open(path, encoding="utf-8") as f:
+            return json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return default
+
+
+def _memory_map(rows):
+    memories = {}
+    if not isinstance(rows, list):
+        return memories
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        memory_id = row.get("id", "")
+        if memory_id:
+            memories[memory_id] = row
+    return memories
+
+
+def _rag_docstore(data):
+    if not isinstance(data, dict):
+        return [], [], []
+    ids = data.get("ids", [])
+    documents = data.get("documents", [])
+    metadatas = data.get("metadatas", [])
+    if not isinstance(ids, list) or not isinstance(documents, list) or not isinstance(metadatas, list):
+        return [], [], []
+    count = min(len(ids), len(documents), len(metadatas))
+    return ids[:count], documents[:count], metadatas[:count]
+
+
 def migrate_memories():
     """Migrate memory vectors from FAISS to ChromaDB."""
     from src.chroma_client import get_chroma_client
     from src.embeddings import get_embedding_client
-    from src.constants import DATA_DIR
+    from src.constants import MEMORY_VECTORS_DIR, MEMORY_FILE
 
-    ids_path = os.path.join(DATA_DIR, "memory_vectors", "ids.json")
-    memory_path = os.path.join(DATA_DIR, "memory.json")
+    ids_path = os.path.join(MEMORY_VECTORS_DIR, "ids.json")
+    memory_path = MEMORY_FILE
 
     if not os.path.exists(ids_path):
         logger.info("No memory FAISS index found, skipping memory migration")
         return
 
-    ids = json.loads(open(ids_path).read())
+    ids = _load_json(ids_path, [])
+    if not isinstance(ids, list):
+        ids = []
     if not ids:
         logger.info("Memory FAISS index is empty, skipping")
         return
@@ -47,8 +82,7 @@ def migrate_memories():
     # Load memory texts
     memories = {}
     if os.path.exists(memory_path):
-        for mem in json.loads(open(memory_path).read()):
-            memories[mem.get("id", "")] = mem
+        memories = _memory_map(_load_json(memory_path, []))
 
     embed = get_embedding_client()
     if not embed:
@@ -97,10 +131,7 @@ def migrate_rag():
         logger.info("No RAG DocStore found, skipping RAG migration")
         return
 
-    data = json.loads(open(docs_path).read())
-    ids = data.get("ids", [])
-    documents = data.get("documents", [])
-    metadatas = data.get("metadatas", [])
+    ids, documents, metadatas = _rag_docstore(_load_json(docs_path, {}))
 
     if not ids:
         logger.info("RAG DocStore is empty, skipping")