From c6196922d6a30fbfa93666f4228de28d251d45ea Mon Sep 17 00:00:00 2001
From: StressTestor <212606152+StressTestor@users.noreply.github.com>
Date: Mon, 20 Jul 2026 11:06:12 -0600
Subject: [PATCH] fix(rag): skip hidden and junk directories when indexing
 (#5559)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

index_personal_documents walked the whole tree with no pruning, so
pointing RAG at a real-world folder silently swept in .obsidian/ plugin
JS, .git/ internals, node_modules/, and __pycache__/ — multiplying
indexing time and polluting retrieval with junk chunks.

Prune hidden directories and well-known junk directories from the walk,
and skip hidden files. The explicitly passed root is exempt, so a user
who deliberately indexes a hidden directory still gets its contents.
---
 src/rag_vector.py                   | 17 ++++++-
 tests/test_rag_index_hidden_dirs.py | 78 +++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_rag_index_hidden_dirs.py

diff --git a/src/rag_vector.py b/src/rag_vector.py
index 9a4c67cfa..d4774224a 100644
--- a/src/rag_vector.py
+++ b/src/rag_vector.py
@@ -34,6 +34,12 @@ DEFAULT_FILE_EXTENSIONS: Set[str] = {
     '.csv', '.html', '.css', '.js', '.pdf'
 }
 
+# Tool-internal directories that match DEFAULT_FILE_EXTENSIONS but are never
+# the user's content: an Obsidian vault's .obsidian/ is minified plugin JS,
+# .git/ is repo internals, node_modules/ and __pycache__/ are build output.
+# Hidden entries are pruned by name; these cover the common non-hidden junk.
+EXCLUDED_DIR_NAMES: Set[str] = {'node_modules', '__pycache__', 'venv'}
+
 VECTOR_WEIGHT = 0.7
 KEYWORD_WEIGHT = 0.3
 
@@ -497,8 +503,17 @@ class VectorRAG:
         failed = 0
 
         try:
-            for root, _, files in os.walk(directory):
+            for root, dirs, files in os.walk(directory):
+                # Prune in place so os.walk never descends into hidden or
+                # junk directories (#5559). The passed-in root is exempt: a
+                # user who deliberately targets a hidden directory gets it.
+                dirs[:] = [
+                    d for d in dirs
+                    if not d.startswith('.') and d not in EXCLUDED_DIR_NAMES
+                ]
                 for fname in files:
+                    if fname.startswith('.'):
+                        continue
                     fpath = os.path.join(root, fname)
                     ext = Path(fname).suffix.lower()
                     if ext not in file_extensions:
diff --git a/tests/test_rag_index_hidden_dirs.py b/tests/test_rag_index_hidden_dirs.py
new file mode 100644
index 000000000..0c21f1c12
--- /dev/null
+++ b/tests/test_rag_index_hidden_dirs.py
@@ -0,0 +1,78 @@
+"""Regression guard for #5559 — directory indexing must skip hidden directories,
+hidden files, and well-known junk directories.
+
+VectorRAG.index_personal_documents walked the whole tree with no pruning, so
+pointing RAG at a real-world folder (an Obsidian vault, a git repo) swept in
+`.obsidian/` plugin JavaScript, `.git/` internals, `node_modules/`, etc. The
+junk multiplied indexing time and polluted retrieval.
+
+These tests are hermetic — no chromadb; VectorRAG is created via __new__ (skip
+Chroma connect) with add_document stubbed to record which files get indexed.
+"""
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
+
+import src.rag_vector as rag_vector
+
+
+def _make_rag(recorded_sources):
+    rag = rag_vector.VectorRAG.__new__(rag_vector.VectorRAG)  # skip Chroma connect
+
+    def _record(text, metadata):
+        recorded_sources.add(metadata["source"])
+        return True
+
+    rag.add_document = _record
+    return rag
+
+
+def _write(path, content="some real content"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+
+
+def test_index_skips_hidden_and_junk_directories(tmp_path):
+    _write(tmp_path / "note.md")
+    _write(tmp_path / "sub" / "deeper.md")
+    _write(tmp_path / ".obsidian" / "plugins" / "plugin.js")
+    _write(tmp_path / ".git" / "hooks.js")
+    _write(tmp_path / "node_modules" / "lib" / "index.js")
+    _write(tmp_path / "__pycache__" / "cached.py")
+    _write(tmp_path / "venv" / "lib" / "site.py")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    result = rag.index_personal_documents(str(tmp_path))
+
+    assert result["success"] is True
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"note.md", os.path.join("sub", "deeper.md")}
+
+
+def test_index_skips_hidden_files(tmp_path):
+    _write(tmp_path / "visible.md")
+    _write(tmp_path / ".hidden.md")
+    _write(tmp_path / "sub" / ".secret.txt")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(tmp_path))
+
+    indexed = {os.path.relpath(p, str(tmp_path)) for p in recorded}
+    assert indexed == {"visible.md"}
+
+
+def test_explicitly_passed_hidden_root_is_still_indexed(tmp_path):
+    """Pruning applies to children only — a user who deliberately points RAG at
+    a hidden directory gets its contents, minus nested hidden/junk dirs."""
+    root = tmp_path / ".notes"
+    _write(root / "idea.md")
+    _write(root / ".obsidian" / "plugin.js")
+
+    recorded = set()
+    rag = _make_rag(recorded)
+    rag.index_personal_documents(str(root))
+
+    indexed = {os.path.relpath(p, str(root)) for p in recorded}
+    assert indexed == {"idea.md"}