Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5d5b033749 | ||
|
|
748d1cbfae | ||
|
|
a9a5731991 | ||
|
|
bf4e8849c0 | ||
|
|
ab07fa9565 | ||
|
|
18ec213504 | ||
|
|
80229f275c | ||
|
|
a687b770ef | ||
|
|
4ad6598129 | ||
|
|
c7c88b9a2c | ||
|
|
b9f55cb58a | ||
|
|
f4c7156527 | ||
|
|
2f00a9cab7 | ||
|
|
aa1483a18c | ||
|
|
c1cedddd08 | ||
|
|
aac293a053 | ||
|
|
06e4543224 |
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"env": {
|
||||
"PQL_VAULT": "/mnt/media/Projects/library-desk"
|
||||
},
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(pql)",
|
||||
"Bash(pql *)",
|
||||
"Bash(git status:*)",
|
||||
"Bash(git log:*)",
|
||||
"Bash(git diff:*)",
|
||||
"Bash(git branch:*)",
|
||||
"Bash(pytest:*)",
|
||||
"Bash(python -m pytest:*)",
|
||||
"Bash(ruff check:*)",
|
||||
"Bash(docker logs library-desk:*)",
|
||||
"Bash(curl -s http://localhost:8089/*)"
|
||||
],
|
||||
"deny": [
|
||||
"Bash(/mnt/media/Projects/cladmin/ops/bin/toj)",
|
||||
"Bash(/mnt/media/Projects/cladmin/ops/bin/toj:*)",
|
||||
"Bash(chmod -R 777 *)",
|
||||
"Bash(chmod 777 *)",
|
||||
"Bash(dd if=*)",
|
||||
"Bash(dropdb *)",
|
||||
"Bash(find * -delete*)",
|
||||
"Bash(find * -exec*)",
|
||||
"Bash(git * add --all*)",
|
||||
"Bash(git * add -A*)",
|
||||
"Bash(git * add .)",
|
||||
"Bash(git * branch -D *)",
|
||||
"Bash(git * checkout -- *)",
|
||||
"Bash(git * clean -fd*)",
|
||||
"Bash(git * clean -fdx*)",
|
||||
"Bash(git * commit --no-verify*)",
|
||||
"Bash(git * merge --no-ff*)",
|
||||
"Bash(git * push --force*)",
|
||||
"Bash(git * push -f*)",
|
||||
"Bash(git * reset --hard*)",
|
||||
"Bash(git * restore .*)",
|
||||
"Bash(git add --all*)",
|
||||
"Bash(git add -A*)",
|
||||
"Bash(git add .)",
|
||||
"Bash(git branch -D *)",
|
||||
"Bash(git checkout -- *)",
|
||||
"Bash(git clean -fd*)",
|
||||
"Bash(git clean -fdx*)",
|
||||
"Bash(git commit --no-verify*)",
|
||||
"Bash(git merge --no-ff*)",
|
||||
"Bash(git push --force*)",
|
||||
"Bash(git push -f*)",
|
||||
"Bash(git reset --hard*)",
|
||||
"Bash(git restore .*)",
|
||||
"Bash(mkfs*)",
|
||||
"Bash(psql * -c DROP*)",
|
||||
"Bash(psql * DROP DATABASE*)",
|
||||
"Bash(psql * TRUNCATE*)",
|
||||
"Bash(redis-cli * FLUSHALL*)",
|
||||
"Bash(redis-cli * FLUSHDB*)",
|
||||
"Bash(redis-cli FLUSHALL*)",
|
||||
"Bash(redis-cli FLUSHDB*)",
|
||||
"Bash(rm -rf $HOME)",
|
||||
"Bash(rm -rf /)",
|
||||
"Bash(rm -rf ~)",
|
||||
"Bash(su *)",
|
||||
"Bash(sudo *)",
|
||||
"Bash(toj)",
|
||||
"Bash(toj:*)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
.pql/changelog/*.sql merge=union
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Trigger only. The checks live in the Makefile, where they can be read, run by
|
||||
# hand (`make pre-push`), and changed under review.
|
||||
#
|
||||
# This file is identical in every repo in this workspace, deliberately: the call
|
||||
# surface is the same everywhere even though what each gate runs is not, so
|
||||
# nobody has to read a repo to find out how to check it (D-27).
|
||||
#
|
||||
# Enable per clone with: git config core.hooksPath .githooks
|
||||
# Never bypass with --no-verify. Suppress a specific finding deliberately
|
||||
# instead, with a reason — see `make pre-push`.
|
||||
set -euo pipefail
|
||||
exec make -C "$(git rev-parse --show-toplevel)" pre-push
|
||||
+17
@@ -136,3 +136,20 @@ Thumbs.db
|
||||
|
||||
# profiling data
|
||||
.prof
|
||||
|
||||
# Claude Code — local settings hold credentials. A global gitignore covers this
|
||||
# on tower-of-joy, but that protection does not travel with the repo.
|
||||
.claude/settings.local.json
|
||||
|
||||
# pql — ignore everything except the changelog, which is the replication log of
|
||||
# record and must be committed for tickets to travel with the repo.
|
||||
.pql/*
|
||||
!.pql/changelog/
|
||||
|
||||
# pql shims planted by `pql init` into the dir core.hooksPath points at.
|
||||
# Per-clone: each embeds the absolute path of the pql binary that planted it.
|
||||
# Only .githooks/pre-push is shared.
|
||||
.githooks/pre-commit
|
||||
.githooks/post-merge
|
||||
.githooks/post-checkout
|
||||
.githooks/post-rewrite
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Changelog format marker, written by pql. Comments only: this file
|
||||
-- is never executed — Import descends into the per-table directories
|
||||
-- and does not read the changelog root.
|
||||
--
|
||||
-- A changelog carrying no marker is format 1, the shape that existed
|
||||
-- before formats were versioned. An older format is migrated forward
|
||||
-- by `pql plan upgrade` (and automatically from the post-merge hook);
|
||||
-- a newer one is refused rather than replayed under rules this binary
|
||||
-- does not know. See D-28 and docs/versions.md.
|
||||
-- pql:changelog_format: 2.0.0
|
||||
-- pql:written_by: 2.2.0
|
||||
@@ -0,0 +1,139 @@
|
||||
-- Auto-generated by pql init. CREATE TABLE statements
|
||||
-- for the planning schema; per-table dir keeps the changelog
|
||||
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||
-- idempotent so running schema files from each directory in
|
||||
-- replay order is harmless.
|
||||
--
|
||||
-- Importer parses the markers below to detect schema drift
|
||||
-- between the producing pql version and the local one — a
|
||||
-- bumped canonical_version means projection rules changed
|
||||
-- and replay must refuse rather than silently corrupt state.
|
||||
-- pql:created_by: 2.2.0
|
||||
-- pql:canonical_version: 2
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||
domain TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK(status IN ('active','superseded','resolved','open')),
|
||||
date TEXT,
|
||||
file_path TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
ref_type TEXT NOT NULL
|
||||
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (source_id, target_id, ref_type)
|
||||
);
|
||||
|
||||
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||
CREATE TABLE IF NOT EXISTS tickets (
|
||||
record_id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||
-- always inserts the configured default explicitly.
|
||||
status TEXT NOT NULL DEFAULT 'backlog',
|
||||
priority TEXT DEFAULT 'medium'
|
||||
CHECK(priority IN ('critical','high','medium','low')),
|
||||
assigned_to TEXT,
|
||||
team TEXT,
|
||||
decision_ref TEXT REFERENCES decisions(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||
ticket_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
field TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by TEXT,
|
||||
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT UNIQUE,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
label TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (ticket_record_id, label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||
@@ -0,0 +1,139 @@
|
||||
-- Auto-generated by pql init. CREATE TABLE statements
|
||||
-- for the planning schema; per-table dir keeps the changelog
|
||||
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||
-- idempotent so running schema files from each directory in
|
||||
-- replay order is harmless.
|
||||
--
|
||||
-- Importer parses the markers below to detect schema drift
|
||||
-- between the producing pql version and the local one — a
|
||||
-- bumped canonical_version means projection rules changed
|
||||
-- and replay must refuse rather than silently corrupt state.
|
||||
-- pql:created_by: 2.2.0
|
||||
-- pql:canonical_version: 2
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||
domain TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK(status IN ('active','superseded','resolved','open')),
|
||||
date TEXT,
|
||||
file_path TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
ref_type TEXT NOT NULL
|
||||
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (source_id, target_id, ref_type)
|
||||
);
|
||||
|
||||
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||
CREATE TABLE IF NOT EXISTS tickets (
|
||||
record_id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||
-- always inserts the configured default explicitly.
|
||||
status TEXT NOT NULL DEFAULT 'backlog',
|
||||
priority TEXT DEFAULT 'medium'
|
||||
CHECK(priority IN ('critical','high','medium','low')),
|
||||
assigned_to TEXT,
|
||||
team TEXT,
|
||||
decision_ref TEXT REFERENCES decisions(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||
ticket_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
field TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by TEXT,
|
||||
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT UNIQUE,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
label TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (ticket_record_id, label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||
@@ -0,0 +1,43 @@
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'description', NULL, 'Dockerfile runs `uvicorn ... --workers 2`, and `startup_event` in src/main.py creates a
|
||||
WikiChangeListener per worker. Each opens its own PostgreSQL LISTEN connection on
|
||||
`wiki_page_changes`, and Postgres delivers NOTIFY to *every* listening session — so both
|
||||
workers process the same event.
|
||||
|
||||
Proven against production v1.9.1 on 2026-08-08:
|
||||
- pg_stat_activity shows two library_desk_listener sessions, both LISTEN "wiki_page_changes"
|
||||
- "Wiki.js change listener started successfully" is logged twice at startup
|
||||
- a single pg_notify(''wiki_page_changes'', ''claude-probe-no-colons'') produced exactly two
|
||||
"Invalid notification payload" lines, one per worker (that payload is rejected before any
|
||||
ingestion runs, so it is a side-effect-free probe)
|
||||
|
||||
The debounce in wiki_change_listener.py (self._recent_notifications) is an in-process dict
|
||||
and cannot dedupe across workers. No Redis lock or other cross-process guard exists in the
|
||||
process_wiki_page_change path in src/routers/webhooks.py.
|
||||
|
||||
IMPACT: each edit runs ingestion twice — duplicate embedding generation against Ollama plus
|
||||
duplicate Qdrant and Neo4j writes. On tower-of-joy, Ollama shares the GPU with
|
||||
Speaches/Whisper, so the wasted work has a cost beyond CPU.
|
||||
|
||||
NOT YET ESTABLISHED: whether this is merely wasteful or actually corrupting. Check whether
|
||||
repeated ingestion of the same page creates duplicate Neo4j entities/relationships or
|
||||
duplicate Qdrant points — that changes the severity.
|
||||
|
||||
OPTIONS TO WEIGH:
|
||||
- run the listener in one place only (single-worker sidecar, or elect an owner) so the
|
||||
subscription is singular by construction
|
||||
- keep per-worker listeners and add a short-TTL Redis claim lock keyed on page_id, so only
|
||||
the first worker to claim a notification processes it. Redis is already a dependency —
|
||||
the job manager uses it.
|
||||
|
||||
Whatever approach is taken must preserve the reconnect supervision added in v1.9.1 and its
|
||||
tests in tests/test_wiki_change_listener.py.
|
||||
|
||||
Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:25', '2026-08-08 21:53:25.051', '2026-08-08 21:53:25.051', NULL, '14eeab6aa32e7b47edf3bdf1fdb572a8', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.143', '2026-08-08 21:53:37.143', NULL, 'd793037ced050d17996991b71199c4e6', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.178', '2026-08-08 21:53:37.178', NULL, 'a9feaf6cf70f87d9f3e12e73311120d8', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.179', '2026-08-08 21:53:37.179', NULL, 'dfbe9ea63d608c433b8af3291c6ade71', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.179', '2026-08-08 21:53:37.179', NULL, 'ecd603354edd2fc778a38e75782a10a1', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'description', NULL, 'Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync. Needs: query existing documents by path, compare content hashes, return list of updates needed. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.284', '2026-08-08 21:53:37.284', NULL, 'f0dc060e8f3931fa30026d5476d6b906', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'description', NULL, 'Get processing status for a document. Needs a status tracking system (Redis or database) and per-document ingestion progress. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.421', '2026-08-08 21:53:37.421', NULL, '9ac8b8db277ef5f96fec157e6e1db2e7', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'description', NULL, 'Get indexing status for an entire repository. Needs repository-level statistics and tracking of which documents from a repo are indexed. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.573', '2026-08-08 21:53:37.573', NULL, 'cf4fca0273bf1de3fa9d2c63ff97323a', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'description', NULL, 'Check for duplicate or highly similar documents using vector similarity and graph analysis. Needs: get document embedding from Qdrant, find similar vectors above threshold, check graph relationships, return candidates with similarity scores. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.698', '2026-08-08 21:53:37.698', NULL, 'a96ed5423c39f04a515770ad6d60fb10', 2) ON CONFLICT(hash) DO NOTHING;
|
||||
@@ -0,0 +1,139 @@
|
||||
-- Auto-generated by pql init. CREATE TABLE statements
|
||||
-- for the planning schema; per-table dir keeps the changelog
|
||||
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||
-- idempotent so running schema files from each directory in
|
||||
-- replay order is harmless.
|
||||
--
|
||||
-- Importer parses the markers below to detect schema drift
|
||||
-- between the producing pql version and the local one — a
|
||||
-- bumped canonical_version means projection rules changed
|
||||
-- and replay must refuse rather than silently corrupt state.
|
||||
-- pql:created_by: 2.2.0
|
||||
-- pql:canonical_version: 2
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||
domain TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK(status IN ('active','superseded','resolved','open')),
|
||||
date TEXT,
|
||||
file_path TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
ref_type TEXT NOT NULL
|
||||
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (source_id, target_id, ref_type)
|
||||
);
|
||||
|
||||
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||
CREATE TABLE IF NOT EXISTS tickets (
|
||||
record_id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||
-- always inserts the configured default explicitly.
|
||||
status TEXT NOT NULL DEFAULT 'backlog',
|
||||
priority TEXT DEFAULT 'medium'
|
||||
CHECK(priority IN ('critical','high','medium','low')),
|
||||
assigned_to TEXT,
|
||||
team TEXT,
|
||||
decision_ref TEXT REFERENCES decisions(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||
ticket_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
field TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by TEXT,
|
||||
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT UNIQUE,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
label TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (ticket_record_id, label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||
@@ -0,0 +1,6 @@
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'T-1', '2026-08-08 21:53:14.322', '2026-08-08 21:53:14.322', NULL, 'f4caa0a362b236ef584eff529bfca422', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF8M3ZYNMK67A0VQYYQ7W', 'T-2', '2026-08-08 21:53:30.273', '2026-08-08 21:53:30.273', NULL, '413e76364f3d44f17cd86ef8badd107b', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'T-3', '2026-08-08 21:53:30.408', '2026-08-08 21:53:30.408', NULL, '9bab8d22e7616f0376e7af232dc789a1', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'T-4', '2026-08-08 21:53:30.525', '2026-08-08 21:53:30.525', NULL, '188f0b5550707f6c55ccdabf01762e6b', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'T-5', '2026-08-08 21:53:30.651', '2026-08-08 21:53:30.651', NULL, '581e4c4fdba8fcfb414aa5212cb43995', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'T-6', '2026-08-08 21:53:30.767', '2026-08-08 21:53:30.767', NULL, 'fb194a09d4e8750537a61782e8d1eb12', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||
@@ -0,0 +1,139 @@
|
||||
-- Auto-generated by pql init. CREATE TABLE statements
|
||||
-- for the planning schema; per-table dir keeps the changelog
|
||||
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||
-- idempotent so running schema files from each directory in
|
||||
-- replay order is harmless.
|
||||
--
|
||||
-- Importer parses the markers below to detect schema drift
|
||||
-- between the producing pql version and the local one — a
|
||||
-- bumped canonical_version means projection rules changed
|
||||
-- and replay must refuse rather than silently corrupt state.
|
||||
-- pql:created_by: 2.2.0
|
||||
-- pql:canonical_version: 2
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||
domain TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK(status IN ('active','superseded','resolved','open')),
|
||||
date TEXT,
|
||||
file_path TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
ref_type TEXT NOT NULL
|
||||
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (source_id, target_id, ref_type)
|
||||
);
|
||||
|
||||
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||
CREATE TABLE IF NOT EXISTS tickets (
|
||||
record_id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||
-- always inserts the configured default explicitly.
|
||||
status TEXT NOT NULL DEFAULT 'backlog',
|
||||
priority TEXT DEFAULT 'medium'
|
||||
CHECK(priority IN ('critical','high','medium','low')),
|
||||
assigned_to TEXT,
|
||||
team TEXT,
|
||||
decision_ref TEXT REFERENCES decisions(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||
ticket_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
field TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by TEXT,
|
||||
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT UNIQUE,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
label TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (ticket_record_id, label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||
@@ -0,0 +1,139 @@
|
||||
-- Auto-generated by pql init. CREATE TABLE statements
|
||||
-- for the planning schema; per-table dir keeps the changelog
|
||||
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||
-- idempotent so running schema files from each directory in
|
||||
-- replay order is harmless.
|
||||
--
|
||||
-- Importer parses the markers below to detect schema drift
|
||||
-- between the producing pql version and the local one — a
|
||||
-- bumped canonical_version means projection rules changed
|
||||
-- and replay must refuse rather than silently corrupt state.
|
||||
-- pql:created_by: 2.2.0
|
||||
-- pql:canonical_version: 2
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decisions (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||
domain TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK(status IN ('active','superseded','resolved','open')),
|
||||
date TEXT,
|
||||
file_path TEXT NOT NULL,
|
||||
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||
ref_type TEXT NOT NULL
|
||||
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||
note TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (source_id, target_id, ref_type)
|
||||
);
|
||||
|
||||
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||
CREATE TABLE IF NOT EXISTS tickets (
|
||||
record_id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||
-- always inserts the configured default explicitly.
|
||||
status TEXT NOT NULL DEFAULT 'backlog',
|
||||
priority TEXT DEFAULT 'medium'
|
||||
CHECK(priority IN ('critical','high','medium','low')),
|
||||
assigned_to TEXT,
|
||||
team TEXT,
|
||||
decision_ref TEXT REFERENCES decisions(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||
ticket_id TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
field TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by TEXT,
|
||||
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT UNIQUE,
|
||||
canonical_version INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||
label TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
deleted_at TEXT,
|
||||
hash TEXT,
|
||||
canonical_version INTEGER,
|
||||
PRIMARY KEY (ticket_record_id, label)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||
@@ -0,0 +1,49 @@
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'bug', NULL, 'Every wiki page change is ingested twice', NULL, 'backlog', 'high', NULL, NULL, NULL, '2026-08-08 21:53:14.322', '2026-08-08 21:53:14.322', NULL, 'e93392a341072352b147a3466ed3ab5b', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'bug', NULL, 'Every wiki page change is ingested twice', 'Dockerfile runs `uvicorn ... --workers 2`, and `startup_event` in src/main.py creates a
|
||||
WikiChangeListener per worker. Each opens its own PostgreSQL LISTEN connection on
|
||||
`wiki_page_changes`, and Postgres delivers NOTIFY to *every* listening session — so both
|
||||
workers process the same event.
|
||||
|
||||
Proven against production v1.9.1 on 2026-08-08:
|
||||
- pg_stat_activity shows two library_desk_listener sessions, both LISTEN "wiki_page_changes"
|
||||
- "Wiki.js change listener started successfully" is logged twice at startup
|
||||
- a single pg_notify(''wiki_page_changes'', ''claude-probe-no-colons'') produced exactly two
|
||||
"Invalid notification payload" lines, one per worker (that payload is rejected before any
|
||||
ingestion runs, so it is a side-effect-free probe)
|
||||
|
||||
The debounce in wiki_change_listener.py (self._recent_notifications) is an in-process dict
|
||||
and cannot dedupe across workers. No Redis lock or other cross-process guard exists in the
|
||||
process_wiki_page_change path in src/routers/webhooks.py.
|
||||
|
||||
IMPACT: each edit runs ingestion twice — duplicate embedding generation against Ollama plus
|
||||
duplicate Qdrant and Neo4j writes. On tower-of-joy, Ollama shares the GPU with
|
||||
Speaches/Whisper, so the wasted work has a cost beyond CPU.
|
||||
|
||||
NOT YET ESTABLISHED: whether this is merely wasteful or actually corrupting. Check whether
|
||||
repeated ingestion of the same page creates duplicate Neo4j entities/relationships or
|
||||
duplicate Qdrant points — that changes the severity.
|
||||
|
||||
OPTIONS TO WEIGH:
|
||||
- run the listener in one place only (single-worker sidecar, or elect an owner) so the
|
||||
subscription is singular by construction
|
||||
- keep per-worker listeners and add a short-TTL Redis claim lock keyed on page_id, so only
|
||||
the first worker to claim a notification processes it. Redis is already a dependency —
|
||||
the job manager uses it.
|
||||
|
||||
Whatever approach is taken must preserve the reconnect supervision added in v1.9.1 and its
|
||||
tests in tests/test_wiki_change_listener.py.
|
||||
|
||||
Migrated from TODO.md 2026-08-08.', 'backlog', 'high', NULL, NULL, NULL, '2026-08-08 21:53:14.322', '2026-08-08 21:53:25.051', NULL, '34ff99ff78472b24fc35096faa05c8df', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF8M3ZYNMK67A0VQYYQ7W', 'epic', NULL, 'Implement the stub endpoints in src/main.py', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.273', '2026-08-08 21:53:30.273', NULL, '84c45755f930b52d22238199335e6641', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', NULL, 'Implement POST /ingest/check-updates', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:30.407', NULL, '2811d4e9372b5c838f8d524a57fc0b7c', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', NULL, 'Implement GET /ingest/status/{document_id}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:30.525', NULL, 'b9f5120a8a63b817dc815bda5615a75a', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', NULL, 'Implement GET /ingest/repo-status/{repository}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:30.651', NULL, 'cb00e75f1f91e819f15f345eb18db460', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', NULL, 'Implement POST /deduplicate/check', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:30.767', NULL, '0300426ec940c0d43e36241d0b5968d1', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /ingest/check-updates', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:37.143', NULL, '2401d236cc1b289afa88a791a69caeec', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/status/{document_id}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:37.178', NULL, '36d4868e94a8f66a98fc370ac83b1e7c', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/repo-status/{repository}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:37.179', NULL, '8b44d54948e21faf5a7fa6bf20f25281', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /deduplicate/check', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:37.179', NULL, 'd78fc0b019f24a626f6c9c800fc54023', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /ingest/check-updates', 'Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync. Needs: query existing documents by path, compare content hashes, return list of updates needed. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:37.284', NULL, 'a4005f13ed2272d82bcd4b3172e9aa57', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/status/{document_id}', 'Get processing status for a document. Needs a status tracking system (Redis or database) and per-document ingestion progress. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:37.421', NULL, '29670c509669343cababf536be083e6f', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/repo-status/{repository}', 'Get indexing status for an entire repository. Needs repository-level statistics and tracking of which documents from a repo are indexed. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:37.572', NULL, '76be99d15816829ef4cd210107630d76', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /deduplicate/check', 'Check for duplicate or highly similar documents using vector similarity and graph analysis. Needs: get document embedding from Qdrant, find similar vectors above threshold, check graph relationships, return candidates with similarity scores. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:37.698', NULL, '2643ba102e78a305f8b70e329cbba84d', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||
@@ -1,87 +0,0 @@
|
||||
|
||||
# AGENTS.md
|
||||
|
||||
> **Start every session by reading this file.**
|
||||
> This file outlines the operational protocols, coding standards, and architectural decisions for this FastAPI project.
|
||||
|
||||
## 1. Agent Operational Protocols
|
||||
|
||||
### 🧠 Work Patterns (Plan-Act-Reflect)
|
||||
* **Plan:** Before writing code, briefly outline your plan. Identify which files you will touch and what the side effects might be.
|
||||
* **Act:** Execute the changes in small, atomic steps.
|
||||
* **Reflect:** After coding, verify your work. Did you break existing tests? Did you add new tests?
|
||||
|
||||
### 🛡️ Git Discipline
|
||||
* **NEVER commit to `main` or `master` directly.** Always create a feature branch: `feature/your-feature-name` or `fix/issue-description`.
|
||||
* **Commit Messages:** Use the [Conventional Commits](https://www.conventionalcommits.org/) format.
|
||||
* `feat: add user login endpoint`
|
||||
* `fix: resolve database connection timeout`
|
||||
* `refactor: split monolith dependency file`
|
||||
* **Atomic Commits:** Keep commits small. One logical change = one commit.
|
||||
|
||||
### 📝 Changelog Maintenance
|
||||
* **Update `CHANGELOG.md`** with every user-facing change.
|
||||
* Format: `## [Unreleased] - YYYY-MM-DD` followed by `### Added`, `### Changed`, or `### Fixed`.
|
||||
|
||||
### 🚀 Release Flow
|
||||
When changes are ready for deployment:
|
||||
|
||||
1. **Ask user if deploy cycle is desired **
|
||||
|
||||
2. **Update version** in `pyproject.toml`:
|
||||
- Bug fixes: bump patch version (1.8.3 → 1.8.4)
|
||||
- New features: bump minor version (1.8.4 → 1.9.0)
|
||||
|
||||
3. **Update CHANGELOG.md**:
|
||||
- Move items from `[Unreleased]` to new version section
|
||||
- Add release date: `## [1.8.4] - 2025-12-16`
|
||||
|
||||
4. **Commit and tag**:
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix: description of changes"
|
||||
git tag v1.8.4
|
||||
git push origin main --tags
|
||||
```
|
||||
|
||||
5. **CI/CD triggers automatically**:
|
||||
- Gitea CI builds Docker image on new tag
|
||||
- Watchtower pulls and deploys to production
|
||||
- Verify deployment: `curl http://192.168.86.149:8089/health`
|
||||
|
||||
---
|
||||
|
||||
### 🧪 Local Development Setup
|
||||
* **Always test locally first** before committing and deploying. The build-deploy loop is slow.
|
||||
* **Start the local server** with `./wakeup.sh` - logs are written to `logs/server.log` for easy tailing
|
||||
* **Auto-reload**: The wakeup script runs uvicorn in reload mode - code changes are picked up automatically without restart (except for requirements.txt changes)
|
||||
* **Test REST endpoints** against `http://localhost:8778` using curl or similar tools
|
||||
* **Only deploy** when a phase or feature is complete and tested locally
|
||||
* **Environment**: Copy `.env.example` to `.env` and configure for your local setup (Ollama, Redis, Neo4j, Qdrant, Wiki.js hosts)
|
||||
* **Running tests**: Always use the venv explicitly to avoid environment mismatches:
|
||||
```bash
|
||||
.venv/bin/python -m pytest tests/ # All tests
|
||||
.venv/bin/python -m pytest tests/ -v # Verbose output
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. FastAPI Architecture & Best Practices
|
||||
*Reference: [FastAPI Best Practices](https://github.com/zhanymkanov/fastapi-best-practices)*
|
||||
|
||||
### 📂 Project Structure (Directory-based, NOT File-type based)
|
||||
Do **not** group files by type (e.g., one huge `routers` folder). Group by **domain/module** inside a `src/` directory.
|
||||
|
||||
**Correct Structure:**
|
||||
```text
|
||||
src/
|
||||
├── auth/
|
||||
│ ├── router.py # Endpoints
|
||||
│ ├── schemas.py # Pydantic models
|
||||
│ ├── service.py # Business logic (CRUD, etc.)
|
||||
│ ├── dependencies.py# Module-specific dependencies
|
||||
│ └── config.py # Module-specific settings
|
||||
├── posts/
|
||||
│ ├── router.py
|
||||
│ └── ...
|
||||
└── main.py # App entry point
|
||||
@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.9.2] - 2026-08-16
|
||||
|
||||
### Fixed
|
||||
|
||||
- `/health` no longer reports the whole service unhealthy because one
|
||||
dependency is merely slow. Its five probes (neo4j, qdrant, wikijs, searxng,
|
||||
ollama) previously ran one after another with no timeout on neo4j or
|
||||
qdrant, so a single hung dependency could block the response past the
|
||||
container healthcheck's 10s timeout and get the container marked
|
||||
unhealthy for a reason unrelated to its own liveness. Each probe is now
|
||||
bounded at 2s and all five run concurrently, so one hanging dependency is
|
||||
reported unhealthy on its own without holding up the others or the
|
||||
response.
|
||||
- Internal-only: the three probes `check_service_health()` also computes
|
||||
(paperless, system_settings, scheduler) received the same bounding and
|
||||
concurrency, for consistency and because they carried the identical
|
||||
unbounded-hang risk internally. This has no visible effect today — none
|
||||
of the three is currently returned by `/health` — but protects a future
|
||||
caller that does read them.
|
||||
|
||||
## [1.9.1] - 2026-08-08
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -1,17 +1,101 @@
|
||||
# Claude Code Instructions
|
||||
# CLAUDE.md — Library Desk
|
||||
|
||||
**MANDATORY: Read AGENTS.md instead of this file.**
|
||||
FastAPI service that ingests Wiki.js content and serves retrieval over it. Talks to
|
||||
Postgres (Wiki.js DB + `LISTEN/NOTIFY`), Qdrant (vectors), Neo4j (graph), Ollama
|
||||
(embeddings) and Redis (job manager). Deployed on tower-of-joy at **:8089**.
|
||||
|
||||
This project uses a unified configuration file for all LLM coding agents.
|
||||
## Ports — these differ, deliberately
|
||||
|
||||
## Instructions
|
||||
| | Port | How |
|
||||
|---|---|---|
|
||||
| Local dev | **8778** | `./wakeup.sh`, uvicorn reload mode, logs to `logs/server.log` |
|
||||
| Production | **8089** | container; health at `http://192.168.86.149:8089/health` |
|
||||
|
||||
1. **Read and follow AGENTS.md** - All project guidelines are located there
|
||||
2. **Do not modify this file** - Only update AGENTS.md
|
||||
3. **Do not create or modify other agent-specific files** - Use AGENTS.md as the single source of truth
|
||||
Testing `localhost:8089` on the dev box hits the *container*, not your reload server.
|
||||
|
||||
This approach ensures consistent behavior across all LLM coding agents without managing separate configuration files.
|
||||
## Work tracking
|
||||
|
||||
---
|
||||
Work lives in **pql**, not a markdown file. `TODO.md` was migrated and removed 2026-08-08.
|
||||
|
||||
If you need to update project guidelines, edit AGENTS.md, not this file.
|
||||
```bash
|
||||
pql ticket list # open work
|
||||
pql plan whatsnext # next unblocked item, with context
|
||||
pql ticket show T-1 # detail
|
||||
pql ticket new bug "title" # file new work
|
||||
```
|
||||
|
||||
Tickets travel with the repo — `.pql/changelog/` is committed and replayed by the git
|
||||
hooks; the databases are ignored and rebuildable with `pql plan rebuild`. Do not add a
|
||||
TODO section to a markdown file.
|
||||
|
||||
## Known live defect
|
||||
|
||||
**Every wiki page change is ingested twice** (T-1, proven against v1.9.1). The Dockerfile
|
||||
runs `uvicorn --workers 2`, `startup_event` creates a `WikiChangeListener` per worker, and
|
||||
Postgres delivers `NOTIFY` to *every* listening session. The debounce in
|
||||
`wiki_change_listener.py` is an in-process dict and cannot dedupe across workers.
|
||||
|
||||
Relevant when touching ingestion: duplicate work reaches Ollama, Qdrant and Neo4j. Whether
|
||||
it merely wastes GPU or actually corrupts state is **not yet established** — check before
|
||||
assuming either.
|
||||
|
||||
## Working here
|
||||
|
||||
**Plan, act, reflect.** Outline which files you will touch and the side effects before
|
||||
writing. Change in small atomic steps. Afterwards, verify: did existing tests break, and
|
||||
does the new behaviour have a test?
|
||||
|
||||
**Test locally first — the build-deploy loop is slow.** `./wakeup.sh` auto-reloads on code
|
||||
changes (not on `requirements.txt` changes). Deploy only when a feature is complete and
|
||||
tested.
|
||||
|
||||
Run tests through the venv explicitly, to avoid environment mismatch:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m pytest tests/
|
||||
```
|
||||
|
||||
Copy `.env.example` to `.env` and configure Ollama, Redis, Neo4j, Qdrant and Wiki.js hosts.
|
||||
|
||||
## Git
|
||||
|
||||
- **History is linear — no merge commits.** Work on `main`, or a short-lived branch that is
|
||||
fast-forwarded and deleted. (A feature-branch mandate was retired 2026-08-08 to match the
|
||||
workspace convention, which now has no per-repo exceptions.)
|
||||
- **Conventional Commits**: `feat:`, `fix:`, `refactor:`, `docs:`, `chore:`.
|
||||
- **Atomic commits** — one logical change each.
|
||||
- **Stage explicitly. Never `git add -A`** — it is denied by policy, and it sweeps in
|
||||
whatever else is dirty, including secrets.
|
||||
- Update `CHANGELOG.md` with every user-facing change, under `[Unreleased]` in `Added` /
|
||||
`Changed` / `Fixed`.
|
||||
|
||||
## Releasing
|
||||
|
||||
Ask whether a deploy is wanted first — it is not automatic.
|
||||
|
||||
1. Bump the version in `pyproject.toml` (patch for fixes, minor for features).
|
||||
2. Move `[Unreleased]` entries into a dated version section in `CHANGELOG.md`.
|
||||
3. Stage the changed files by name, commit, tag `vX.Y.Z`, `git push origin main --tags`.
|
||||
4. Gitea CI builds the image on the tag; Watchtower deploys it.
|
||||
5. Verify: `curl http://192.168.86.149:8089/health`.
|
||||
|
||||
## Architecture
|
||||
|
||||
Group by **domain, not by file type**. A single large `routers/` folder is the thing to
|
||||
avoid.
|
||||
|
||||
```text
|
||||
src/
|
||||
├── auth/
|
||||
│ ├── router.py # endpoints
|
||||
│ ├── schemas.py # pydantic models
|
||||
│ ├── service.py # business logic
|
||||
│ ├── dependencies.py # module-scoped dependencies
|
||||
│ └── config.py # module-scoped settings
|
||||
└── main.py # app entry point
|
||||
```
|
||||
|
||||
Reference: [FastAPI best practices](https://github.com/zhanymkanov/fastapi-best-practices).
|
||||
|
||||
The live contract is always `http://localhost:8089/openapi.json` (70 paths) — generated from
|
||||
running code, so it cannot drift the way this file can.
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# library-desk — the repo's command surface (D-27).
|
||||
#
|
||||
# Note the port split, which is the trap this repo's CLAUDE.md leads with:
|
||||
# `make run` serves on 8778 with reload, while 8089 is the *container*. Testing
|
||||
# localhost:8089 on this box hits the deployed service, not your dev server.
|
||||
#
|
||||
# Paths resolve here (D-10): `python3` is 3.8 on this host and a bare `pytest`
|
||||
# resolves only in a login shell, so both go through the venv explicitly.
|
||||
|
||||
VENV := $(CURDIR)/.venv
|
||||
PYTHON ?= python3.12
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
.PHONY: help
|
||||
help: ## Show this help
|
||||
@grep -hE '^[a-z][a-z0-9_-]*:.*?## ' $(MAKEFILE_LIST) \
|
||||
| awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
.PHONY: setup
|
||||
setup: ## Create the venv, install dependencies, and prove the result works
|
||||
$(PYTHON) -m venv .venv
|
||||
# requirements-dev.txt pulls in requirements.txt via -r, so this is still
|
||||
# the full runtime set plus pytest/ruff. Installing only requirements.txt
|
||||
# here (as this line originally did) left `make test` and `make lint`
|
||||
# unreachable from a clean `make setup` since the Makefile's introduction
|
||||
# in b9f55cb — pytest was never installed by setup at all. Found by the
|
||||
# check below failing on its first clean-tree run; fixed in the same
|
||||
# commit rather than filed separately, since the new check cannot pass
|
||||
# honestly against the old line.
|
||||
$(VENV)/bin/pip install -r requirements-dev.txt
|
||||
# Exit 0 from pip install is not evidence (D-24) — it is the same exit code
|
||||
# whether requirements.txt matches what's on disk or a dependency silently
|
||||
# failed to install. Two cheap checks, for two different drift modes:
|
||||
# pip check - installed packages satisfy each other's declared
|
||||
# version constraints (a stale/partial install).
|
||||
# pytest --collect-only - every test module actually imports, which
|
||||
# walks the full src/ import graph and is exactly
|
||||
# what catches a *missing* declared dependency
|
||||
# (core-api's sqlalchemy case in T-47). It builds
|
||||
# no client and opens no socket - dependencies.py
|
||||
# wraps client construction in lru_cache getters,
|
||||
# never called at collection time - so this needs
|
||||
# none of the five backing services running.
|
||||
$(VENV)/bin/pip check
|
||||
$(VENV)/bin/python -m pytest tests/ --collect-only -q
|
||||
|
||||
.PHONY: run
|
||||
run: ## Dev server on :8778 with reload (8089 is the container, not this)
|
||||
./wakeup.sh
|
||||
|
||||
.PHONY: test
|
||||
test: ## Run the test suite (no network needed — live tests are marked and self-skip)
|
||||
@test -x $(VENV)/bin/python || { echo "FAIL — no venv; run: make setup"; exit 69; }
|
||||
$(VENV)/bin/python -m pytest tests/
|
||||
|
||||
# The `integration` marker and its RUN_INTEGRATION_TESTS/TEST_TENANT gate already
|
||||
# lived in tests/conftest.py before T-55 (see test_integration.py,
|
||||
# test_tenant_isolation_live.py, test_quality_report_live.py, and the
|
||||
# TestWikiChangeListenerIntegration class) — `make test` never ran them because
|
||||
# `pytest_collection_modifyitems` skips anything carrying the marker unless
|
||||
# RUN_INTEGRATION_TESTS=1. That gate made them silent under `make test`, but
|
||||
# nothing ran them WITH the flag set either, so "runnable" had never been
|
||||
# reasserted. This target is that home (D-26): it sets the flag, selects the
|
||||
# marker, and — the part that matters — fails loudly if selection ever drops to
|
||||
# zero, since a target that passes by collecting nothing is worse than one that
|
||||
# needs a network (T-55).
|
||||
.PHONY: test-integration
|
||||
test-integration: ## Live tests against neo4j/qdrant/wikijs/searxng/ollama (needs network + services)
|
||||
@test -x $(VENV)/bin/python || { echo "FAIL — no venv; run: make setup"; exit 69; }
|
||||
@$(VENV)/bin/python -m pytest tests/ -m integration --collect-only -q >/dev/null 2>&1; \
|
||||
rc=$$?; \
|
||||
if [ "$$rc" = "5" ]; then \
|
||||
echo "FAIL test-integration — selected 0 tests (marker renamed, moved, or lost — this is a defect, not a pass)"; \
|
||||
exit 1; \
|
||||
elif [ "$$rc" != "0" ]; then \
|
||||
echo "FAIL test-integration — collection errored (rc=$$rc)"; \
|
||||
exit $$rc; \
|
||||
fi
|
||||
RUN_INTEGRATION_TESTS=1 $(VENV)/bin/python -m pytest tests/ -m integration -v
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## ruff check over the sources
|
||||
@test -x $(VENV)/bin/ruff || { echo "FAIL — ruff not installed; run: make setup"; exit 69; }
|
||||
$(VENV)/bin/ruff check src tests
|
||||
|
||||
# git hands a hook a non-login shell, which never sees ~/.local/bin — where
|
||||
# gitleaks lands. Without this the scan reports "not installed" on every push,
|
||||
# which is a check that fails open (D-24).
|
||||
export PATH := $(HOME)/.local/bin:/usr/local/bin:$(PATH)
|
||||
|
||||
.PHONY: secrets
|
||||
secrets: ## Scan the commits about to be pushed for credentials
|
||||
@ci/secrets.sh
|
||||
|
||||
# The call surface is identical in every repo; what it runs is not.
|
||||
#
|
||||
# `secrets` runs first, deliberately: it is the only failure here that cannot be
|
||||
# undone by fixing it afterwards. A failed lint costs another commit; a pushed
|
||||
# credential is cached and indexed whether or not it is later deleted.
|
||||
#
|
||||
# Some of these fail today, and are left wired anyway. The state was measured
|
||||
# once and written down in T-56 rather than being worked around here — a gate
|
||||
# quietly narrowed to what already passes is a gate that reports success for
|
||||
# doing nothing, which is the failure this workspace keeps rediscovering.
|
||||
.PHONY: pre-push
|
||||
pre-push: secrets lint ## Everything the pre-push hook runs
|
||||
@echo " -- not gated here yet: test (T-56)"
|
||||
@@ -1,43 +0,0 @@
|
||||
# TODO
|
||||
|
||||
Outstanding work items for Library Desk.
|
||||
|
||||
## Stub Endpoints to Implement
|
||||
|
||||
The following endpoints in `src/main.py` return stub responses and need real implementations:
|
||||
|
||||
### Ingestion Status Endpoints
|
||||
|
||||
#### `POST /ingest/check-updates`
|
||||
Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync.
|
||||
|
||||
**Implementation needed:**
|
||||
1. Query existing documents by path
|
||||
2. Compare content hashes
|
||||
3. Return list of updates needed
|
||||
|
||||
#### `GET /ingest/status/{document_id}`
|
||||
Get processing status for a document.
|
||||
|
||||
**Implementation needed:**
|
||||
- Status tracking system (Redis or database)
|
||||
- Track ingestion progress per document
|
||||
|
||||
#### `GET /ingest/repo-status/{repository}`
|
||||
Get indexing status for an entire repository.
|
||||
|
||||
**Implementation needed:**
|
||||
- Repository-level statistics
|
||||
- Track which documents from a repo are indexed
|
||||
|
||||
### Deduplication
|
||||
|
||||
#### `POST /deduplicate/check`
|
||||
Check for duplicate or highly similar documents using vector similarity and graph analysis.
|
||||
|
||||
**Implementation needed:**
|
||||
1. Get document embedding from Qdrant
|
||||
2. Find similar vectors above threshold
|
||||
3. Check graph relationships
|
||||
4. Return candidates with similarity scores
|
||||
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Secret scan over the commits about to be pushed.
|
||||
#
|
||||
# Lives here rather than inside .githooks/pre-push so it can be read, run by
|
||||
# hand (`make secrets`), and changed under review. A hook is a trigger; it is
|
||||
# not a home for logic. Identical in every repo in this workspace (D-27).
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
# A non-login shell — which is what git gives a hook — skips /etc/profile.d
|
||||
# and never sees ~/.local/bin, where the gitleaks release tarball lands.
|
||||
# Without this the scan reports "not installed" on every push.
|
||||
[ -d "$HOME/.local/bin" ] && PATH="$HOME/.local/bin:$PATH"
|
||||
|
||||
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||
echo "FAIL secrets — gitleaks not installed, so this check would be a no-op pretending to pass." >&2
|
||||
echo " https://github.com/gitleaks/gitleaks/releases → ~/.local/bin/gitleaks" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Scan the outgoing range, not full history. History here carries findings
|
||||
# that are settled — test fixtures and vendored third-party code — and a gate
|
||||
# that fails on something unfixable gets bypassed within a week. What matters
|
||||
# is what is about to leave this machine.
|
||||
if upstream=$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null); then
|
||||
range="$upstream..HEAD"
|
||||
elif git rev-parse --verify --quiet origin/main >/dev/null; then
|
||||
range="origin/main..HEAD"
|
||||
else
|
||||
range=""
|
||||
fi
|
||||
|
||||
if [ -z "$range" ]; then
|
||||
gitleaks dir . --redact --no-banner --exit-code 1 || {
|
||||
echo "FAIL secrets — gitleaks found a credential in the working tree." >&2; exit 1; }
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ -n "$(git log --oneline "$range" 2>/dev/null)" ] || exit 0
|
||||
|
||||
gitleaks git . --log-opts="$range" --redact --no-banner --exit-code 1 >/dev/null 2>&1 || {
|
||||
echo "FAIL secrets — gitleaks found a credential in the commits being pushed." >&2
|
||||
echo " inspect (values redacted): gitleaks git . --log-opts=\"$range\" --redact" >&2
|
||||
echo " then remove and rotate it, or suppress deliberately:" >&2
|
||||
echo " inline '# gitleaks:allow <reason>'" >&2
|
||||
echo " or add the fingerprint to .gitleaksignore WITH a reason" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " ok secrets"
|
||||
@@ -0,0 +1,54 @@
|
||||
# Decisions, Questions, Rejected
|
||||
|
||||
This directory holds structured planning records that pql parses
|
||||
into pql.db. Each record is a `### [DQR]-N: Title` heading inside
|
||||
a markdown file. Files live in three per-type subdirectories:
|
||||
|
||||
- `decisions/<domain>.md` — confirmed design decisions
|
||||
- `questions/<domain>.md` — open questions that may resolve into
|
||||
decisions or rejected proposals
|
||||
- `rejected/<domain>.md` — rejected proposals (kept for the audit
|
||||
trail)
|
||||
|
||||
The parser infers domain from the filename stem and record type
|
||||
from the parent subdirectory.
|
||||
|
||||
D-records that propose implementation work link to `initiative`-type
|
||||
tickets via `decision_ref`. Run `pql decisions show <id>
|
||||
--with-tickets` to inspect implementation status.
|
||||
|
||||
## Recommended domains
|
||||
|
||||
Start with this canonical set; create files as records land in
|
||||
each domain:
|
||||
|
||||
- **architecture** — structural commitments (storage, layering,
|
||||
languages, libraries)
|
||||
- **process** — team workflow (commits, branches, releases, reviews)
|
||||
- **design** — user-facing surface (UX, UI, public APIs)
|
||||
- **coding-conventions** — team-internal code shape (style, lint,
|
||||
file layout)
|
||||
- **testing** — quality strategy (coverage, layers, gates)
|
||||
|
||||
You might also want, project-permitting:
|
||||
|
||||
- `accessibility` — if you ship user-facing software
|
||||
- `security` — if you handle user data or network surfaces
|
||||
- `licensing` — if you release open-source or commercial
|
||||
- `documentation` — if user-docs are non-trivial
|
||||
- `deployment` — if shipping is non-trivial
|
||||
- `performance` — if you have perf budgets / SLOs
|
||||
|
||||
<!-- pql:records (auto-generated; do not edit manually) -->
|
||||
|
||||
## Decisions
|
||||
|
||||
- _(none)_
|
||||
|
||||
## Open questions
|
||||
|
||||
- _(none)_
|
||||
|
||||
## Rejected
|
||||
|
||||
- _(none)_
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "library-desk"
|
||||
version = "1.9.1"
|
||||
version = "1.9.2"
|
||||
description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
@@ -8,7 +8,6 @@ Source selection is driven by user preferences in the settings database.
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from .base import NewsProvider, NewsItem, NewsFeed
|
||||
from .nos import NOSProvider
|
||||
|
||||
@@ -8,7 +8,7 @@ Provides async Neo4j operations with:
|
||||
- Automatic retry on transient failures
|
||||
"""
|
||||
|
||||
from neo4j import AsyncGraphDatabase, AsyncDriver, AsyncSession, READ_ACCESS
|
||||
from neo4j import AsyncGraphDatabase, AsyncDriver, READ_ACCESS
|
||||
from typing import Optional, List, Dict, Any
|
||||
import logging
|
||||
|
||||
|
||||
@@ -529,7 +529,7 @@ class QdrantClientWrapper:
|
||||
query_filter = Filter(must=conditions)
|
||||
|
||||
# Delete points
|
||||
result = await self.client.delete(
|
||||
await self.client.delete(
|
||||
collection_name=collection_name,
|
||||
points_selector=query_filter
|
||||
)
|
||||
|
||||
+201
-81
@@ -8,13 +8,19 @@ Provides FastAPI dependencies for service clients with:
|
||||
- Type aliases for clean endpoint signatures
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from functools import lru_cache
|
||||
from typing import Annotated
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
from fastapi import Depends, HTTPException, Query
|
||||
import logging
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
import secrets
|
||||
|
||||
from fastapi import Request, Security
|
||||
from fastapi.security import HTTPBearer
|
||||
|
||||
from src.config import Settings, get_settings
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
@@ -560,10 +566,150 @@ async def shutdown_clients():
|
||||
logger.info("Service clients shutdown complete")
|
||||
|
||||
|
||||
# Per-probe timeout for the concurrent health checks below, in seconds.
|
||||
#
|
||||
# Bounded well under the container healthcheck's 10s timeout (see Dockerfile).
|
||||
# Because every probe runs concurrently under one asyncio.gather, the wall time
|
||||
# is one bound rather than the sum, so adding probes does not erode the margin —
|
||||
# all eight can hang and the call still returns in ~2s.
|
||||
#
|
||||
# A probe with no bound of its own falls back to its client's default: 30s for
|
||||
# the neo4j and qdrant drivers, 30s for the scheduler client, and none at all
|
||||
# for system settings. Each is past the 10s budget on its own, which is what let
|
||||
# a hung dependency — not a failing one — flip the container unhealthy.
|
||||
HEALTH_PROBE_TIMEOUT = 2.0
|
||||
|
||||
|
||||
async def _probe_neo4j() -> bool:
|
||||
"""Neo4j connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
neo4j = get_neo4j_client()
|
||||
# Simple query to check connectivity
|
||||
await neo4j.execute_query("RETURN 1 as test", {})
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Neo4j health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_qdrant() -> bool:
|
||||
"""Qdrant connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
qdrant = get_qdrant_client()
|
||||
# Check if we can list collections
|
||||
await qdrant.client.get_collections()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Qdrant health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_wikijs() -> bool:
|
||||
"""Wiki.js connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
wikijs = get_wikijs_client()
|
||||
# Try a simple query (list pages with limit 1)
|
||||
await wikijs.list_pages(limit=1)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Wiki.js health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_searxng() -> bool:
|
||||
"""SearXNG connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
searxng = get_searxng_client()
|
||||
# Just check if service is up (no actual search)
|
||||
return await searxng.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"SearXNG health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_ollama() -> bool:
|
||||
"""Ollama connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
ollama = get_ollama_client()
|
||||
return await ollama.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_paperless() -> bool:
|
||||
"""Paperless-ngx connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
paperless = get_paperless_client()
|
||||
return await paperless.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Paperless health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_system_settings() -> bool:
|
||||
"""System settings database connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
return await settings_client.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"System settings health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_scheduler() -> bool:
|
||||
"""Scheduler connectivity probe. Returns True if healthy."""
|
||||
try:
|
||||
scheduler = get_scheduler_client()
|
||||
return await scheduler.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Scheduler health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _bounded_probe(coro) -> bool:
|
||||
"""
|
||||
Run a probe coroutine bounded by HEALTH_PROBE_TIMEOUT.
|
||||
|
||||
A probe that times out is reported unhealthy, the same as one that
|
||||
raises. Wrapping happens here rather than in each _probe_* function so
|
||||
the bound applies uniformly regardless of whether the underlying client
|
||||
has its own (looser, or absent) timeout.
|
||||
"""
|
||||
try:
|
||||
return await asyncio.wait_for(coro, timeout=HEALTH_PROBE_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Health probe timed out after {HEALTH_PROBE_TIMEOUT}s")
|
||||
return False
|
||||
|
||||
|
||||
async def check_service_health() -> dict:
|
||||
"""
|
||||
Check health of all service clients.
|
||||
|
||||
All eight probes (neo4j, qdrant, wikijs, searxng, ollama, paperless,
|
||||
system_settings, scheduler) run concurrently in one gather, each bounded
|
||||
at HEALTH_PROBE_TIMEOUT, so a single hung dependency cannot block the
|
||||
others or push the endpoint past the container healthcheck's timeout.
|
||||
return_exceptions=True means one probe raising cannot cancel its
|
||||
siblings.
|
||||
|
||||
paperless and system_settings are three-state: None means "not
|
||||
configured" (no probe is run at all — a probe never joins the gather
|
||||
for a service that has no credentials to check), False means configured
|
||||
but unreachable/unhealthy (including a timeout), True means healthy.
|
||||
Collapsing "not configured" into "unhealthy" would be a different claim
|
||||
than the one this function is making, so that decision is made before
|
||||
the gather rather than by feeding an unconfigured probe through the
|
||||
same bool-returning bound as the rest.
|
||||
|
||||
Only neo4j, qdrant, wikijs, searxng and ollama are read by /health
|
||||
(src/main.py) — paperless, system_settings and scheduler are computed
|
||||
here but not currently surfaced by any caller (checked: the only two
|
||||
callers are src/main.py and tests/test_integration.py, and the test
|
||||
asserts only the five). Bounded rather than removed, since bounding
|
||||
cannot break a hypothetical consumer and deleting could.
|
||||
|
||||
Returns:
|
||||
Dictionary with health status of each service:
|
||||
{
|
||||
@@ -571,7 +717,10 @@ async def check_service_health() -> dict:
|
||||
"qdrant": bool,
|
||||
"wikijs": bool,
|
||||
"searxng": bool,
|
||||
"ollama": bool
|
||||
"ollama": bool,
|
||||
"paperless": bool | None,
|
||||
"system_settings": bool | None,
|
||||
"scheduler": bool
|
||||
}
|
||||
|
||||
Usage:
|
||||
@@ -580,85 +729,47 @@ async def check_service_health() -> dict:
|
||||
True
|
||||
"""
|
||||
health = {}
|
||||
|
||||
# Neo4j
|
||||
try:
|
||||
neo4j = get_neo4j_client()
|
||||
# Simple query to check connectivity
|
||||
await neo4j.execute_query("RETURN 1 as test", {})
|
||||
health["neo4j"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Neo4j health check failed: {e}")
|
||||
health["neo4j"] = False
|
||||
|
||||
# Qdrant
|
||||
try:
|
||||
qdrant = get_qdrant_client()
|
||||
# Check if we can list collections
|
||||
await qdrant.client.get_collections()
|
||||
health["qdrant"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Qdrant health check failed: {e}")
|
||||
health["qdrant"] = False
|
||||
|
||||
# Wiki.js
|
||||
try:
|
||||
wikijs = get_wikijs_client()
|
||||
# Try a simple query (list pages with limit 1)
|
||||
await wikijs.list_pages(limit=1)
|
||||
health["wikijs"] = True
|
||||
except Exception as e:
|
||||
logger.error(f"Wiki.js health check failed: {e}")
|
||||
health["wikijs"] = False
|
||||
|
||||
# SearXNG
|
||||
try:
|
||||
searxng = get_searxng_client()
|
||||
# Just check if service is up (no actual search)
|
||||
health["searxng"] = await searxng.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"SearXNG health check failed: {e}")
|
||||
health["searxng"] = False
|
||||
|
||||
# Ollama
|
||||
try:
|
||||
ollama = get_ollama_client()
|
||||
is_healthy = await ollama.health_check()
|
||||
health["ollama"] = is_healthy
|
||||
except Exception as e:
|
||||
logger.error(f"Ollama health check failed: {e}")
|
||||
health["ollama"] = False
|
||||
|
||||
# Paperless-ngx
|
||||
settings = get_settings()
|
||||
|
||||
# Build the probe list dynamically: paperless and system_settings only
|
||||
# join it when configured, so an unconfigured service is never bounded,
|
||||
# timed out, or reported False — it is set to None directly, below.
|
||||
probe_names = ["neo4j", "qdrant", "wikijs", "searxng", "ollama"]
|
||||
probes = [
|
||||
_bounded_probe(_probe_neo4j()),
|
||||
_bounded_probe(_probe_qdrant()),
|
||||
_bounded_probe(_probe_wikijs()),
|
||||
_bounded_probe(_probe_searxng()),
|
||||
_bounded_probe(_probe_ollama()),
|
||||
]
|
||||
|
||||
if settings.paperless_token:
|
||||
try:
|
||||
paperless = get_paperless_client()
|
||||
health["paperless"] = await paperless.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Paperless health check failed: {e}")
|
||||
health["paperless"] = False
|
||||
probe_names.append("paperless")
|
||||
probes.append(_bounded_probe(_probe_paperless()))
|
||||
else:
|
||||
health["paperless"] = None # Not configured
|
||||
|
||||
# System Settings database
|
||||
if settings.system_settings_password:
|
||||
try:
|
||||
settings_client = get_settings_client()
|
||||
health["system_settings"] = await settings_client.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"System settings health check failed: {e}")
|
||||
health["system_settings"] = False
|
||||
probe_names.append("system_settings")
|
||||
probes.append(_bounded_probe(_probe_system_settings()))
|
||||
else:
|
||||
health["system_settings"] = None # Not configured
|
||||
|
||||
# Scheduler
|
||||
try:
|
||||
scheduler = get_scheduler_client()
|
||||
health["scheduler"] = await scheduler.health_check()
|
||||
except Exception as e:
|
||||
logger.error(f"Scheduler health check failed: {e}")
|
||||
health["scheduler"] = False
|
||||
# Scheduler has no config gate - it always runs.
|
||||
probe_names.append("scheduler")
|
||||
probes.append(_bounded_probe(_probe_scheduler()))
|
||||
|
||||
results = await asyncio.gather(*probes, return_exceptions=True)
|
||||
|
||||
# _bounded_probe already catches everything from its own probe, but
|
||||
# return_exceptions=True also guards against a bug in _bounded_probe
|
||||
# itself surfacing as an unhandled exception here.
|
||||
for name, result in zip(probe_names, results):
|
||||
if isinstance(result, BaseException):
|
||||
logger.error(f"{name} health check raised unexpectedly: {result}")
|
||||
health[name] = False
|
||||
else:
|
||||
health[name] = result
|
||||
|
||||
return health
|
||||
|
||||
@@ -764,11 +875,6 @@ def get_volatile_cache_service() -> "VolatileCacheService":
|
||||
)
|
||||
|
||||
|
||||
# Authentication
|
||||
import secrets
|
||||
from fastapi import Security, HTTPException, Request
|
||||
from fastapi.security import HTTPBearer
|
||||
|
||||
security = HTTPBearer()
|
||||
|
||||
|
||||
@@ -833,10 +939,24 @@ async def verify_browser_request(
|
||||
raise HTTPException(status_code=401, detail="Unauthenticated")
|
||||
|
||||
|
||||
# Service type aliases for FastAPI endpoint dependencies
|
||||
# These are defined after the factory functions
|
||||
from src.services.vector_service import VectorService
|
||||
from src.services.graph_service import GraphService
|
||||
# Service type aliases for FastAPI endpoint dependencies.
|
||||
# Deliberately imported here rather than at the top: these modules import back
|
||||
# into this one, so a module-level import would cycle. The factory functions
|
||||
# above must exist before they are pulled in.
|
||||
from src.services.vector_service import VectorService # noqa: E402
|
||||
from src.services.graph_service import GraphService # noqa: E402
|
||||
|
||||
# Imported for annotations only. The real imports live inside the functions
|
||||
# that use them, to break an import cycle; a quoted annotation is never
|
||||
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||
# block costs nothing at import time and makes them resolvable again.
|
||||
if TYPE_CHECKING:
|
||||
from src.services.consolidation_service import ConsolidationService
|
||||
from src.services.hybrid_rag_service import HybridRAGService
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.services.rag_search_service import RAGSearchService
|
||||
from src.services.volatile_service import VolatileCacheService
|
||||
from src.services.wiki_service import WikiService
|
||||
|
||||
VectorServiceDep = Annotated[VectorService, Depends(get_vector_service)]
|
||||
GraphServiceDep = Annotated[GraphService, Depends(get_graph_service)]
|
||||
|
||||
+3
-2
@@ -55,8 +55,9 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Register routers
|
||||
from src.routers import (
|
||||
# Register routers. Imported after the app and middleware exist, because the
|
||||
# routers import dependencies that expect a configured app.
|
||||
from src.routers import ( # noqa: E402
|
||||
wiki, tools, graph, vector, hybrid_rag, consolidation,
|
||||
ingestion, entity_linking, webhooks, rag_search, content,
|
||||
maintenance, volatile, documents
|
||||
|
||||
@@ -5,7 +5,7 @@ Used by the consolidation endpoint to process SearchQuery nodes
|
||||
and consolidate knowledge into wiki pages.
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Dict, Any
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class ConsolidationRequest(BaseModel):
|
||||
|
||||
@@ -6,7 +6,6 @@ Provides models for knowledge graph nodes, relationships, and queries.
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from src.core.multi_tenancy import RequiredUser
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
Pydantic models for Document Ingestion system.
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
from src.core.multi_tenancy import RequiredUser
|
||||
|
||||
@@ -9,7 +9,6 @@ Models for:
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
from src.core.multi_tenancy import RequiredUser
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ async def capture_webhook(request: Request):
|
||||
# Try to parse as JSON
|
||||
try:
|
||||
capture["body_json"] = json.loads(body)
|
||||
except:
|
||||
except Exception:
|
||||
capture["body_json"] = None
|
||||
|
||||
# Write to file
|
||||
|
||||
@@ -9,7 +9,7 @@ Creates both:
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from typing import List, Dict, Any, Tuple
|
||||
import re
|
||||
|
||||
from src.config import get_settings
|
||||
|
||||
@@ -10,7 +10,7 @@ import logging
|
||||
|
||||
from src.models.graph import (
|
||||
CypherQueryRequest, CypherQueryResponse,
|
||||
UpdateFromPageRequest, GraphUpdateSummary,
|
||||
GraphUpdateSummary,
|
||||
NodeListResponse, GraphNodeDetail,
|
||||
MindMapResponse
|
||||
)
|
||||
|
||||
@@ -5,19 +5,15 @@ Endpoints for semantic search and vector operations.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query
|
||||
from typing import Optional
|
||||
import logging
|
||||
|
||||
from src.models.vector import (
|
||||
SearchRequest, SearchResponse,
|
||||
VectorUpdateRequest, VectorUpdateSummary,
|
||||
VectorUpdateSummary,
|
||||
CollectionListResponse,
|
||||
DeletePageChunksRequest, DeletePageChunksResponse
|
||||
DeletePageChunksResponse
|
||||
)
|
||||
from src.services.vector_service import VectorService
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.core.dependencies import (
|
||||
QdrantDep, WikiJSDep, OllamaDep, verify_api_key, RequiredUserQuery
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ import logging
|
||||
from src.models.volatile import (
|
||||
VolatileRecordCreate,
|
||||
VolatileRecordResponse,
|
||||
VolatileListResponse,
|
||||
VolatileScheduledResponse,
|
||||
VolatileStatsResponse,
|
||||
VolatileDeleteResponse,
|
||||
|
||||
@@ -5,14 +5,12 @@ Receives webhook events from Wiki.js for page CRUD operations
|
||||
and processes them identically to AI-generated content.
|
||||
"""
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
||||
from fastapi import APIRouter, Depends, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, Literal
|
||||
|
||||
from src.core.dependencies import (
|
||||
get_ingestion_service,
|
||||
get_wiki_service,
|
||||
get_graph_service,
|
||||
verify_api_key
|
||||
)
|
||||
from src.services.ingestion_service import IngestionService
|
||||
@@ -373,7 +371,7 @@ async def process_page_rename(
|
||||
logger.error(f"Failed to apply entity linking: {e}")
|
||||
|
||||
elif path_changed:
|
||||
logger.info(f"Path changed only (move), no re-processing needed")
|
||||
logger.info("Path changed only (move), no re-processing needed")
|
||||
|
||||
logger.info(f"Rename processing complete for page {page_id}")
|
||||
|
||||
@@ -470,9 +468,9 @@ async def cleanup_deleted_page(
|
||||
logger.error(f"Failed to delete orphaned entity {entity_name}: {e}")
|
||||
|
||||
# STEP 5: Clean up broken SearchQuery relationships
|
||||
cleanup_search_query = f"""
|
||||
cleanup_search_query = """
|
||||
MATCH (sq:SearchQuery)-[r:FOUND]->(d:Document)
|
||||
WHERE NOT EXISTS {{(d)}}
|
||||
WHERE NOT EXISTS {(d)}
|
||||
DELETE r
|
||||
RETURN count(r) as cleaned_count
|
||||
"""
|
||||
|
||||
@@ -18,10 +18,6 @@ from src.models.wiki import (
|
||||
from src.services.wiki_service import WikiService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.services.vector_service import VectorService
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.core.dependencies import (
|
||||
WikiJSDep, Neo4jDep, QdrantDep, OllamaDep,
|
||||
verify_api_key, get_settings, get_hybrid_rag_service, get_ingestion_service,
|
||||
|
||||
@@ -15,14 +15,13 @@ import logging
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import TYPE_CHECKING, List, Dict, Any, Optional
|
||||
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.services.wiki_page_writer import WikiPageWriter
|
||||
from src.models.consolidation import (
|
||||
SearchQueryInfo,
|
||||
ConsolidationResult,
|
||||
ConsolidationResponse,
|
||||
MemoryRouteClassification,
|
||||
@@ -30,6 +29,16 @@ from src.models.consolidation import (
|
||||
)
|
||||
from src.config import Settings
|
||||
|
||||
# Imported for annotations only. The real imports live inside the functions
|
||||
# that use them, to break an import cycle; a quoted annotation is never
|
||||
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||
# block costs nothing at import time and makes them resolvable again.
|
||||
if TYPE_CHECKING:
|
||||
from src.clients.scheduler_client import SchedulerClient
|
||||
from src.clients.settings_client import SettingsClient
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.services.volatile_service import VolatileCacheService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -7,10 +7,19 @@ Provides bidirectional entity linking functionality that can be used by:
|
||||
- Any other service that creates wiki pages
|
||||
"""
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
|
||||
from src.core.multi_tenancy import get_neo4j_user_base_label
|
||||
|
||||
# Imported for annotations only. The real imports live inside the functions
|
||||
# that use them, to break an import cycle; a quoted annotation is never
|
||||
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||
# block costs nothing at import time and makes them resolvable again.
|
||||
if TYPE_CHECKING:
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.services.wiki_service import WikiService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -49,12 +58,9 @@ async def apply_bidirectional_entity_linking(
|
||||
"""
|
||||
from src.routers.entity_linking import (
|
||||
link_entities_in_page,
|
||||
EntityLinkingRequest,
|
||||
get_entities_with_paths,
|
||||
add_entity_links_to_content
|
||||
EntityLinkingRequest
|
||||
)
|
||||
from src.core.dependencies import get_graph_service, get_wiki_service, get_ingestion_service
|
||||
from src.models.wiki import WikiPageUpdate
|
||||
from src.core.dependencies import get_graph_service, get_ingestion_service
|
||||
|
||||
forward_links = 0
|
||||
backward_links = 0
|
||||
|
||||
@@ -879,7 +879,7 @@ Feel free to expand it with more details!
|
||||
{
|
||||
"name": r["name"],
|
||||
# Get the entity type label (not the user label)
|
||||
"type": [l for l in r["labels"] if l not in [user_base_label, "Document"]][0]
|
||||
"type": [line for line in r["labels"] if line not in [user_base_label, "Document"]][0]
|
||||
if r["labels"] else "Entity",
|
||||
"path": r.get("path") # Include path if it exists (for entity stub pages)
|
||||
}
|
||||
@@ -1524,7 +1524,7 @@ Feel free to expand it with more details!
|
||||
for r in results:
|
||||
labels = r.get("labels", [])
|
||||
entity_type = next(
|
||||
(l for l in labels if l != user_base_label),
|
||||
(line for line in labels if line != user_base_label),
|
||||
"Unknown"
|
||||
)
|
||||
orphans.append({
|
||||
|
||||
@@ -26,7 +26,7 @@ from src.clients.ollama_client import OllamaClient
|
||||
from src.clients.content_extractor import ContentExtractor
|
||||
from src.config import Settings
|
||||
from src.models.hybrid_rag import (
|
||||
HybridRAGConfig, HybridRAGRequest, HybridRAGResponse,
|
||||
HybridRAGConfig, HybridRAGResponse,
|
||||
HybridRAGResult, TimingBreakdown, KeywordExtraction,
|
||||
RelatedDossier
|
||||
)
|
||||
@@ -123,7 +123,6 @@ class HybridRAGService:
|
||||
timing["query_enhancement_ms"] = (time.time() - phase0_start) * 1000
|
||||
|
||||
# Phase 1: Parallel Retrieval
|
||||
phase1_start = time.time()
|
||||
raw_results = await self._retrieve_parallel(query, user, config, keywords_data)
|
||||
timing["vector_ms"] = raw_results.get("timing", {}).get("vector_ms", 0)
|
||||
timing["graph_ms"] = raw_results.get("timing", {}).get("graph_ms", 0)
|
||||
@@ -357,7 +356,6 @@ JSON:"""
|
||||
"source_status" ('ok', 'failed', or 'disabled')
|
||||
"""
|
||||
tasks = {}
|
||||
timing = {}
|
||||
|
||||
# Vector search
|
||||
if config.enable_vector:
|
||||
|
||||
@@ -14,16 +14,13 @@ This service is called by:
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
import time
|
||||
|
||||
from src.services.vector_service import VectorService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.models.ingestion import (
|
||||
IngestionRequest,
|
||||
IngestionResult,
|
||||
BatchIngestionRequest,
|
||||
BatchIngestionResult
|
||||
)
|
||||
|
||||
|
||||
@@ -21,7 +21,6 @@ from src.clients.content_extractor import ContentExtractor
|
||||
from src.config import Settings
|
||||
from src.models.rag_search import (
|
||||
SearchType,
|
||||
RAGSearchRequest,
|
||||
RAGSearchResult,
|
||||
RAGSearchResponse,
|
||||
)
|
||||
|
||||
@@ -6,9 +6,8 @@ Handles semantic search, document chunking, and embeddings.
|
||||
|
||||
import re
|
||||
import time
|
||||
import hashlib
|
||||
import uuid
|
||||
from typing import List, Dict, Any, Optional
|
||||
from typing import List, Dict, Any
|
||||
import logging
|
||||
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
@@ -17,7 +16,7 @@ from src.clients.ollama_client import OllamaClient
|
||||
from src.core.multi_tenancy import get_qdrant_collection_name, is_path_in_user_namespace
|
||||
from src.models.vector import (
|
||||
SearchResult, SearchResponse, VectorUpdateSummary,
|
||||
DocumentChunk, CollectionInfo, CollectionListResponse
|
||||
CollectionInfo, CollectionListResponse
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -14,12 +14,6 @@ from src.apis import (
|
||||
OpenMeteoProvider,
|
||||
AggregatedNewsProvider,
|
||||
AlphaVantageProvider,
|
||||
CurrentWeather,
|
||||
WeatherForecast,
|
||||
SunTimes,
|
||||
AirQuality,
|
||||
NewsFeed,
|
||||
StockQuote,
|
||||
)
|
||||
from src.services.volatile_service import VolatileCacheService
|
||||
from src.models.volatile import VolatileRecordResponse, VolatileNamespace
|
||||
|
||||
@@ -14,7 +14,6 @@ from datetime import datetime
|
||||
|
||||
from src.config import get_settings
|
||||
from src.core.dependencies import get_ingestion_service
|
||||
from src.services.consolidation_service import ConsolidationService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ Handles business logic for wiki operations with:
|
||||
- Search functionality
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Dict, Any
|
||||
from typing import TYPE_CHECKING, List, Optional, Dict, Any
|
||||
import logging
|
||||
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
@@ -19,6 +19,14 @@ from src.models.wiki import (
|
||||
DossierInfo, DossierList
|
||||
)
|
||||
|
||||
# Imported for annotations only. The real imports live inside the functions
|
||||
# that use them, to break an import cycle; a quoted annotation is never
|
||||
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||
# block costs nothing at import time and makes them resolvable again.
|
||||
if TYPE_CHECKING:
|
||||
from src.services.hybrid_rag_service import HybridRAGService
|
||||
from src.services.wiki_page_writer import WikiPageWriter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
@@ -12,9 +12,7 @@ Run with: pytest tests/test_consolidation.py -v -s
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from typing import AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from datetime import datetime
|
||||
import json
|
||||
|
||||
@@ -22,8 +20,7 @@ from src.services.consolidation_service import ConsolidationService
|
||||
from src.models.consolidation import (
|
||||
ConsolidationRequest,
|
||||
ConsolidationResponse,
|
||||
ConsolidationResult,
|
||||
SearchQueryInfo
|
||||
ConsolidationResult
|
||||
)
|
||||
|
||||
# Test constants
|
||||
|
||||
@@ -285,6 +285,7 @@ class TestAddEntityLinksToContent:
|
||||
# Integration Tests - Full Entity Linking Flow
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestEntityLinkingIntegration:
|
||||
"""Test full entity linking flow."""
|
||||
|
||||
@@ -421,6 +422,7 @@ class TestEntityLinkingIntegration:
|
||||
# Multi-Tenancy Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestEntityLinkingMultiTenancy:
|
||||
"""Test multi-tenancy isolation in entity linking."""
|
||||
|
||||
@@ -464,6 +466,7 @@ class TestEntityLinkingMultiTenancy:
|
||||
# Cleanup
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_entity_linking_test_data(neo4j_client):
|
||||
"""Clean up all test data created by entity linking tests."""
|
||||
@@ -480,4 +483,4 @@ async def test_cleanup_entity_linking_test_data(neo4j_client):
|
||||
"""
|
||||
await neo4j_client.execute_query(cleanup_query)
|
||||
|
||||
print(f"\n✓ Cleaned up entity linking test data")
|
||||
print("\n✓ Cleaned up entity linking test data")
|
||||
|
||||
@@ -10,9 +10,7 @@ Run with: pytest tests/test_graph_service.py -v -s
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from typing import AsyncGenerator
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from src.services.graph_service import GraphService
|
||||
|
||||
|
||||
@@ -29,9 +29,16 @@ from src.clients.content_extractor import ContentExtractor
|
||||
from src.services.hybrid_rag_service import HybridRAGService
|
||||
from src.services.vector_service import VectorService
|
||||
from src.services.graph_service import GraphService
|
||||
from src.models.hybrid_rag import HybridRAGConfig, HybridRAGRequest
|
||||
from src.models.hybrid_rag import HybridRAGConfig
|
||||
from src.config import get_settings
|
||||
|
||||
# Every test in this module goes through hybrid_rag_service -> graph_service ->
|
||||
# neo4j_client, which opens a real Bolt connection at fixture setup (T-55/D-26).
|
||||
# There is no unit/integration split within the file: even the fusion/formatting
|
||||
# classes that look like pure logic still resolve the full fixture chain, so the
|
||||
# whole module is marked rather than picking classes apart from underneath.
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
# Test user to isolate test data
|
||||
TEST_USER = "llm-tester"
|
||||
|
||||
@@ -176,7 +183,7 @@ We deploy microservices using Helm charts and manage them with kubectl.
|
||||
# Cleanup
|
||||
try:
|
||||
await wiki_client.delete_page(page["id"])
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.skip(f"Could not create test page: {e}")
|
||||
@@ -531,7 +538,7 @@ class TestPhase6_Persistence:
|
||||
result = await neo4j_client.execute_query(query, {"search_id": search_id})
|
||||
assert len(result) == 1
|
||||
assert result[0]["query"] == "test query"
|
||||
assert result[0]["processed"] == False
|
||||
assert not result[0]["processed"]
|
||||
|
||||
# Cleanup
|
||||
cleanup_query = f"""
|
||||
@@ -614,7 +621,7 @@ class TestHybridRAG_EndToEnd:
|
||||
assert len(response.context) > 0
|
||||
|
||||
# Log results for inspection
|
||||
print(f"\n=== HybridRAG E2E Test Results ===")
|
||||
print("\n=== HybridRAG E2E Test Results ===")
|
||||
print(f"Query: {response.query}")
|
||||
print(f"Total Results: {response.total_results}")
|
||||
print(f"Source Counts: {response.source_counts}")
|
||||
@@ -623,7 +630,7 @@ class TestHybridRAG_EndToEnd:
|
||||
print(f"Search ID: {response.search_id}")
|
||||
|
||||
if response.results:
|
||||
print(f"\nTop Result:")
|
||||
print("\nTop Result:")
|
||||
top = response.results[0]
|
||||
print(f" Title: {top.title}")
|
||||
print(f" Source: {top.source_type}")
|
||||
@@ -674,7 +681,7 @@ class TestHybridRAG_EndToEnd:
|
||||
config = HybridRAGConfig()
|
||||
|
||||
start = time.time()
|
||||
response = await hybrid_rag_service.search(
|
||||
await hybrid_rag_service.search(
|
||||
query="kubernetes orchestration",
|
||||
user=TEST_USER,
|
||||
config=config
|
||||
@@ -722,7 +729,7 @@ async def test_cleanup_test_data(neo4j_client, qdrant_client):
|
||||
collection_name = get_qdrant_collection_name(TEST_USER)
|
||||
try:
|
||||
await qdrant_client.delete_collection(collection_name)
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f"\n✓ Cleaned up test data for user: {TEST_USER}")
|
||||
|
||||
@@ -11,16 +11,12 @@ Run with: pytest tests/test_ingestion.py -v -s
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from typing import AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from src.services.ingestion_service import IngestionService
|
||||
from src.models.ingestion import (
|
||||
IngestionRequest,
|
||||
IngestionResult,
|
||||
BatchIngestionRequest,
|
||||
BatchIngestionResult
|
||||
BatchIngestionRequest
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ Tests cleanup of:
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from src.routers.maintenance import (
|
||||
cleanup_vectors,
|
||||
cleanup_graph,
|
||||
@@ -18,9 +18,7 @@ from src.routers.maintenance import (
|
||||
CleanupResult,
|
||||
VectorCleanupResponse,
|
||||
GraphCleanupResponse,
|
||||
FullCleanupResponse,
|
||||
HealthCheckResponse,
|
||||
ReindexResponse
|
||||
HealthCheckResponse
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Tests for RAG search service and endpoints."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from src.models.rag_search import (
|
||||
SearchType,
|
||||
|
||||
@@ -538,8 +538,6 @@ class TestSmartCreateEndpoint:
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_returns_201(self, mock_clients):
|
||||
"""Test that successful creation returns 201 status."""
|
||||
from fastapi.testclient import TestClient
|
||||
from unittest.mock import patch
|
||||
|
||||
# This test would require more setup with FastAPI TestClient
|
||||
# For now, we test the model validation
|
||||
|
||||
@@ -539,7 +539,7 @@ class TestVolatileCleanupEndpoint:
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_volatile(self):
|
||||
"""Test volatile cleanup endpoint."""
|
||||
from src.routers.maintenance import cleanup_volatile, VolatileCleanupResponse
|
||||
from src.routers.maintenance import cleanup_volatile
|
||||
|
||||
mock_qdrant = AsyncMock()
|
||||
mock_qdrant.get_volatile_collections = AsyncMock(return_value=[
|
||||
|
||||
@@ -10,7 +10,6 @@ Tests the PostgreSQL NOTIFY/LISTEN change detection system including:
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
@@ -238,7 +237,7 @@ class TestWikiChangeListener:
|
||||
async def test_process_page_delete_calls_cleanup(self, listener):
|
||||
"""Test that DELETE events call cleanup_deleted_page."""
|
||||
with patch('src.routers.webhooks.cleanup_deleted_page', new_callable=AsyncMock) as mock_cleanup, \
|
||||
patch('src.services.wiki_change_listener.get_ingestion_service') as mock_service:
|
||||
patch('src.services.wiki_change_listener.get_ingestion_service'):
|
||||
|
||||
await listener._process_page_change(
|
||||
page_id=123,
|
||||
|
||||
Reference in New Issue
Block a user