Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5d5b033749 | ||
|
|
748d1cbfae | ||
|
|
a9a5731991 | ||
|
|
bf4e8849c0 | ||
|
|
ab07fa9565 | ||
|
|
18ec213504 | ||
|
|
80229f275c | ||
|
|
a687b770ef | ||
|
|
4ad6598129 | ||
|
|
c7c88b9a2c | ||
|
|
b9f55cb58a | ||
|
|
f4c7156527 | ||
|
|
2f00a9cab7 | ||
|
|
aa1483a18c | ||
|
|
c1cedddd08 | ||
|
|
aac293a053 | ||
|
|
06e4543224 | ||
|
|
ecbb0861a0 | ||
|
|
4f09a12171 | ||
|
|
c5f90cdb4f | ||
|
|
f5983c379f | ||
|
|
90515e0f6d | ||
|
|
83d9ade910 |
@@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"env": {
|
||||||
|
"PQL_VAULT": "/mnt/media/Projects/library-desk"
|
||||||
|
},
|
||||||
|
"permissions": {
|
||||||
|
"allow": [
|
||||||
|
"Bash(pql)",
|
||||||
|
"Bash(pql *)",
|
||||||
|
"Bash(git status:*)",
|
||||||
|
"Bash(git log:*)",
|
||||||
|
"Bash(git diff:*)",
|
||||||
|
"Bash(git branch:*)",
|
||||||
|
"Bash(pytest:*)",
|
||||||
|
"Bash(python -m pytest:*)",
|
||||||
|
"Bash(ruff check:*)",
|
||||||
|
"Bash(docker logs library-desk:*)",
|
||||||
|
"Bash(curl -s http://localhost:8089/*)"
|
||||||
|
],
|
||||||
|
"deny": [
|
||||||
|
"Bash(/mnt/media/Projects/cladmin/ops/bin/toj)",
|
||||||
|
"Bash(/mnt/media/Projects/cladmin/ops/bin/toj:*)",
|
||||||
|
"Bash(chmod -R 777 *)",
|
||||||
|
"Bash(chmod 777 *)",
|
||||||
|
"Bash(dd if=*)",
|
||||||
|
"Bash(dropdb *)",
|
||||||
|
"Bash(find * -delete*)",
|
||||||
|
"Bash(find * -exec*)",
|
||||||
|
"Bash(git * add --all*)",
|
||||||
|
"Bash(git * add -A*)",
|
||||||
|
"Bash(git * add .)",
|
||||||
|
"Bash(git * branch -D *)",
|
||||||
|
"Bash(git * checkout -- *)",
|
||||||
|
"Bash(git * clean -fd*)",
|
||||||
|
"Bash(git * clean -fdx*)",
|
||||||
|
"Bash(git * commit --no-verify*)",
|
||||||
|
"Bash(git * merge --no-ff*)",
|
||||||
|
"Bash(git * push --force*)",
|
||||||
|
"Bash(git * push -f*)",
|
||||||
|
"Bash(git * reset --hard*)",
|
||||||
|
"Bash(git * restore .*)",
|
||||||
|
"Bash(git add --all*)",
|
||||||
|
"Bash(git add -A*)",
|
||||||
|
"Bash(git add .)",
|
||||||
|
"Bash(git branch -D *)",
|
||||||
|
"Bash(git checkout -- *)",
|
||||||
|
"Bash(git clean -fd*)",
|
||||||
|
"Bash(git clean -fdx*)",
|
||||||
|
"Bash(git commit --no-verify*)",
|
||||||
|
"Bash(git merge --no-ff*)",
|
||||||
|
"Bash(git push --force*)",
|
||||||
|
"Bash(git push -f*)",
|
||||||
|
"Bash(git reset --hard*)",
|
||||||
|
"Bash(git restore .*)",
|
||||||
|
"Bash(mkfs*)",
|
||||||
|
"Bash(psql * -c DROP*)",
|
||||||
|
"Bash(psql * DROP DATABASE*)",
|
||||||
|
"Bash(psql * TRUNCATE*)",
|
||||||
|
"Bash(redis-cli * FLUSHALL*)",
|
||||||
|
"Bash(redis-cli * FLUSHDB*)",
|
||||||
|
"Bash(redis-cli FLUSHALL*)",
|
||||||
|
"Bash(redis-cli FLUSHDB*)",
|
||||||
|
"Bash(rm -rf $HOME)",
|
||||||
|
"Bash(rm -rf /)",
|
||||||
|
"Bash(rm -rf ~)",
|
||||||
|
"Bash(su *)",
|
||||||
|
"Bash(sudo *)",
|
||||||
|
"Bash(toj)",
|
||||||
|
"Bash(toj:*)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-5
@@ -1,13 +1,13 @@
|
|||||||
# Service URLs for local dev (pointing to your server)
|
# Service URLs for local dev (pointing to your server)
|
||||||
TEST_HOST=192.168.86.149
|
TEST_HOST=192.168.86.149
|
||||||
WIKIJS_URL=http://192.168.86.149:8088
|
WIKIJS_URL=http://wiki:3000
|
||||||
NEO4J_URI=bolt://192.168.86.149:7687
|
NEO4J_URI=bolt://neo4j:7687
|
||||||
QDRANT_HOST=192.168.86.149
|
QDRANT_HOST=192.168.86.149
|
||||||
QDRANT_PORT=6333
|
QDRANT_PORT=6333
|
||||||
OLLAMA_URL=http://192.168.86.149:11434
|
OLLAMA_URL=http://ollama:11434
|
||||||
SEARXNG_URL=http://192.168.86.149:8080
|
SEARXNG_URL=http://searxng:8080
|
||||||
REDIS_HOST=192.168.86.149
|
REDIS_HOST=192.168.86.149
|
||||||
PAPERLESS_URL=http://192.168.86.149:8091
|
PAPERLESS_URL=http://paperless:8000
|
||||||
|
|
||||||
OLLAMA_LLM_MODEL=gemma4:e2b
|
OLLAMA_LLM_MODEL=gemma4:e2b
|
||||||
OLLAMA_EMBEDDING_MODEL=nomic-embed-text
|
OLLAMA_EMBEDDING_MODEL=nomic-embed-text
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
.pql/changelog/*.sql merge=union
|
||||||
@@ -26,7 +26,7 @@ jobs:
|
|||||||
- name: Login to Gitea Registry
|
- name: Login to Gitea Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v3
|
||||||
with:
|
with:
|
||||||
registry: git.schweitz.internal
|
registry: git.schweitz.net
|
||||||
username: ${{ secrets.REGISTRY_USER }}
|
username: ${{ secrets.REGISTRY_USER }}
|
||||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||||
|
|
||||||
@@ -36,8 +36,8 @@ jobs:
|
|||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
tags: |
|
tags: |
|
||||||
git.schweitz.internal/jpmschweitzer/library-desk:latest
|
git.schweitz.net/jpmschweitzer/library-desk:latest
|
||||||
git.schweitz.internal/jpmschweitzer/library-desk:${{ github.ref_name }}
|
git.schweitz.net/jpmschweitzer/library-desk:${{ github.ref_name }}
|
||||||
|
|
||||||
- name: Trigger Watchtower update
|
- name: Trigger Watchtower update
|
||||||
if: success()
|
if: success()
|
||||||
|
|||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Trigger only. The checks live in the Makefile, where they can be read, run by
|
||||||
|
# hand (`make pre-push`), and changed under review.
|
||||||
|
#
|
||||||
|
# This file is identical in every repo in this workspace, deliberately: the call
|
||||||
|
# surface is the same everywhere even though what each gate runs is not, so
|
||||||
|
# nobody has to read a repo to find out how to check it (D-27).
|
||||||
|
#
|
||||||
|
# Enable per clone with: git config core.hooksPath .githooks
|
||||||
|
# Never bypass with --no-verify. Suppress a specific finding deliberately
|
||||||
|
# instead, with a reason — see `make pre-push`.
|
||||||
|
set -euo pipefail
|
||||||
|
exec make -C "$(git rev-parse --show-toplevel)" pre-push
|
||||||
+17
@@ -136,3 +136,20 @@ Thumbs.db
|
|||||||
|
|
||||||
# profiling data
|
# profiling data
|
||||||
.prof
|
.prof
|
||||||
|
|
||||||
|
# Claude Code — local settings hold credentials. A global gitignore covers this
|
||||||
|
# on tower-of-joy, but that protection does not travel with the repo.
|
||||||
|
.claude/settings.local.json
|
||||||
|
|
||||||
|
# pql — ignore everything except the changelog, which is the replication log of
|
||||||
|
# record and must be committed for tickets to travel with the repo.
|
||||||
|
.pql/*
|
||||||
|
!.pql/changelog/
|
||||||
|
|
||||||
|
# pql shims planted by `pql init` into the dir core.hooksPath points at.
|
||||||
|
# Per-clone: each embeds the absolute path of the pql binary that planted it.
|
||||||
|
# Only .githooks/pre-push is shared.
|
||||||
|
.githooks/pre-commit
|
||||||
|
.githooks/post-merge
|
||||||
|
.githooks/post-checkout
|
||||||
|
.githooks/post-rewrite
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
-- Changelog format marker, written by pql. Comments only: this file
|
||||||
|
-- is never executed — Import descends into the per-table directories
|
||||||
|
-- and does not read the changelog root.
|
||||||
|
--
|
||||||
|
-- A changelog carrying no marker is format 1, the shape that existed
|
||||||
|
-- before formats were versioned. An older format is migrated forward
|
||||||
|
-- by `pql plan upgrade` (and automatically from the post-merge hook);
|
||||||
|
-- a newer one is refused rather than replayed under rules this binary
|
||||||
|
-- does not know. See D-28 and docs/versions.md.
|
||||||
|
-- pql:changelog_format: 2.0.0
|
||||||
|
-- pql:written_by: 2.2.0
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
-- Auto-generated by pql init. CREATE TABLE statements
|
||||||
|
-- for the planning schema; per-table dir keeps the changelog
|
||||||
|
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||||
|
-- idempotent so running schema files from each directory in
|
||||||
|
-- replay order is harmless.
|
||||||
|
--
|
||||||
|
-- Importer parses the markers below to detect schema drift
|
||||||
|
-- between the producing pql version and the local one — a
|
||||||
|
-- bumped canonical_version means projection rules changed
|
||||||
|
-- and replay must refuse rather than silently corrupt state.
|
||||||
|
-- pql:created_by: 2.2.0
|
||||||
|
-- pql:canonical_version: 2
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decisions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||||
|
domain TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK(status IN ('active','superseded','resolved','open')),
|
||||||
|
date TEXT,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||||
|
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
ref_type TEXT NOT NULL
|
||||||
|
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||||
|
note TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (source_id, target_id, ref_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||||
|
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||||
|
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||||
|
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||||
|
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||||
|
CREATE TABLE IF NOT EXISTS tickets (
|
||||||
|
record_id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||||
|
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||||
|
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||||
|
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||||
|
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||||
|
-- always inserts the configured default explicitly.
|
||||||
|
status TEXT NOT NULL DEFAULT 'backlog',
|
||||||
|
priority TEXT DEFAULT 'medium'
|
||||||
|
CHECK(priority IN ('critical','high','medium','low')),
|
||||||
|
assigned_to TEXT,
|
||||||
|
team TEXT,
|
||||||
|
decision_ref TEXT REFERENCES decisions(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||||
|
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||||
|
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||||
|
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||||
|
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||||
|
ticket_id TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||||
|
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
changed_by TEXT,
|
||||||
|
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT UNIQUE,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (ticket_record_id, label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
-- Auto-generated by pql init. CREATE TABLE statements
|
||||||
|
-- for the planning schema; per-table dir keeps the changelog
|
||||||
|
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||||
|
-- idempotent so running schema files from each directory in
|
||||||
|
-- replay order is harmless.
|
||||||
|
--
|
||||||
|
-- Importer parses the markers below to detect schema drift
|
||||||
|
-- between the producing pql version and the local one — a
|
||||||
|
-- bumped canonical_version means projection rules changed
|
||||||
|
-- and replay must refuse rather than silently corrupt state.
|
||||||
|
-- pql:created_by: 2.2.0
|
||||||
|
-- pql:canonical_version: 2
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decisions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||||
|
domain TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK(status IN ('active','superseded','resolved','open')),
|
||||||
|
date TEXT,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||||
|
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
ref_type TEXT NOT NULL
|
||||||
|
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||||
|
note TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (source_id, target_id, ref_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||||
|
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||||
|
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||||
|
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||||
|
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||||
|
CREATE TABLE IF NOT EXISTS tickets (
|
||||||
|
record_id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||||
|
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||||
|
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||||
|
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||||
|
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||||
|
-- always inserts the configured default explicitly.
|
||||||
|
status TEXT NOT NULL DEFAULT 'backlog',
|
||||||
|
priority TEXT DEFAULT 'medium'
|
||||||
|
CHECK(priority IN ('critical','high','medium','low')),
|
||||||
|
assigned_to TEXT,
|
||||||
|
team TEXT,
|
||||||
|
decision_ref TEXT REFERENCES decisions(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||||
|
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||||
|
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||||
|
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||||
|
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||||
|
ticket_id TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||||
|
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
changed_by TEXT,
|
||||||
|
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT UNIQUE,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (ticket_record_id, label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'description', NULL, 'Dockerfile runs `uvicorn ... --workers 2`, and `startup_event` in src/main.py creates a
|
||||||
|
WikiChangeListener per worker. Each opens its own PostgreSQL LISTEN connection on
|
||||||
|
`wiki_page_changes`, and Postgres delivers NOTIFY to *every* listening session — so both
|
||||||
|
workers process the same event.
|
||||||
|
|
||||||
|
Proven against production v1.9.1 on 2026-08-08:
|
||||||
|
- pg_stat_activity shows two library_desk_listener sessions, both LISTEN "wiki_page_changes"
|
||||||
|
- "Wiki.js change listener started successfully" is logged twice at startup
|
||||||
|
- a single pg_notify(''wiki_page_changes'', ''claude-probe-no-colons'') produced exactly two
|
||||||
|
"Invalid notification payload" lines, one per worker (that payload is rejected before any
|
||||||
|
ingestion runs, so it is a side-effect-free probe)
|
||||||
|
|
||||||
|
The debounce in wiki_change_listener.py (self._recent_notifications) is an in-process dict
|
||||||
|
and cannot dedupe across workers. No Redis lock or other cross-process guard exists in the
|
||||||
|
process_wiki_page_change path in src/routers/webhooks.py.
|
||||||
|
|
||||||
|
IMPACT: each edit runs ingestion twice — duplicate embedding generation against Ollama plus
|
||||||
|
duplicate Qdrant and Neo4j writes. On tower-of-joy, Ollama shares the GPU with
|
||||||
|
Speaches/Whisper, so the wasted work has a cost beyond CPU.
|
||||||
|
|
||||||
|
NOT YET ESTABLISHED: whether this is merely wasteful or actually corrupting. Check whether
|
||||||
|
repeated ingestion of the same page creates duplicate Neo4j entities/relationships or
|
||||||
|
duplicate Qdrant points — that changes the severity.
|
||||||
|
|
||||||
|
OPTIONS TO WEIGH:
|
||||||
|
- run the listener in one place only (single-worker sidecar, or elect an owner) so the
|
||||||
|
subscription is singular by construction
|
||||||
|
- keep per-worker listeners and add a short-TTL Redis claim lock keyed on page_id, so only
|
||||||
|
the first worker to claim a notification processes it. Redis is already a dependency —
|
||||||
|
the job manager uses it.
|
||||||
|
|
||||||
|
Whatever approach is taken must preserve the reconnect supervision added in v1.9.1 and its
|
||||||
|
tests in tests/test_wiki_change_listener.py.
|
||||||
|
|
||||||
|
Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:25', '2026-08-08 21:53:25.051', '2026-08-08 21:53:25.051', NULL, '14eeab6aa32e7b47edf3bdf1fdb572a8', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.143', '2026-08-08 21:53:37.143', NULL, 'd793037ced050d17996991b71199c4e6', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.178', '2026-08-08 21:53:37.178', NULL, 'a9feaf6cf70f87d9f3e12e73311120d8', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.179', '2026-08-08 21:53:37.179', NULL, 'dfbe9ea63d608c433b8af3291c6ade71', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'parent_id', NULL, 'T-2', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.179', '2026-08-08 21:53:37.179', NULL, 'ecd603354edd2fc778a38e75782a10a1', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'description', NULL, 'Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync. Needs: query existing documents by path, compare content hashes, return list of updates needed. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.284', '2026-08-08 21:53:37.284', NULL, 'f0dc060e8f3931fa30026d5476d6b906', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'description', NULL, 'Get processing status for a document. Needs a status tracking system (Redis or database) and per-document ingestion progress. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.421', '2026-08-08 21:53:37.421', NULL, '9ac8b8db277ef5f96fec157e6e1db2e7', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'description', NULL, 'Get indexing status for an entire repository. Needs repository-level statistics and tracking of which documents from a repo are indexed. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.573', '2026-08-08 21:53:37.573', NULL, 'cf4fca0273bf1de3fa9d2c63ff97323a', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
|
INSERT INTO ticket_history (ticket_record_id, field, old_value, new_value, changed_by, changed_at, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'description', NULL, 'Check for duplicate or highly similar documents using vector similarity and graph analysis. Needs: get document embedding from Qdrant, find similar vectors above threshold, check graph relationships, return candidates with similarity scores. Migrated from TODO.md 2026-08-08.', NULL, '2026-08-08 21:53:37', '2026-08-08 21:53:37.698', '2026-08-08 21:53:37.698', NULL, 'a96ed5423c39f04a515770ad6d60fb10', 2) ON CONFLICT(hash) DO NOTHING;
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
-- Auto-generated by pql init. CREATE TABLE statements
|
||||||
|
-- for the planning schema; per-table dir keeps the changelog
|
||||||
|
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||||
|
-- idempotent so running schema files from each directory in
|
||||||
|
-- replay order is harmless.
|
||||||
|
--
|
||||||
|
-- Importer parses the markers below to detect schema drift
|
||||||
|
-- between the producing pql version and the local one — a
|
||||||
|
-- bumped canonical_version means projection rules changed
|
||||||
|
-- and replay must refuse rather than silently corrupt state.
|
||||||
|
-- pql:created_by: 2.2.0
|
||||||
|
-- pql:canonical_version: 2
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decisions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||||
|
domain TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK(status IN ('active','superseded','resolved','open')),
|
||||||
|
date TEXT,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||||
|
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
ref_type TEXT NOT NULL
|
||||||
|
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||||
|
note TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (source_id, target_id, ref_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||||
|
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||||
|
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||||
|
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||||
|
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||||
|
CREATE TABLE IF NOT EXISTS tickets (
|
||||||
|
record_id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||||
|
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||||
|
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||||
|
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||||
|
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||||
|
-- always inserts the configured default explicitly.
|
||||||
|
status TEXT NOT NULL DEFAULT 'backlog',
|
||||||
|
priority TEXT DEFAULT 'medium'
|
||||||
|
CHECK(priority IN ('critical','high','medium','low')),
|
||||||
|
assigned_to TEXT,
|
||||||
|
team TEXT,
|
||||||
|
decision_ref TEXT REFERENCES decisions(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||||
|
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||||
|
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||||
|
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||||
|
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||||
|
ticket_id TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||||
|
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
changed_by TEXT,
|
||||||
|
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT UNIQUE,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (ticket_record_id, label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'T-1', '2026-08-08 21:53:14.322', '2026-08-08 21:53:14.322', NULL, 'f4caa0a362b236ef584eff529bfca422', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF8M3ZYNMK67A0VQYYQ7W', 'T-2', '2026-08-08 21:53:30.273', '2026-08-08 21:53:30.273', NULL, '413e76364f3d44f17cd86ef8badd107b', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'T-3', '2026-08-08 21:53:30.408', '2026-08-08 21:53:30.408', NULL, '9bab8d22e7616f0376e7af232dc789a1', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'T-4', '2026-08-08 21:53:30.525', '2026-08-08 21:53:30.525', NULL, '188f0b5550707f6c55ccdabf01762e6b', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'T-5', '2026-08-08 21:53:30.651', '2026-08-08 21:53:30.651', NULL, '581e4c4fdba8fcfb414aa5212cb43995', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
|
INSERT INTO ticket_idmap (record_id, ticket_id, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'T-6', '2026-08-08 21:53:30.767', '2026-08-08 21:53:30.767', NULL, 'fb194a09d4e8750537a61782e8d1eb12', 2) ON CONFLICT(record_id) DO UPDATE SET ticket_id=excluded.ticket_id, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= ticket_idmap.updated_at;
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
-- Auto-generated by pql init. CREATE TABLE statements
|
||||||
|
-- for the planning schema; per-table dir keeps the changelog
|
||||||
|
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||||
|
-- idempotent so running schema files from each directory in
|
||||||
|
-- replay order is harmless.
|
||||||
|
--
|
||||||
|
-- Importer parses the markers below to detect schema drift
|
||||||
|
-- between the producing pql version and the local one — a
|
||||||
|
-- bumped canonical_version means projection rules changed
|
||||||
|
-- and replay must refuse rather than silently corrupt state.
|
||||||
|
-- pql:created_by: 2.2.0
|
||||||
|
-- pql:canonical_version: 2
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decisions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||||
|
domain TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK(status IN ('active','superseded','resolved','open')),
|
||||||
|
date TEXT,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||||
|
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
ref_type TEXT NOT NULL
|
||||||
|
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||||
|
note TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (source_id, target_id, ref_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||||
|
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||||
|
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||||
|
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||||
|
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||||
|
CREATE TABLE IF NOT EXISTS tickets (
|
||||||
|
record_id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||||
|
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||||
|
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||||
|
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||||
|
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||||
|
-- always inserts the configured default explicitly.
|
||||||
|
status TEXT NOT NULL DEFAULT 'backlog',
|
||||||
|
priority TEXT DEFAULT 'medium'
|
||||||
|
CHECK(priority IN ('critical','high','medium','low')),
|
||||||
|
assigned_to TEXT,
|
||||||
|
team TEXT,
|
||||||
|
decision_ref TEXT REFERENCES decisions(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||||
|
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||||
|
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||||
|
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||||
|
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||||
|
ticket_id TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||||
|
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
changed_by TEXT,
|
||||||
|
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT UNIQUE,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (ticket_record_id, label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
-- Auto-generated by pql init. CREATE TABLE statements
|
||||||
|
-- for the planning schema; per-table dir keeps the changelog
|
||||||
|
-- self-describing per D-15. CREATE TABLE IF NOT EXISTS is
|
||||||
|
-- idempotent so running schema files from each directory in
|
||||||
|
-- replay order is harmless.
|
||||||
|
--
|
||||||
|
-- Importer parses the markers below to detect schema drift
|
||||||
|
-- between the producing pql version and the local one — a
|
||||||
|
-- bumped canonical_version means projection rules changed
|
||||||
|
-- and replay must refuse rather than silently corrupt state.
|
||||||
|
-- pql:created_by: 2.2.0
|
||||||
|
-- pql:canonical_version: 2
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decisions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('confirmed','question','rejected')),
|
||||||
|
domain TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active'
|
||||||
|
CHECK(status IN ('active','superseded','resolved','open')),
|
||||||
|
date TEXT,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
synced_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS decision_refs (
|
||||||
|
source_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
target_id TEXT NOT NULL REFERENCES decisions(id) ON DELETE CASCADE,
|
||||||
|
ref_type TEXT NOT NULL
|
||||||
|
CHECK(ref_type IN ('supersedes','references','resolves','depends_on','amends')),
|
||||||
|
note TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (source_id, target_id, ref_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Identity split (D-26): a ticket's stable, collision-proof identity is its
|
||||||
|
-- record_id (a locally-generated ULID, planning.NewRecordID); the friendly
|
||||||
|
-- T-NNN label lives in ticket_idmap and may be reconciled. Every structural
|
||||||
|
-- reference (parent, deps, history, labels) targets record_id, so a label
|
||||||
|
-- clash never corrupts the graph — only ticket_idmap needs a relabel.
|
||||||
|
CREATE TABLE IF NOT EXISTS tickets (
|
||||||
|
record_id TEXT PRIMARY KEY,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('initiative','epic','story','task','bug')),
|
||||||
|
parent_record_id TEXT REFERENCES tickets(record_id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
-- No CHECK enumeration: the ticket status vocabulary is per-vault
|
||||||
|
-- configurable (ticket_statuses in .pql/config.yaml). Validation lives
|
||||||
|
-- in Go (planning.StatusSet), so adding/renaming statuses needs no
|
||||||
|
-- schema change. The DEFAULT is a harmless fallback — CreateTicket
|
||||||
|
-- always inserts the configured default explicitly.
|
||||||
|
status TEXT NOT NULL DEFAULT 'backlog',
|
||||||
|
priority TEXT DEFAULT 'medium'
|
||||||
|
CHECK(priority IN ('critical','high','medium','low')),
|
||||||
|
assigned_to TEXT,
|
||||||
|
team TEXT,
|
||||||
|
decision_ref TEXT REFERENCES decisions(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ticket_idmap maps a record_id to its current friendly label (T-NNN).
|
||||||
|
-- ticket_id is intentionally NOT globally unique: two uncoordinated clones
|
||||||
|
-- can mint the same label, which surfaces as a duplicate-label collision
|
||||||
|
-- (detected at replay) and is fixed with "pql ticket relabel".
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_idmap (
|
||||||
|
record_id TEXT PRIMARY KEY REFERENCES tickets(record_id),
|
||||||
|
ticket_id TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_deps (
|
||||||
|
blocker_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
blocked_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (blocker_record_id, blocked_record_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_history (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
field TEXT NOT NULL,
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
changed_by TEXT,
|
||||||
|
changed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT UNIQUE,
|
||||||
|
canonical_version INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS ticket_labels (
|
||||||
|
ticket_record_id TEXT NOT NULL REFERENCES tickets(record_id),
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
deleted_at TEXT,
|
||||||
|
hash TEXT,
|
||||||
|
canonical_version INTEGER,
|
||||||
|
PRIMARY KEY (ticket_record_id, label)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_team ON tickets(team);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_decision_ref ON tickets(decision_ref);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_assigned ON tickets(assigned_to);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tickets_parent ON tickets(parent_record_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ticket_idmap_label ON ticket_idmap(ticket_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_domain ON decisions(domain);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decisions_type ON decisions(type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_decision_refs_target ON decision_refs(target_id);
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'bug', NULL, 'Every wiki page change is ingested twice', NULL, 'backlog', 'high', NULL, NULL, NULL, '2026-08-08 21:53:14.322', '2026-08-08 21:53:14.322', NULL, 'e93392a341072352b147a3466ed3ab5b', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QDAA8EWH7D2KKQF3TBHXR', 'bug', NULL, 'Every wiki page change is ingested twice', 'Dockerfile runs `uvicorn ... --workers 2`, and `startup_event` in src/main.py creates a
|
||||||
|
WikiChangeListener per worker. Each opens its own PostgreSQL LISTEN connection on
|
||||||
|
`wiki_page_changes`, and Postgres delivers NOTIFY to *every* listening session — so both
|
||||||
|
workers process the same event.
|
||||||
|
|
||||||
|
Proven against production v1.9.1 on 2026-08-08:
|
||||||
|
- pg_stat_activity shows two library_desk_listener sessions, both LISTEN "wiki_page_changes"
|
||||||
|
- "Wiki.js change listener started successfully" is logged twice at startup
|
||||||
|
- a single pg_notify(''wiki_page_changes'', ''claude-probe-no-colons'') produced exactly two
|
||||||
|
"Invalid notification payload" lines, one per worker (that payload is rejected before any
|
||||||
|
ingestion runs, so it is a side-effect-free probe)
|
||||||
|
|
||||||
|
The debounce in wiki_change_listener.py (self._recent_notifications) is an in-process dict
|
||||||
|
and cannot dedupe across workers. No Redis lock or other cross-process guard exists in the
|
||||||
|
process_wiki_page_change path in src/routers/webhooks.py.
|
||||||
|
|
||||||
|
IMPACT: each edit runs ingestion twice — duplicate embedding generation against Ollama plus
|
||||||
|
duplicate Qdrant and Neo4j writes. On tower-of-joy, Ollama shares the GPU with
|
||||||
|
Speaches/Whisper, so the wasted work has a cost beyond CPU.
|
||||||
|
|
||||||
|
NOT YET ESTABLISHED: whether this is merely wasteful or actually corrupting. Check whether
|
||||||
|
repeated ingestion of the same page creates duplicate Neo4j entities/relationships or
|
||||||
|
duplicate Qdrant points — that changes the severity.
|
||||||
|
|
||||||
|
OPTIONS TO WEIGH:
|
||||||
|
- run the listener in one place only (single-worker sidecar, or elect an owner) so the
|
||||||
|
subscription is singular by construction
|
||||||
|
- keep per-worker listeners and add a short-TTL Redis claim lock keyed on page_id, so only
|
||||||
|
the first worker to claim a notification processes it. Redis is already a dependency —
|
||||||
|
the job manager uses it.
|
||||||
|
|
||||||
|
Whatever approach is taken must preserve the reconnect supervision added in v1.9.1 and its
|
||||||
|
tests in tests/test_wiki_change_listener.py.
|
||||||
|
|
||||||
|
Migrated from TODO.md 2026-08-08.', 'backlog', 'high', NULL, NULL, NULL, '2026-08-08 21:53:14.322', '2026-08-08 21:53:25.051', NULL, '34ff99ff78472b24fc35096faa05c8df', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF8M3ZYNMK67A0VQYYQ7W', 'epic', NULL, 'Implement the stub endpoints in src/main.py', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.273', '2026-08-08 21:53:30.273', NULL, '84c45755f930b52d22238199335e6641', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', NULL, 'Implement POST /ingest/check-updates', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:30.407', NULL, '2811d4e9372b5c838f8d524a57fc0b7c', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', NULL, 'Implement GET /ingest/status/{document_id}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:30.525', NULL, 'b9f5120a8a63b817dc815bda5615a75a', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', NULL, 'Implement GET /ingest/repo-status/{repository}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:30.651', NULL, 'cb00e75f1f91e819f15f345eb18db460', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', NULL, 'Implement POST /deduplicate/check', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:30.767', NULL, '0300426ec940c0d43e36241d0b5968d1', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /ingest/check-updates', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:37.143', NULL, '2401d236cc1b289afa88a791a69caeec', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/status/{document_id}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:37.178', NULL, '36d4868e94a8f66a98fc370ac83b1e7c', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/repo-status/{repository}', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:37.179', NULL, '8b44d54948e21faf5a7fa6bf20f25281', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /deduplicate/check', NULL, 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:37.179', NULL, 'd78fc0b019f24a626f6c9c800fc54023', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF94YZ1RRP8MYEZCZKJRM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /ingest/check-updates', 'Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync. Needs: query existing documents by path, compare content hashes, return list of updates needed. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.407', '2026-08-08 21:53:37.284', NULL, 'a4005f13ed2272d82bcd4b3172e9aa57', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QF9KJ96C95T493C8WHQM0', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/status/{document_id}', 'Get processing status for a document. Needs a status tracking system (Redis or database) and per-document ingestion progress. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.525', '2026-08-08 21:53:37.421', NULL, '29670c509669343cababf536be083e6f', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFA3CB42112FNPTADNCCG', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement GET /ingest/repo-status/{repository}', 'Get indexing status for an entire repository. Needs repository-level statistics and tracking of which documents from a repo are indexed. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.651', '2026-08-08 21:53:37.572', NULL, '76be99d15816829ef4cd210107630d76', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
|
INSERT INTO tickets (record_id, type, parent_record_id, title, description, status, priority, assigned_to, team, decision_ref, created_at, updated_at, deleted_at, hash, canonical_version) VALUES ('06FY6QFAHYRXZBK58NY73K3YSM', 'task', '06FY6QF8M3ZYNMK67A0VQYYQ7W', 'Implement POST /deduplicate/check', 'Check for duplicate or highly similar documents using vector similarity and graph analysis. Needs: get document embedding from Qdrant, find similar vectors above threshold, check graph relationships, return candidates with similarity scores. Migrated from TODO.md 2026-08-08.', 'backlog', 'medium', NULL, NULL, NULL, '2026-08-08 21:53:30.767', '2026-08-08 21:53:37.698', NULL, '2643ba102e78a305f8b70e329cbba84d', 2) ON CONFLICT(record_id) DO UPDATE SET type=excluded.type, parent_record_id=excluded.parent_record_id, title=excluded.title, description=excluded.description, status=excluded.status, priority=excluded.priority, assigned_to=excluded.assigned_to, team=excluded.team, decision_ref=excluded.decision_ref, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, hash=excluded.hash, canonical_version=excluded.canonical_version WHERE excluded.updated_at >= tickets.updated_at;
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
|
|
||||||
# AGENTS.md
|
|
||||||
|
|
||||||
> **Start every session by reading this file.**
|
|
||||||
> This file outlines the operational protocols, coding standards, and architectural decisions for this FastAPI project.
|
|
||||||
|
|
||||||
## 1. Agent Operational Protocols
|
|
||||||
|
|
||||||
### 🧠 Work Patterns (Plan-Act-Reflect)
|
|
||||||
* **Plan:** Before writing code, briefly outline your plan. Identify which files you will touch and what the side effects might be.
|
|
||||||
* **Act:** Execute the changes in small, atomic steps.
|
|
||||||
* **Reflect:** After coding, verify your work. Did you break existing tests? Did you add new tests?
|
|
||||||
|
|
||||||
### 🛡️ Git Discipline
|
|
||||||
* **NEVER commit to `main` or `master` directly.** Always create a feature branch: `feature/your-feature-name` or `fix/issue-description`.
|
|
||||||
* **Commit Messages:** Use the [Conventional Commits](https://www.conventionalcommits.org/) format.
|
|
||||||
* `feat: add user login endpoint`
|
|
||||||
* `fix: resolve database connection timeout`
|
|
||||||
* `refactor: split monolith dependency file`
|
|
||||||
* **Atomic Commits:** Keep commits small. One logical change = one commit.
|
|
||||||
|
|
||||||
### 📝 Changelog Maintenance
|
|
||||||
* **Update `CHANGELOG.md`** with every user-facing change.
|
|
||||||
* Format: `## [Unreleased] - YYYY-MM-DD` followed by `### Added`, `### Changed`, or `### Fixed`.
|
|
||||||
|
|
||||||
### 🚀 Release Flow
|
|
||||||
When changes are ready for deployment:
|
|
||||||
|
|
||||||
1. **Ask user if deploy cycle is desired **
|
|
||||||
|
|
||||||
2. **Update version** in `pyproject.toml`:
|
|
||||||
- Bug fixes: bump patch version (1.8.3 → 1.8.4)
|
|
||||||
- New features: bump minor version (1.8.4 → 1.9.0)
|
|
||||||
|
|
||||||
3. **Update CHANGELOG.md**:
|
|
||||||
- Move items from `[Unreleased]` to new version section
|
|
||||||
- Add release date: `## [1.8.4] - 2025-12-16`
|
|
||||||
|
|
||||||
4. **Commit and tag**:
|
|
||||||
```bash
|
|
||||||
git add -A
|
|
||||||
git commit -m "fix: description of changes"
|
|
||||||
git tag v1.8.4
|
|
||||||
git push origin main --tags
|
|
||||||
```
|
|
||||||
|
|
||||||
5. **CI/CD triggers automatically**:
|
|
||||||
- Gitea CI builds Docker image on new tag
|
|
||||||
- Watchtower pulls and deploys to production
|
|
||||||
- Verify deployment: `curl http://192.168.86.149:8000/health`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 🧪 Local Development Setup
|
|
||||||
* **Always test locally first** before committing and deploying. The build-deploy loop is slow.
|
|
||||||
* **Start the local server** with `./wakeup.sh` - logs are written to `logs/server.log` for easy tailing
|
|
||||||
* **Auto-reload**: The wakeup script runs uvicorn in reload mode - code changes are picked up automatically without restart (except for requirements.txt changes)
|
|
||||||
* **Test REST endpoints** against `http://localhost:8778` using curl or similar tools
|
|
||||||
* **Only deploy** when a phase or feature is complete and tested locally
|
|
||||||
* **Environment**: Copy `.env.example` to `.env` and configure for your local setup (Ollama, Redis, Neo4j, Qdrant, Wiki.js hosts)
|
|
||||||
* **Running tests**: Always use the venv explicitly to avoid environment mismatches:
|
|
||||||
```bash
|
|
||||||
.venv/bin/python -m pytest tests/ # All tests
|
|
||||||
.venv/bin/python -m pytest tests/ -v # Verbose output
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. FastAPI Architecture & Best Practices
|
|
||||||
*Reference: [FastAPI Best Practices](https://github.com/zhanymkanov/fastapi-best-practices)*
|
|
||||||
|
|
||||||
### 📂 Project Structure (Directory-based, NOT File-type based)
|
|
||||||
Do **not** group files by type (e.g., one huge `routers` folder). Group by **domain/module** inside a `src/` directory.
|
|
||||||
|
|
||||||
**Correct Structure:**
|
|
||||||
```text
|
|
||||||
src/
|
|
||||||
├── auth/
|
|
||||||
│ ├── router.py # Endpoints
|
|
||||||
│ ├── schemas.py # Pydantic models
|
|
||||||
│ ├── service.py # Business logic (CRUD, etc.)
|
|
||||||
│ ├── dependencies.py# Module-specific dependencies
|
|
||||||
│ └── config.py # Module-specific settings
|
|
||||||
├── posts/
|
|
||||||
│ ├── router.py
|
|
||||||
│ └── ...
|
|
||||||
└── main.py # App entry point
|
|
||||||
@@ -7,6 +7,76 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [1.9.2] - 2026-08-16
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- `/health` no longer reports the whole service unhealthy because one
|
||||||
|
dependency is merely slow. Its five probes (neo4j, qdrant, wikijs, searxng,
|
||||||
|
ollama) previously ran one after another with no timeout on neo4j or
|
||||||
|
qdrant, so a single hung dependency could block the response past the
|
||||||
|
container healthcheck's 10s timeout and get the container marked
|
||||||
|
unhealthy for a reason unrelated to its own liveness. Each probe is now
|
||||||
|
bounded at 2s and all five run concurrently, so one hanging dependency is
|
||||||
|
reported unhealthy on its own without holding up the others or the
|
||||||
|
response.
|
||||||
|
- Internal-only: the three probes `check_service_health()` also computes
|
||||||
|
(paperless, system_settings, scheduler) received the same bounding and
|
||||||
|
concurrency, for consistency and because they carried the identical
|
||||||
|
unbounded-hang risk internally. This has no visible effect today — none
|
||||||
|
of the three is currently returned by `/health` — but protects a future
|
||||||
|
caller that does read them.
|
||||||
|
|
||||||
|
## [1.9.1] - 2026-08-08
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The Wiki.js change listener now survives a database restart. It held a single
|
||||||
|
`LISTEN` connection with no supervision, so when the connection dropped it was
|
||||||
|
gone permanently while `running` stayed `True` — the service kept reporting
|
||||||
|
healthy and silently stopped indexing every page edit until someone restarted
|
||||||
|
the container. This happened for real on 2026-08-08 when `postgres-shared` was
|
||||||
|
redeployed. It now detects the drop via asyncpg's termination callback and
|
||||||
|
reconnects with bounded exponential backoff (1s doubling to a 60s cap),
|
||||||
|
retrying indefinitely because a database in maintenance does come back and
|
||||||
|
giving up would recreate the same silent deafness.
|
||||||
|
- `WikiChangeListener.running` is derived from the live connection instead of
|
||||||
|
being assigned once at startup, so it can no longer claim a subscription that
|
||||||
|
does not exist.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `/health` reports the change listener under `services.wiki_listener`
|
||||||
|
(`subscribed`, `reconnects`, `last_gap_seconds`). Nothing previously exposed
|
||||||
|
its state anywhere, which is why a dead listener went unnoticed. It is
|
||||||
|
deliberately excluded from the overall healthy/degraded verdict: it recovers on
|
||||||
|
its own, and flipping the container unhealthy for the duration of a database
|
||||||
|
outage would add a restart loop to an incident rather than information.
|
||||||
|
- On reconnect the listener logs the outage duration and warns that `NOTIFY`
|
||||||
|
events emitted during the gap were lost and cannot be replayed, pointing at
|
||||||
|
`POST /maintenance/integrity-check` to reconcile. A reconciliation pass is not
|
||||||
|
performed automatically — tenant attribution for a changed page is non-trivial
|
||||||
|
here, and guessing it wrong writes content into the wrong user's namespace.
|
||||||
|
|
||||||
|
## [1.9.0] - 2026-07-20
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- The Wiki.js integration buttons no longer embed an API key in the browser. The re-index and entity-link endpoints now authenticate via the NPM `/library-desk/` proxy (Authentik session for external users, LAN bypass for internal), verified by a trusted proxy marker header. The previously-embedded key was a full-privilege key served to every wiki visitor; it has been rotated out of service.
|
||||||
|
- `verify_api_key` now uses a constant-time comparison.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `static/wikijs-integration.js` calls library-desk same-origin (`/library-desk/...`) with `credentials: same-origin` and no `Authorization` header. Update the Wiki.js code-injection snippet to `<script src="/library-desk/static/wikijs-integration.js">`.
|
||||||
|
- Machine callers (the Scheduler) continue to use the Bearer API key against the container-network endpoints; only the two browser endpoints switched to proxy auth.
|
||||||
|
|
||||||
|
## [1.8.1] - 2026-07-19
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Container images are now pushed via the `git.schweitz.net` registry endpoint (the `.internal` registry domain is being retired); no change to the image name consumed by Watchtower.
|
||||||
|
- `.env.example` service URL defaults now use docker-dataplane container names (`wiki:3000`, `neo4j:7687`, `searxng:8080`, `paperless:8000`, `ollama:11434`) instead of host IP + published port, ahead of the Phase 4 port lockdown; AGENTS.md deploy health-check URL corrected from port 8000 to 8089.
|
||||||
|
|
||||||
## [1.8.0] - 2026-07-14
|
## [1.8.0] - 2026-07-14
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
@@ -1,17 +1,101 @@
|
|||||||
# Claude Code Instructions
|
# CLAUDE.md — Library Desk
|
||||||
|
|
||||||
**MANDATORY: Read AGENTS.md instead of this file.**
|
FastAPI service that ingests Wiki.js content and serves retrieval over it. Talks to
|
||||||
|
Postgres (Wiki.js DB + `LISTEN/NOTIFY`), Qdrant (vectors), Neo4j (graph), Ollama
|
||||||
|
(embeddings) and Redis (job manager). Deployed on tower-of-joy at **:8089**.
|
||||||
|
|
||||||
This project uses a unified configuration file for all LLM coding agents.
|
## Ports — these differ, deliberately
|
||||||
|
|
||||||
## Instructions
|
| | Port | How |
|
||||||
|
|---|---|---|
|
||||||
|
| Local dev | **8778** | `./wakeup.sh`, uvicorn reload mode, logs to `logs/server.log` |
|
||||||
|
| Production | **8089** | container; health at `http://192.168.86.149:8089/health` |
|
||||||
|
|
||||||
1. **Read and follow AGENTS.md** - All project guidelines are located there
|
Testing `localhost:8089` on the dev box hits the *container*, not your reload server.
|
||||||
2. **Do not modify this file** - Only update AGENTS.md
|
|
||||||
3. **Do not create or modify other agent-specific files** - Use AGENTS.md as the single source of truth
|
|
||||||
|
|
||||||
This approach ensures consistent behavior across all LLM coding agents without managing separate configuration files.
|
## Work tracking
|
||||||
|
|
||||||
---
|
Work lives in **pql**, not a markdown file. `TODO.md` was migrated and removed 2026-08-08.
|
||||||
|
|
||||||
If you need to update project guidelines, edit AGENTS.md, not this file.
|
```bash
|
||||||
|
pql ticket list # open work
|
||||||
|
pql plan whatsnext # next unblocked item, with context
|
||||||
|
pql ticket show T-1 # detail
|
||||||
|
pql ticket new bug "title" # file new work
|
||||||
|
```
|
||||||
|
|
||||||
|
Tickets travel with the repo — `.pql/changelog/` is committed and replayed by the git
|
||||||
|
hooks; the databases are ignored and rebuildable with `pql plan rebuild`. Do not add a
|
||||||
|
TODO section to a markdown file.
|
||||||
|
|
||||||
|
## Known live defect
|
||||||
|
|
||||||
|
**Every wiki page change is ingested twice** (T-1, proven against v1.9.1). The Dockerfile
|
||||||
|
runs `uvicorn --workers 2`, `startup_event` creates a `WikiChangeListener` per worker, and
|
||||||
|
Postgres delivers `NOTIFY` to *every* listening session. The debounce in
|
||||||
|
`wiki_change_listener.py` is an in-process dict and cannot dedupe across workers.
|
||||||
|
|
||||||
|
Relevant when touching ingestion: duplicate work reaches Ollama, Qdrant and Neo4j. Whether
|
||||||
|
it merely wastes GPU or actually corrupts state is **not yet established** — check before
|
||||||
|
assuming either.
|
||||||
|
|
||||||
|
## Working here
|
||||||
|
|
||||||
|
**Plan, act, reflect.** Outline which files you will touch and the side effects before
|
||||||
|
writing. Change in small atomic steps. Afterwards, verify: did existing tests break, and
|
||||||
|
does the new behaviour have a test?
|
||||||
|
|
||||||
|
**Test locally first — the build-deploy loop is slow.** `./wakeup.sh` auto-reloads on code
|
||||||
|
changes (not on `requirements.txt` changes). Deploy only when a feature is complete and
|
||||||
|
tested.
|
||||||
|
|
||||||
|
Run tests through the venv explicitly, to avoid environment mismatch:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python -m pytest tests/
|
||||||
|
```
|
||||||
|
|
||||||
|
Copy `.env.example` to `.env` and configure Ollama, Redis, Neo4j, Qdrant and Wiki.js hosts.
|
||||||
|
|
||||||
|
## Git
|
||||||
|
|
||||||
|
- **History is linear — no merge commits.** Work on `main`, or a short-lived branch that is
|
||||||
|
fast-forwarded and deleted. (A feature-branch mandate was retired 2026-08-08 to match the
|
||||||
|
workspace convention, which now has no per-repo exceptions.)
|
||||||
|
- **Conventional Commits**: `feat:`, `fix:`, `refactor:`, `docs:`, `chore:`.
|
||||||
|
- **Atomic commits** — one logical change each.
|
||||||
|
- **Stage explicitly. Never `git add -A`** — it is denied by policy, and it sweeps in
|
||||||
|
whatever else is dirty, including secrets.
|
||||||
|
- Update `CHANGELOG.md` with every user-facing change, under `[Unreleased]` in `Added` /
|
||||||
|
`Changed` / `Fixed`.
|
||||||
|
|
||||||
|
## Releasing
|
||||||
|
|
||||||
|
Ask whether a deploy is wanted first — it is not automatic.
|
||||||
|
|
||||||
|
1. Bump the version in `pyproject.toml` (patch for fixes, minor for features).
|
||||||
|
2. Move `[Unreleased]` entries into a dated version section in `CHANGELOG.md`.
|
||||||
|
3. Stage the changed files by name, commit, tag `vX.Y.Z`, `git push origin main --tags`.
|
||||||
|
4. Gitea CI builds the image on the tag; Watchtower deploys it.
|
||||||
|
5. Verify: `curl http://192.168.86.149:8089/health`.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
Group by **domain, not by file type**. A single large `routers/` folder is the thing to
|
||||||
|
avoid.
|
||||||
|
|
||||||
|
```text
|
||||||
|
src/
|
||||||
|
├── auth/
|
||||||
|
│ ├── router.py # endpoints
|
||||||
|
│ ├── schemas.py # pydantic models
|
||||||
|
│ ├── service.py # business logic
|
||||||
|
│ ├── dependencies.py # module-scoped dependencies
|
||||||
|
│ └── config.py # module-scoped settings
|
||||||
|
└── main.py # app entry point
|
||||||
|
```
|
||||||
|
|
||||||
|
Reference: [FastAPI best practices](https://github.com/zhanymkanov/fastapi-best-practices).
|
||||||
|
|
||||||
|
The live contract is always `http://localhost:8089/openapi.json` (70 paths) — generated from
|
||||||
|
running code, so it cannot drift the way this file can.
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# library-desk — the repo's command surface (D-27).
|
||||||
|
#
|
||||||
|
# Note the port split, which is the trap this repo's CLAUDE.md leads with:
|
||||||
|
# `make run` serves on 8778 with reload, while 8089 is the *container*. Testing
|
||||||
|
# localhost:8089 on this box hits the deployed service, not your dev server.
|
||||||
|
#
|
||||||
|
# Paths resolve here (D-10): `python3` is 3.8 on this host and a bare `pytest`
|
||||||
|
# resolves only in a login shell, so both go through the venv explicitly.
|
||||||
|
|
||||||
|
VENV := $(CURDIR)/.venv
|
||||||
|
PYTHON ?= python3.12
|
||||||
|
|
||||||
|
.DEFAULT_GOAL := help
|
||||||
|
|
||||||
|
.PHONY: help
|
||||||
|
help: ## Show this help
|
||||||
|
@grep -hE '^[a-z][a-z0-9_-]*:.*?## ' $(MAKEFILE_LIST) \
|
||||||
|
| awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}'
|
||||||
|
|
||||||
|
.PHONY: setup
|
||||||
|
setup: ## Create the venv, install dependencies, and prove the result works
|
||||||
|
$(PYTHON) -m venv .venv
|
||||||
|
# requirements-dev.txt pulls in requirements.txt via -r, so this is still
|
||||||
|
# the full runtime set plus pytest/ruff. Installing only requirements.txt
|
||||||
|
# here (as this line originally did) left `make test` and `make lint`
|
||||||
|
# unreachable from a clean `make setup` since the Makefile's introduction
|
||||||
|
# in b9f55cb — pytest was never installed by setup at all. Found by the
|
||||||
|
# check below failing on its first clean-tree run; fixed in the same
|
||||||
|
# commit rather than filed separately, since the new check cannot pass
|
||||||
|
# honestly against the old line.
|
||||||
|
$(VENV)/bin/pip install -r requirements-dev.txt
|
||||||
|
# Exit 0 from pip install is not evidence (D-24) — it is the same exit code
|
||||||
|
# whether requirements.txt matches what's on disk or a dependency silently
|
||||||
|
# failed to install. Two cheap checks, for two different drift modes:
|
||||||
|
# pip check - installed packages satisfy each other's declared
|
||||||
|
# version constraints (a stale/partial install).
|
||||||
|
# pytest --collect-only - every test module actually imports, which
|
||||||
|
# walks the full src/ import graph and is exactly
|
||||||
|
# what catches a *missing* declared dependency
|
||||||
|
# (core-api's sqlalchemy case in T-47). It builds
|
||||||
|
# no client and opens no socket - dependencies.py
|
||||||
|
# wraps client construction in lru_cache getters,
|
||||||
|
# never called at collection time - so this needs
|
||||||
|
# none of the five backing services running.
|
||||||
|
$(VENV)/bin/pip check
|
||||||
|
$(VENV)/bin/python -m pytest tests/ --collect-only -q
|
||||||
|
|
||||||
|
.PHONY: run
|
||||||
|
run: ## Dev server on :8778 with reload (8089 is the container, not this)
|
||||||
|
./wakeup.sh
|
||||||
|
|
||||||
|
.PHONY: test
|
||||||
|
test: ## Run the test suite (no network needed — live tests are marked and self-skip)
|
||||||
|
@test -x $(VENV)/bin/python || { echo "FAIL — no venv; run: make setup"; exit 69; }
|
||||||
|
$(VENV)/bin/python -m pytest tests/
|
||||||
|
|
||||||
|
# The `integration` marker and its RUN_INTEGRATION_TESTS/TEST_TENANT gate already
|
||||||
|
# lived in tests/conftest.py before T-55 (see test_integration.py,
|
||||||
|
# test_tenant_isolation_live.py, test_quality_report_live.py, and the
|
||||||
|
# TestWikiChangeListenerIntegration class) — `make test` never ran them because
|
||||||
|
# `pytest_collection_modifyitems` skips anything carrying the marker unless
|
||||||
|
# RUN_INTEGRATION_TESTS=1. That gate made them silent under `make test`, but
|
||||||
|
# nothing ran them WITH the flag set either, so "runnable" had never been
|
||||||
|
# reasserted. This target is that home (D-26): it sets the flag, selects the
|
||||||
|
# marker, and — the part that matters — fails loudly if selection ever drops to
|
||||||
|
# zero, since a target that passes by collecting nothing is worse than one that
|
||||||
|
# needs a network (T-55).
|
||||||
|
.PHONY: test-integration
|
||||||
|
test-integration: ## Live tests against neo4j/qdrant/wikijs/searxng/ollama (needs network + services)
|
||||||
|
@test -x $(VENV)/bin/python || { echo "FAIL — no venv; run: make setup"; exit 69; }
|
||||||
|
@$(VENV)/bin/python -m pytest tests/ -m integration --collect-only -q >/dev/null 2>&1; \
|
||||||
|
rc=$$?; \
|
||||||
|
if [ "$$rc" = "5" ]; then \
|
||||||
|
echo "FAIL test-integration — selected 0 tests (marker renamed, moved, or lost — this is a defect, not a pass)"; \
|
||||||
|
exit 1; \
|
||||||
|
elif [ "$$rc" != "0" ]; then \
|
||||||
|
echo "FAIL test-integration — collection errored (rc=$$rc)"; \
|
||||||
|
exit $$rc; \
|
||||||
|
fi
|
||||||
|
RUN_INTEGRATION_TESTS=1 $(VENV)/bin/python -m pytest tests/ -m integration -v
|
||||||
|
|
||||||
|
.PHONY: lint
|
||||||
|
lint: ## ruff check over the sources
|
||||||
|
@test -x $(VENV)/bin/ruff || { echo "FAIL — ruff not installed; run: make setup"; exit 69; }
|
||||||
|
$(VENV)/bin/ruff check src tests
|
||||||
|
|
||||||
|
# git hands a hook a non-login shell, which never sees ~/.local/bin — where
|
||||||
|
# gitleaks lands. Without this the scan reports "not installed" on every push,
|
||||||
|
# which is a check that fails open (D-24).
|
||||||
|
export PATH := $(HOME)/.local/bin:/usr/local/bin:$(PATH)
|
||||||
|
|
||||||
|
.PHONY: secrets
|
||||||
|
secrets: ## Scan the commits about to be pushed for credentials
|
||||||
|
@ci/secrets.sh
|
||||||
|
|
||||||
|
# The call surface is identical in every repo; what it runs is not.
|
||||||
|
#
|
||||||
|
# `secrets` runs first, deliberately: it is the only failure here that cannot be
|
||||||
|
# undone by fixing it afterwards. A failed lint costs another commit; a pushed
|
||||||
|
# credential is cached and indexed whether or not it is later deleted.
|
||||||
|
#
|
||||||
|
# Some of these fail today, and are left wired anyway. The state was measured
|
||||||
|
# once and written down in T-56 rather than being worked around here — a gate
|
||||||
|
# quietly narrowed to what already passes is a gate that reports success for
|
||||||
|
# doing nothing, which is the failure this workspace keeps rediscovering.
|
||||||
|
.PHONY: pre-push
|
||||||
|
pre-push: secrets lint ## Everything the pre-push hook runs
|
||||||
|
@echo " -- not gated here yet: test (T-56)"
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
# TODO
|
|
||||||
|
|
||||||
Outstanding work items for Library Desk.
|
|
||||||
|
|
||||||
## Stub Endpoints to Implement
|
|
||||||
|
|
||||||
The following endpoints in `src/main.py` return stub responses and need real implementations:
|
|
||||||
|
|
||||||
### Ingestion Status Endpoints
|
|
||||||
|
|
||||||
#### `POST /ingest/check-updates`
|
|
||||||
Check which documents need updating based on content hashes. Used by Scheduler to determine what changed since last sync.
|
|
||||||
|
|
||||||
**Implementation needed:**
|
|
||||||
1. Query existing documents by path
|
|
||||||
2. Compare content hashes
|
|
||||||
3. Return list of updates needed
|
|
||||||
|
|
||||||
#### `GET /ingest/status/{document_id}`
|
|
||||||
Get processing status for a document.
|
|
||||||
|
|
||||||
**Implementation needed:**
|
|
||||||
- Status tracking system (Redis or database)
|
|
||||||
- Track ingestion progress per document
|
|
||||||
|
|
||||||
#### `GET /ingest/repo-status/{repository}`
|
|
||||||
Get indexing status for an entire repository.
|
|
||||||
|
|
||||||
**Implementation needed:**
|
|
||||||
- Repository-level statistics
|
|
||||||
- Track which documents from a repo are indexed
|
|
||||||
|
|
||||||
### Deduplication
|
|
||||||
|
|
||||||
#### `POST /deduplicate/check`
|
|
||||||
Check for duplicate or highly similar documents using vector similarity and graph analysis.
|
|
||||||
|
|
||||||
**Implementation needed:**
|
|
||||||
1. Get document embedding from Qdrant
|
|
||||||
2. Find similar vectors above threshold
|
|
||||||
3. Check graph relationships
|
|
||||||
4. Return candidates with similarity scores
|
|
||||||
|
|
||||||
Executable
+50
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Secret scan over the commits about to be pushed.
|
||||||
|
#
|
||||||
|
# Lives here rather than inside .githooks/pre-push so it can be read, run by
|
||||||
|
# hand (`make secrets`), and changed under review. A hook is a trigger; it is
|
||||||
|
# not a home for logic. Identical in every repo in this workspace (D-27).
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(git rev-parse --show-toplevel)"
|
||||||
|
|
||||||
|
# A non-login shell — which is what git gives a hook — skips /etc/profile.d
|
||||||
|
# and never sees ~/.local/bin, where the gitleaks release tarball lands.
|
||||||
|
# Without this the scan reports "not installed" on every push.
|
||||||
|
[ -d "$HOME/.local/bin" ] && PATH="$HOME/.local/bin:$PATH"
|
||||||
|
|
||||||
|
if ! command -v gitleaks >/dev/null 2>&1; then
|
||||||
|
echo "FAIL secrets — gitleaks not installed, so this check would be a no-op pretending to pass." >&2
|
||||||
|
echo " https://github.com/gitleaks/gitleaks/releases → ~/.local/bin/gitleaks" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Scan the outgoing range, not full history. History here carries findings
|
||||||
|
# that are settled — test fixtures and vendored third-party code — and a gate
|
||||||
|
# that fails on something unfixable gets bypassed within a week. What matters
|
||||||
|
# is what is about to leave this machine.
|
||||||
|
if upstream=$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null); then
|
||||||
|
range="$upstream..HEAD"
|
||||||
|
elif git rev-parse --verify --quiet origin/main >/dev/null; then
|
||||||
|
range="origin/main..HEAD"
|
||||||
|
else
|
||||||
|
range=""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$range" ]; then
|
||||||
|
gitleaks dir . --redact --no-banner --exit-code 1 || {
|
||||||
|
echo "FAIL secrets — gitleaks found a credential in the working tree." >&2; exit 1; }
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ -n "$(git log --oneline "$range" 2>/dev/null)" ] || exit 0
|
||||||
|
|
||||||
|
gitleaks git . --log-opts="$range" --redact --no-banner --exit-code 1 >/dev/null 2>&1 || {
|
||||||
|
echo "FAIL secrets — gitleaks found a credential in the commits being pushed." >&2
|
||||||
|
echo " inspect (values redacted): gitleaks git . --log-opts=\"$range\" --redact" >&2
|
||||||
|
echo " then remove and rotate it, or suppress deliberately:" >&2
|
||||||
|
echo " inline '# gitleaks:allow <reason>'" >&2
|
||||||
|
echo " or add the fingerprint to .gitleaksignore WITH a reason" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
echo " ok secrets"
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# Decisions, Questions, Rejected
|
||||||
|
|
||||||
|
This directory holds structured planning records that pql parses
|
||||||
|
into pql.db. Each record is a `### [DQR]-N: Title` heading inside
|
||||||
|
a markdown file. Files live in three per-type subdirectories:
|
||||||
|
|
||||||
|
- `decisions/<domain>.md` — confirmed design decisions
|
||||||
|
- `questions/<domain>.md` — open questions that may resolve into
|
||||||
|
decisions or rejected proposals
|
||||||
|
- `rejected/<domain>.md` — rejected proposals (kept for the audit
|
||||||
|
trail)
|
||||||
|
|
||||||
|
The parser infers domain from the filename stem and record type
|
||||||
|
from the parent subdirectory.
|
||||||
|
|
||||||
|
D-records that propose implementation work link to `initiative`-type
|
||||||
|
tickets via `decision_ref`. Run `pql decisions show <id>
|
||||||
|
--with-tickets` to inspect implementation status.
|
||||||
|
|
||||||
|
## Recommended domains
|
||||||
|
|
||||||
|
Start with this canonical set; create files as records land in
|
||||||
|
each domain:
|
||||||
|
|
||||||
|
- **architecture** — structural commitments (storage, layering,
|
||||||
|
languages, libraries)
|
||||||
|
- **process** — team workflow (commits, branches, releases, reviews)
|
||||||
|
- **design** — user-facing surface (UX, UI, public APIs)
|
||||||
|
- **coding-conventions** — team-internal code shape (style, lint,
|
||||||
|
file layout)
|
||||||
|
- **testing** — quality strategy (coverage, layers, gates)
|
||||||
|
|
||||||
|
You might also want, project-permitting:
|
||||||
|
|
||||||
|
- `accessibility` — if you ship user-facing software
|
||||||
|
- `security` — if you handle user data or network surfaces
|
||||||
|
- `licensing` — if you release open-source or commercial
|
||||||
|
- `documentation` — if user-docs are non-trivial
|
||||||
|
- `deployment` — if shipping is non-trivial
|
||||||
|
- `performance` — if you have perf budgets / SLOs
|
||||||
|
|
||||||
|
<!-- pql:records (auto-generated; do not edit manually) -->
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
- _(none)_
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
- _(none)_
|
||||||
|
|
||||||
|
## Rejected
|
||||||
|
|
||||||
|
- _(none)_
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "library-desk"
|
name = "library-desk"
|
||||||
version = "1.8.0"
|
version = "1.9.2"
|
||||||
description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation"
|
description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ Source selection is driven by user preferences in the settings database.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from .base import NewsProvider, NewsItem, NewsFeed
|
from .base import NewsProvider, NewsItem, NewsFeed
|
||||||
from .nos import NOSProvider
|
from .nos import NOSProvider
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Provides async Neo4j operations with:
|
|||||||
- Automatic retry on transient failures
|
- Automatic retry on transient failures
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from neo4j import AsyncGraphDatabase, AsyncDriver, AsyncSession, READ_ACCESS
|
from neo4j import AsyncGraphDatabase, AsyncDriver, READ_ACCESS
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
|||||||
@@ -529,7 +529,7 @@ class QdrantClientWrapper:
|
|||||||
query_filter = Filter(must=conditions)
|
query_filter = Filter(must=conditions)
|
||||||
|
|
||||||
# Delete points
|
# Delete points
|
||||||
result = await self.client.delete(
|
await self.client.delete(
|
||||||
collection_name=collection_name,
|
collection_name=collection_name,
|
||||||
points_selector=query_filter
|
points_selector=query_filter
|
||||||
)
|
)
|
||||||
|
|||||||
+238
-81
@@ -8,13 +8,19 @@ Provides FastAPI dependencies for service clients with:
|
|||||||
- Type aliases for clean endpoint signatures
|
- Type aliases for clean endpoint signatures
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import Annotated
|
from typing import TYPE_CHECKING, Annotated
|
||||||
from fastapi import Depends, HTTPException, Query
|
from fastapi import Depends, HTTPException, Query
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
|
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
from fastapi import Request, Security
|
||||||
|
from fastapi.security import HTTPBearer
|
||||||
|
|
||||||
from src.config import Settings, get_settings
|
from src.config import Settings, get_settings
|
||||||
from src.clients.neo4j_client import Neo4jClient
|
from src.clients.neo4j_client import Neo4jClient
|
||||||
from src.clients.qdrant_client import QdrantClientWrapper
|
from src.clients.qdrant_client import QdrantClientWrapper
|
||||||
@@ -560,10 +566,150 @@ async def shutdown_clients():
|
|||||||
logger.info("Service clients shutdown complete")
|
logger.info("Service clients shutdown complete")
|
||||||
|
|
||||||
|
|
||||||
|
# Per-probe timeout for the concurrent health checks below, in seconds.
|
||||||
|
#
|
||||||
|
# Bounded well under the container healthcheck's 10s timeout (see Dockerfile).
|
||||||
|
# Because every probe runs concurrently under one asyncio.gather, the wall time
|
||||||
|
# is one bound rather than the sum, so adding probes does not erode the margin —
|
||||||
|
# all eight can hang and the call still returns in ~2s.
|
||||||
|
#
|
||||||
|
# A probe with no bound of its own falls back to its client's default: 30s for
|
||||||
|
# the neo4j and qdrant drivers, 30s for the scheduler client, and none at all
|
||||||
|
# for system settings. Each is past the 10s budget on its own, which is what let
|
||||||
|
# a hung dependency — not a failing one — flip the container unhealthy.
|
||||||
|
HEALTH_PROBE_TIMEOUT = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_neo4j() -> bool:
|
||||||
|
"""Neo4j connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
neo4j = get_neo4j_client()
|
||||||
|
# Simple query to check connectivity
|
||||||
|
await neo4j.execute_query("RETURN 1 as test", {})
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Neo4j health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_qdrant() -> bool:
|
||||||
|
"""Qdrant connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
qdrant = get_qdrant_client()
|
||||||
|
# Check if we can list collections
|
||||||
|
await qdrant.client.get_collections()
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Qdrant health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_wikijs() -> bool:
|
||||||
|
"""Wiki.js connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
wikijs = get_wikijs_client()
|
||||||
|
# Try a simple query (list pages with limit 1)
|
||||||
|
await wikijs.list_pages(limit=1)
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Wiki.js health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_searxng() -> bool:
|
||||||
|
"""SearXNG connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
searxng = get_searxng_client()
|
||||||
|
# Just check if service is up (no actual search)
|
||||||
|
return await searxng.health_check()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"SearXNG health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_ollama() -> bool:
|
||||||
|
"""Ollama connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
ollama = get_ollama_client()
|
||||||
|
return await ollama.health_check()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Ollama health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_paperless() -> bool:
|
||||||
|
"""Paperless-ngx connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
paperless = get_paperless_client()
|
||||||
|
return await paperless.health_check()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Paperless health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_system_settings() -> bool:
|
||||||
|
"""System settings database connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
settings_client = get_settings_client()
|
||||||
|
return await settings_client.health_check()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"System settings health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_scheduler() -> bool:
|
||||||
|
"""Scheduler connectivity probe. Returns True if healthy."""
|
||||||
|
try:
|
||||||
|
scheduler = get_scheduler_client()
|
||||||
|
return await scheduler.health_check()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Scheduler health check failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _bounded_probe(coro) -> bool:
|
||||||
|
"""
|
||||||
|
Run a probe coroutine bounded by HEALTH_PROBE_TIMEOUT.
|
||||||
|
|
||||||
|
A probe that times out is reported unhealthy, the same as one that
|
||||||
|
raises. Wrapping happens here rather than in each _probe_* function so
|
||||||
|
the bound applies uniformly regardless of whether the underlying client
|
||||||
|
has its own (looser, or absent) timeout.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await asyncio.wait_for(coro, timeout=HEALTH_PROBE_TIMEOUT)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
logger.error(f"Health probe timed out after {HEALTH_PROBE_TIMEOUT}s")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
async def check_service_health() -> dict:
|
async def check_service_health() -> dict:
|
||||||
"""
|
"""
|
||||||
Check health of all service clients.
|
Check health of all service clients.
|
||||||
|
|
||||||
|
All eight probes (neo4j, qdrant, wikijs, searxng, ollama, paperless,
|
||||||
|
system_settings, scheduler) run concurrently in one gather, each bounded
|
||||||
|
at HEALTH_PROBE_TIMEOUT, so a single hung dependency cannot block the
|
||||||
|
others or push the endpoint past the container healthcheck's timeout.
|
||||||
|
return_exceptions=True means one probe raising cannot cancel its
|
||||||
|
siblings.
|
||||||
|
|
||||||
|
paperless and system_settings are three-state: None means "not
|
||||||
|
configured" (no probe is run at all — a probe never joins the gather
|
||||||
|
for a service that has no credentials to check), False means configured
|
||||||
|
but unreachable/unhealthy (including a timeout), True means healthy.
|
||||||
|
Collapsing "not configured" into "unhealthy" would be a different claim
|
||||||
|
than the one this function is making, so that decision is made before
|
||||||
|
the gather rather than by feeding an unconfigured probe through the
|
||||||
|
same bool-returning bound as the rest.
|
||||||
|
|
||||||
|
Only neo4j, qdrant, wikijs, searxng and ollama are read by /health
|
||||||
|
(src/main.py) — paperless, system_settings and scheduler are computed
|
||||||
|
here but not currently surfaced by any caller (checked: the only two
|
||||||
|
callers are src/main.py and tests/test_integration.py, and the test
|
||||||
|
asserts only the five). Bounded rather than removed, since bounding
|
||||||
|
cannot break a hypothetical consumer and deleting could.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with health status of each service:
|
Dictionary with health status of each service:
|
||||||
{
|
{
|
||||||
@@ -571,7 +717,10 @@ async def check_service_health() -> dict:
|
|||||||
"qdrant": bool,
|
"qdrant": bool,
|
||||||
"wikijs": bool,
|
"wikijs": bool,
|
||||||
"searxng": bool,
|
"searxng": bool,
|
||||||
"ollama": bool
|
"ollama": bool,
|
||||||
|
"paperless": bool | None,
|
||||||
|
"system_settings": bool | None,
|
||||||
|
"scheduler": bool
|
||||||
}
|
}
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
@@ -580,85 +729,47 @@ async def check_service_health() -> dict:
|
|||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
health = {}
|
health = {}
|
||||||
|
|
||||||
# Neo4j
|
|
||||||
try:
|
|
||||||
neo4j = get_neo4j_client()
|
|
||||||
# Simple query to check connectivity
|
|
||||||
await neo4j.execute_query("RETURN 1 as test", {})
|
|
||||||
health["neo4j"] = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Neo4j health check failed: {e}")
|
|
||||||
health["neo4j"] = False
|
|
||||||
|
|
||||||
# Qdrant
|
|
||||||
try:
|
|
||||||
qdrant = get_qdrant_client()
|
|
||||||
# Check if we can list collections
|
|
||||||
await qdrant.client.get_collections()
|
|
||||||
health["qdrant"] = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Qdrant health check failed: {e}")
|
|
||||||
health["qdrant"] = False
|
|
||||||
|
|
||||||
# Wiki.js
|
|
||||||
try:
|
|
||||||
wikijs = get_wikijs_client()
|
|
||||||
# Try a simple query (list pages with limit 1)
|
|
||||||
await wikijs.list_pages(limit=1)
|
|
||||||
health["wikijs"] = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Wiki.js health check failed: {e}")
|
|
||||||
health["wikijs"] = False
|
|
||||||
|
|
||||||
# SearXNG
|
|
||||||
try:
|
|
||||||
searxng = get_searxng_client()
|
|
||||||
# Just check if service is up (no actual search)
|
|
||||||
health["searxng"] = await searxng.health_check()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"SearXNG health check failed: {e}")
|
|
||||||
health["searxng"] = False
|
|
||||||
|
|
||||||
# Ollama
|
|
||||||
try:
|
|
||||||
ollama = get_ollama_client()
|
|
||||||
is_healthy = await ollama.health_check()
|
|
||||||
health["ollama"] = is_healthy
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Ollama health check failed: {e}")
|
|
||||||
health["ollama"] = False
|
|
||||||
|
|
||||||
# Paperless-ngx
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
|
|
||||||
|
# Build the probe list dynamically: paperless and system_settings only
|
||||||
|
# join it when configured, so an unconfigured service is never bounded,
|
||||||
|
# timed out, or reported False — it is set to None directly, below.
|
||||||
|
probe_names = ["neo4j", "qdrant", "wikijs", "searxng", "ollama"]
|
||||||
|
probes = [
|
||||||
|
_bounded_probe(_probe_neo4j()),
|
||||||
|
_bounded_probe(_probe_qdrant()),
|
||||||
|
_bounded_probe(_probe_wikijs()),
|
||||||
|
_bounded_probe(_probe_searxng()),
|
||||||
|
_bounded_probe(_probe_ollama()),
|
||||||
|
]
|
||||||
|
|
||||||
if settings.paperless_token:
|
if settings.paperless_token:
|
||||||
try:
|
probe_names.append("paperless")
|
||||||
paperless = get_paperless_client()
|
probes.append(_bounded_probe(_probe_paperless()))
|
||||||
health["paperless"] = await paperless.health_check()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Paperless health check failed: {e}")
|
|
||||||
health["paperless"] = False
|
|
||||||
else:
|
else:
|
||||||
health["paperless"] = None # Not configured
|
health["paperless"] = None # Not configured
|
||||||
|
|
||||||
# System Settings database
|
|
||||||
if settings.system_settings_password:
|
if settings.system_settings_password:
|
||||||
try:
|
probe_names.append("system_settings")
|
||||||
settings_client = get_settings_client()
|
probes.append(_bounded_probe(_probe_system_settings()))
|
||||||
health["system_settings"] = await settings_client.health_check()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"System settings health check failed: {e}")
|
|
||||||
health["system_settings"] = False
|
|
||||||
else:
|
else:
|
||||||
health["system_settings"] = None # Not configured
|
health["system_settings"] = None # Not configured
|
||||||
|
|
||||||
# Scheduler
|
# Scheduler has no config gate - it always runs.
|
||||||
try:
|
probe_names.append("scheduler")
|
||||||
scheduler = get_scheduler_client()
|
probes.append(_bounded_probe(_probe_scheduler()))
|
||||||
health["scheduler"] = await scheduler.health_check()
|
|
||||||
except Exception as e:
|
results = await asyncio.gather(*probes, return_exceptions=True)
|
||||||
logger.error(f"Scheduler health check failed: {e}")
|
|
||||||
health["scheduler"] = False
|
# _bounded_probe already catches everything from its own probe, but
|
||||||
|
# return_exceptions=True also guards against a bug in _bounded_probe
|
||||||
|
# itself surfacing as an unhandled exception here.
|
||||||
|
for name, result in zip(probe_names, results):
|
||||||
|
if isinstance(result, BaseException):
|
||||||
|
logger.error(f"{name} health check raised unexpectedly: {result}")
|
||||||
|
health[name] = False
|
||||||
|
else:
|
||||||
|
health[name] = result
|
||||||
|
|
||||||
return health
|
return health
|
||||||
|
|
||||||
@@ -764,10 +875,6 @@ def get_volatile_cache_service() -> "VolatileCacheService":
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Authentication
|
|
||||||
from fastapi import Security, HTTPException
|
|
||||||
from fastapi.security import HTTPBearer
|
|
||||||
|
|
||||||
security = HTTPBearer()
|
security = HTTPBearer()
|
||||||
|
|
||||||
|
|
||||||
@@ -788,7 +895,7 @@ async def verify_api_key(
|
|||||||
Raises:
|
Raises:
|
||||||
HTTPException: If API key is invalid
|
HTTPException: If API key is invalid
|
||||||
"""
|
"""
|
||||||
if credentials.credentials != settings.library_api_key:
|
if not secrets.compare_digest(credentials.credentials, settings.library_api_key):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=403,
|
status_code=403,
|
||||||
detail="Invalid API key"
|
detail="Invalid API key"
|
||||||
@@ -796,10 +903,60 @@ async def verify_api_key(
|
|||||||
return credentials.credentials
|
return credentials.credentials
|
||||||
|
|
||||||
|
|
||||||
# Service type aliases for FastAPI endpoint dependencies
|
# Header set by NPM only on the authenticated /library-desk/ proxy location.
|
||||||
# These are defined after the factory functions
|
# library-desk is bound to loopback (127.0.0.1:8089), so NPM is the only path
|
||||||
from src.services.vector_service import VectorService
|
# that can reach it and set this header — a client cannot forge it. NPM also
|
||||||
from src.services.graph_service import GraphService
|
# overwrites any client-supplied value via proxy_set_header.
|
||||||
|
_PROXY_MARKER_HEADER = "x-library-desk-proxy"
|
||||||
|
|
||||||
|
|
||||||
|
async def verify_browser_request(
|
||||||
|
request: Request,
|
||||||
|
settings: SettingsDep,
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Auth for browser-facing endpoints (the Wiki.js integration buttons).
|
||||||
|
|
||||||
|
Accepts the request when it arrives through the authenticated NPM proxy
|
||||||
|
location (Authentik session for external users, or the LAN bypass for
|
||||||
|
internal ones) — identified by the trusted proxy marker header. No secret
|
||||||
|
is carried in the browser. Machine callers may still authenticate with the
|
||||||
|
Bearer API key. Returns the acting user's identity.
|
||||||
|
"""
|
||||||
|
if request.headers.get(_PROXY_MARKER_HEADER) == "1":
|
||||||
|
# Authentik injects the identity for externally-authenticated users;
|
||||||
|
# on the LAN bypass these are empty and the endpoint falls back to the
|
||||||
|
# user supplied in the request body.
|
||||||
|
return request.headers.get("x-authentik-email") or "lan"
|
||||||
|
|
||||||
|
# Fallback: server-to-server Bearer API key.
|
||||||
|
auth = request.headers.get("authorization", "")
|
||||||
|
if auth.startswith("Bearer ") and secrets.compare_digest(
|
||||||
|
auth[len("Bearer "):], settings.library_api_key
|
||||||
|
):
|
||||||
|
return auth[len("Bearer "):]
|
||||||
|
|
||||||
|
raise HTTPException(status_code=401, detail="Unauthenticated")
|
||||||
|
|
||||||
|
|
||||||
|
# Service type aliases for FastAPI endpoint dependencies.
|
||||||
|
# Deliberately imported here rather than at the top: these modules import back
|
||||||
|
# into this one, so a module-level import would cycle. The factory functions
|
||||||
|
# above must exist before they are pulled in.
|
||||||
|
from src.services.vector_service import VectorService # noqa: E402
|
||||||
|
from src.services.graph_service import GraphService # noqa: E402
|
||||||
|
|
||||||
|
# Imported for annotations only. The real imports live inside the functions
|
||||||
|
# that use them, to break an import cycle; a quoted annotation is never
|
||||||
|
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||||
|
# block costs nothing at import time and makes them resolvable again.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from src.services.consolidation_service import ConsolidationService
|
||||||
|
from src.services.hybrid_rag_service import HybridRAGService
|
||||||
|
from src.services.ingestion_service import IngestionService
|
||||||
|
from src.services.rag_search_service import RAGSearchService
|
||||||
|
from src.services.volatile_service import VolatileCacheService
|
||||||
|
from src.services.wiki_service import WikiService
|
||||||
|
|
||||||
VectorServiceDep = Annotated[VectorService, Depends(get_vector_service)]
|
VectorServiceDep = Annotated[VectorService, Depends(get_vector_service)]
|
||||||
GraphServiceDep = Annotated[GraphService, Depends(get_graph_service)]
|
GraphServiceDep = Annotated[GraphService, Depends(get_graph_service)]
|
||||||
|
|||||||
+18
-3
@@ -55,8 +55,9 @@ app.add_middleware(
|
|||||||
allow_headers=["*"],
|
allow_headers=["*"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Register routers
|
# Register routers. Imported after the app and middleware exist, because the
|
||||||
from src.routers import (
|
# routers import dependencies that expect a configured app.
|
||||||
|
from src.routers import ( # noqa: E402
|
||||||
wiki, tools, graph, vector, hybrid_rag, consolidation,
|
wiki, tools, graph, vector, hybrid_rag, consolidation,
|
||||||
ingestion, entity_linking, webhooks, rag_search, content,
|
ingestion, entity_linking, webhooks, rag_search, content,
|
||||||
maintenance, volatile, documents
|
maintenance, volatile, documents
|
||||||
@@ -123,10 +124,19 @@ async def health(settings: Settings = Depends(get_settings)) -> HealthResponse:
|
|||||||
# Check service connectivity
|
# Check service connectivity
|
||||||
service_health = await check_service_health()
|
service_health = await check_service_health()
|
||||||
|
|
||||||
# Overall status is healthy if at least Neo4j and Qdrant are up
|
# Overall status is healthy if at least Neo4j and Qdrant are up.
|
||||||
|
#
|
||||||
|
# The Wiki.js change listener is reported below but deliberately excluded
|
||||||
|
# from this decision. It supervises and reconnects itself, and a database
|
||||||
|
# restart would otherwise flip the container unhealthy for the duration of
|
||||||
|
# an outage it is already recovering from. It is reported so the state is
|
||||||
|
# observable at all — previously nothing anywhere exposed it, which is how a
|
||||||
|
# dead listener went unnoticed while this endpoint answered "healthy".
|
||||||
all_healthy = service_health.get("neo4j", False) and service_health.get("qdrant", False)
|
all_healthy = service_health.get("neo4j", False) and service_health.get("qdrant", False)
|
||||||
overall_status = "healthy" if all_healthy else "degraded"
|
overall_status = "healthy" if all_healthy else "degraded"
|
||||||
|
|
||||||
|
wiki_listener = getattr(app.state, "wiki_listener", None)
|
||||||
|
|
||||||
return HealthResponse(
|
return HealthResponse(
|
||||||
status=overall_status,
|
status=overall_status,
|
||||||
app_name=settings.app_name,
|
app_name=settings.app_name,
|
||||||
@@ -152,6 +162,11 @@ async def health(settings: Settings = Depends(get_settings)) -> HealthResponse:
|
|||||||
"url": settings.ollama_url,
|
"url": settings.ollama_url,
|
||||||
"model": settings.ollama_llm_model,
|
"model": settings.ollama_llm_model,
|
||||||
"healthy": service_health.get("ollama", False)
|
"healthy": service_health.get("ollama", False)
|
||||||
|
},
|
||||||
|
"wiki_listener": {
|
||||||
|
"subscribed": bool(wiki_listener and wiki_listener.running),
|
||||||
|
"reconnects": getattr(wiki_listener, "reconnects", 0),
|
||||||
|
"last_gap_seconds": getattr(wiki_listener, "last_gap_seconds", None)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ Used by the consolidation endpoint to process SearchQuery nodes
|
|||||||
and consolidate knowledge into wiki pages.
|
and consolidate knowledge into wiki pages.
|
||||||
"""
|
"""
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Optional, Dict, Any
|
from typing import List, Optional
|
||||||
|
|
||||||
|
|
||||||
class ConsolidationRequest(BaseModel):
|
class ConsolidationRequest(BaseModel):
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ Provides models for knowledge graph nodes, relationships, and queries.
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any, Optional
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from src.core.multi_tenancy import RequiredUser
|
from src.core.multi_tenancy import RequiredUser
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
Pydantic models for Document Ingestion system.
|
Pydantic models for Document Ingestion system.
|
||||||
"""
|
"""
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from src.core.multi_tenancy import RequiredUser
|
from src.core.multi_tenancy import RequiredUser
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ Models for:
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field, field_validator
|
from pydantic import BaseModel, Field, field_validator
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from src.core.multi_tenancy import RequiredUser
|
from src.core.multi_tenancy import RequiredUser
|
||||||
|
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ async def capture_webhook(request: Request):
|
|||||||
# Try to parse as JSON
|
# Try to parse as JSON
|
||||||
try:
|
try:
|
||||||
capture["body_json"] = json.loads(body)
|
capture["body_json"] = json.loads(body)
|
||||||
except:
|
except Exception:
|
||||||
capture["body_json"] = None
|
capture["body_json"] = None
|
||||||
|
|
||||||
# Write to file
|
# Write to file
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ Creates both:
|
|||||||
import logging
|
import logging
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from typing import List, Dict, Any, Optional, Tuple
|
from typing import List, Dict, Any, Tuple
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from src.config import get_settings
|
from src.config import get_settings
|
||||||
@@ -18,7 +18,7 @@ from src.core.dependencies import (
|
|||||||
get_wiki_service,
|
get_wiki_service,
|
||||||
get_graph_service,
|
get_graph_service,
|
||||||
get_ingestion_service,
|
get_ingestion_service,
|
||||||
verify_api_key
|
verify_browser_request,
|
||||||
)
|
)
|
||||||
from src.services.wiki_service import WikiService
|
from src.services.wiki_service import WikiService
|
||||||
from src.services.graph_service import GraphService
|
from src.services.graph_service import GraphService
|
||||||
@@ -57,7 +57,7 @@ async def link_entities_in_page(
|
|||||||
wiki_service: WikiService = Depends(get_wiki_service),
|
wiki_service: WikiService = Depends(get_wiki_service),
|
||||||
graph_service: GraphService = Depends(get_graph_service),
|
graph_service: GraphService = Depends(get_graph_service),
|
||||||
ingestion_service: IngestionService = Depends(get_ingestion_service),
|
ingestion_service: IngestionService = Depends(get_ingestion_service),
|
||||||
api_key: str = Depends(verify_api_key)
|
actor: str = Depends(verify_browser_request)
|
||||||
) -> EntityLinkingResult:
|
) -> EntityLinkingResult:
|
||||||
"""
|
"""
|
||||||
Find and link entities mentioned in a wiki page.
|
Find and link entities mentioned in a wiki page.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import logging
|
|||||||
|
|
||||||
from src.models.graph import (
|
from src.models.graph import (
|
||||||
CypherQueryRequest, CypherQueryResponse,
|
CypherQueryRequest, CypherQueryResponse,
|
||||||
UpdateFromPageRequest, GraphUpdateSummary,
|
GraphUpdateSummary,
|
||||||
NodeListResponse, GraphNodeDetail,
|
NodeListResponse, GraphNodeDetail,
|
||||||
MindMapResponse
|
MindMapResponse
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ from src.models.ingestion import (
|
|||||||
BatchIngestionResult
|
BatchIngestionResult
|
||||||
)
|
)
|
||||||
from src.core.dependencies import (
|
from src.core.dependencies import (
|
||||||
get_ingestion_service, verify_api_key, RequiredUserQuery, JobManagerDep
|
get_ingestion_service, verify_api_key, verify_browser_request,
|
||||||
|
RequiredUserQuery, JobManagerDep
|
||||||
)
|
)
|
||||||
from src.jobs.job_manager import JobManager, JobStatus, JobType
|
from src.jobs.job_manager import JobManager, JobStatus, JobType
|
||||||
|
|
||||||
@@ -65,7 +66,7 @@ async def ingest_page(
|
|||||||
request: IngestionRequest,
|
request: IngestionRequest,
|
||||||
ingestion: IngestionService = Depends(get_ingestion_service),
|
ingestion: IngestionService = Depends(get_ingestion_service),
|
||||||
job_manager: JobManagerDep = None,
|
job_manager: JobManagerDep = None,
|
||||||
api_key: str = Depends(verify_api_key)
|
actor: str = Depends(verify_browser_request)
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Ingest a single wiki page into the knowledge base.
|
Ingest a single wiki page into the knowledge base.
|
||||||
|
|||||||
@@ -5,19 +5,15 @@ Endpoints for semantic search and vector operations.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Depends, Query
|
from fastapi import APIRouter, HTTPException, Depends, Query
|
||||||
from typing import Optional
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from src.models.vector import (
|
from src.models.vector import (
|
||||||
SearchRequest, SearchResponse,
|
SearchRequest, SearchResponse,
|
||||||
VectorUpdateRequest, VectorUpdateSummary,
|
VectorUpdateSummary,
|
||||||
CollectionListResponse,
|
CollectionListResponse,
|
||||||
DeletePageChunksRequest, DeletePageChunksResponse
|
DeletePageChunksResponse
|
||||||
)
|
)
|
||||||
from src.services.vector_service import VectorService
|
from src.services.vector_service import VectorService
|
||||||
from src.clients.qdrant_client import QdrantClientWrapper
|
|
||||||
from src.clients.wikijs_client import WikiJSClient
|
|
||||||
from src.clients.ollama_client import OllamaClient
|
|
||||||
from src.core.dependencies import (
|
from src.core.dependencies import (
|
||||||
QdrantDep, WikiJSDep, OllamaDep, verify_api_key, RequiredUserQuery
|
QdrantDep, WikiJSDep, OllamaDep, verify_api_key, RequiredUserQuery
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import logging
|
|||||||
from src.models.volatile import (
|
from src.models.volatile import (
|
||||||
VolatileRecordCreate,
|
VolatileRecordCreate,
|
||||||
VolatileRecordResponse,
|
VolatileRecordResponse,
|
||||||
VolatileListResponse,
|
|
||||||
VolatileScheduledResponse,
|
VolatileScheduledResponse,
|
||||||
VolatileStatsResponse,
|
VolatileStatsResponse,
|
||||||
VolatileDeleteResponse,
|
VolatileDeleteResponse,
|
||||||
|
|||||||
@@ -5,14 +5,12 @@ Receives webhook events from Wiki.js for page CRUD operations
|
|||||||
and processes them identically to AI-generated content.
|
and processes them identically to AI-generated content.
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
|
from fastapi import APIRouter, Depends, BackgroundTasks
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from typing import Optional, Literal
|
from typing import Optional, Literal
|
||||||
|
|
||||||
from src.core.dependencies import (
|
from src.core.dependencies import (
|
||||||
get_ingestion_service,
|
get_ingestion_service,
|
||||||
get_wiki_service,
|
|
||||||
get_graph_service,
|
|
||||||
verify_api_key
|
verify_api_key
|
||||||
)
|
)
|
||||||
from src.services.ingestion_service import IngestionService
|
from src.services.ingestion_service import IngestionService
|
||||||
@@ -373,7 +371,7 @@ async def process_page_rename(
|
|||||||
logger.error(f"Failed to apply entity linking: {e}")
|
logger.error(f"Failed to apply entity linking: {e}")
|
||||||
|
|
||||||
elif path_changed:
|
elif path_changed:
|
||||||
logger.info(f"Path changed only (move), no re-processing needed")
|
logger.info("Path changed only (move), no re-processing needed")
|
||||||
|
|
||||||
logger.info(f"Rename processing complete for page {page_id}")
|
logger.info(f"Rename processing complete for page {page_id}")
|
||||||
|
|
||||||
@@ -470,9 +468,9 @@ async def cleanup_deleted_page(
|
|||||||
logger.error(f"Failed to delete orphaned entity {entity_name}: {e}")
|
logger.error(f"Failed to delete orphaned entity {entity_name}: {e}")
|
||||||
|
|
||||||
# STEP 5: Clean up broken SearchQuery relationships
|
# STEP 5: Clean up broken SearchQuery relationships
|
||||||
cleanup_search_query = f"""
|
cleanup_search_query = """
|
||||||
MATCH (sq:SearchQuery)-[r:FOUND]->(d:Document)
|
MATCH (sq:SearchQuery)-[r:FOUND]->(d:Document)
|
||||||
WHERE NOT EXISTS {{(d)}}
|
WHERE NOT EXISTS {(d)}
|
||||||
DELETE r
|
DELETE r
|
||||||
RETURN count(r) as cleaned_count
|
RETURN count(r) as cleaned_count
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -18,10 +18,6 @@ from src.models.wiki import (
|
|||||||
from src.services.wiki_service import WikiService
|
from src.services.wiki_service import WikiService
|
||||||
from src.services.graph_service import GraphService
|
from src.services.graph_service import GraphService
|
||||||
from src.services.vector_service import VectorService
|
from src.services.vector_service import VectorService
|
||||||
from src.clients.wikijs_client import WikiJSClient
|
|
||||||
from src.clients.neo4j_client import Neo4jClient
|
|
||||||
from src.clients.qdrant_client import QdrantClientWrapper
|
|
||||||
from src.clients.ollama_client import OllamaClient
|
|
||||||
from src.core.dependencies import (
|
from src.core.dependencies import (
|
||||||
WikiJSDep, Neo4jDep, QdrantDep, OllamaDep,
|
WikiJSDep, Neo4jDep, QdrantDep, OllamaDep,
|
||||||
verify_api_key, get_settings, get_hybrid_rag_service, get_ingestion_service,
|
verify_api_key, get_settings, get_hybrid_rag_service, get_ingestion_service,
|
||||||
|
|||||||
@@ -15,14 +15,13 @@ import logging
|
|||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import List, Dict, Any, Optional
|
from typing import TYPE_CHECKING, List, Dict, Any, Optional
|
||||||
|
|
||||||
from src.clients.neo4j_client import Neo4jClient
|
from src.clients.neo4j_client import Neo4jClient
|
||||||
from src.clients.ollama_client import OllamaClient
|
from src.clients.ollama_client import OllamaClient
|
||||||
from src.clients.wikijs_client import WikiJSClient
|
from src.clients.wikijs_client import WikiJSClient
|
||||||
from src.services.wiki_page_writer import WikiPageWriter
|
from src.services.wiki_page_writer import WikiPageWriter
|
||||||
from src.models.consolidation import (
|
from src.models.consolidation import (
|
||||||
SearchQueryInfo,
|
|
||||||
ConsolidationResult,
|
ConsolidationResult,
|
||||||
ConsolidationResponse,
|
ConsolidationResponse,
|
||||||
MemoryRouteClassification,
|
MemoryRouteClassification,
|
||||||
@@ -30,6 +29,16 @@ from src.models.consolidation import (
|
|||||||
)
|
)
|
||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
|
|
||||||
|
# Imported for annotations only. The real imports live inside the functions
|
||||||
|
# that use them, to break an import cycle; a quoted annotation is never
|
||||||
|
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||||
|
# block costs nothing at import time and makes them resolvable again.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from src.clients.scheduler_client import SchedulerClient
|
||||||
|
from src.clients.settings_client import SettingsClient
|
||||||
|
from src.services.ingestion_service import IngestionService
|
||||||
|
from src.services.volatile_service import VolatileCacheService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -7,10 +7,19 @@ Provides bidirectional entity linking functionality that can be used by:
|
|||||||
- Any other service that creates wiki pages
|
- Any other service that creates wiki pages
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Any, Optional
|
from typing import TYPE_CHECKING, Dict, Optional
|
||||||
|
|
||||||
from src.core.multi_tenancy import get_neo4j_user_base_label
|
from src.core.multi_tenancy import get_neo4j_user_base_label
|
||||||
|
|
||||||
|
# Imported for annotations only. The real imports live inside the functions
|
||||||
|
# that use them, to break an import cycle; a quoted annotation is never
|
||||||
|
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||||
|
# block costs nothing at import time and makes them resolvable again.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from src.clients.neo4j_client import Neo4jClient
|
||||||
|
from src.services.ingestion_service import IngestionService
|
||||||
|
from src.services.wiki_service import WikiService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -49,12 +58,9 @@ async def apply_bidirectional_entity_linking(
|
|||||||
"""
|
"""
|
||||||
from src.routers.entity_linking import (
|
from src.routers.entity_linking import (
|
||||||
link_entities_in_page,
|
link_entities_in_page,
|
||||||
EntityLinkingRequest,
|
EntityLinkingRequest
|
||||||
get_entities_with_paths,
|
|
||||||
add_entity_links_to_content
|
|
||||||
)
|
)
|
||||||
from src.core.dependencies import get_graph_service, get_wiki_service, get_ingestion_service
|
from src.core.dependencies import get_graph_service, get_ingestion_service
|
||||||
from src.models.wiki import WikiPageUpdate
|
|
||||||
|
|
||||||
forward_links = 0
|
forward_links = 0
|
||||||
backward_links = 0
|
backward_links = 0
|
||||||
|
|||||||
@@ -879,7 +879,7 @@ Feel free to expand it with more details!
|
|||||||
{
|
{
|
||||||
"name": r["name"],
|
"name": r["name"],
|
||||||
# Get the entity type label (not the user label)
|
# Get the entity type label (not the user label)
|
||||||
"type": [l for l in r["labels"] if l not in [user_base_label, "Document"]][0]
|
"type": [line for line in r["labels"] if line not in [user_base_label, "Document"]][0]
|
||||||
if r["labels"] else "Entity",
|
if r["labels"] else "Entity",
|
||||||
"path": r.get("path") # Include path if it exists (for entity stub pages)
|
"path": r.get("path") # Include path if it exists (for entity stub pages)
|
||||||
}
|
}
|
||||||
@@ -1524,7 +1524,7 @@ Feel free to expand it with more details!
|
|||||||
for r in results:
|
for r in results:
|
||||||
labels = r.get("labels", [])
|
labels = r.get("labels", [])
|
||||||
entity_type = next(
|
entity_type = next(
|
||||||
(l for l in labels if l != user_base_label),
|
(line for line in labels if line != user_base_label),
|
||||||
"Unknown"
|
"Unknown"
|
||||||
)
|
)
|
||||||
orphans.append({
|
orphans.append({
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from src.clients.ollama_client import OllamaClient
|
|||||||
from src.clients.content_extractor import ContentExtractor
|
from src.clients.content_extractor import ContentExtractor
|
||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
from src.models.hybrid_rag import (
|
from src.models.hybrid_rag import (
|
||||||
HybridRAGConfig, HybridRAGRequest, HybridRAGResponse,
|
HybridRAGConfig, HybridRAGResponse,
|
||||||
HybridRAGResult, TimingBreakdown, KeywordExtraction,
|
HybridRAGResult, TimingBreakdown, KeywordExtraction,
|
||||||
RelatedDossier
|
RelatedDossier
|
||||||
)
|
)
|
||||||
@@ -123,7 +123,6 @@ class HybridRAGService:
|
|||||||
timing["query_enhancement_ms"] = (time.time() - phase0_start) * 1000
|
timing["query_enhancement_ms"] = (time.time() - phase0_start) * 1000
|
||||||
|
|
||||||
# Phase 1: Parallel Retrieval
|
# Phase 1: Parallel Retrieval
|
||||||
phase1_start = time.time()
|
|
||||||
raw_results = await self._retrieve_parallel(query, user, config, keywords_data)
|
raw_results = await self._retrieve_parallel(query, user, config, keywords_data)
|
||||||
timing["vector_ms"] = raw_results.get("timing", {}).get("vector_ms", 0)
|
timing["vector_ms"] = raw_results.get("timing", {}).get("vector_ms", 0)
|
||||||
timing["graph_ms"] = raw_results.get("timing", {}).get("graph_ms", 0)
|
timing["graph_ms"] = raw_results.get("timing", {}).get("graph_ms", 0)
|
||||||
@@ -357,7 +356,6 @@ JSON:"""
|
|||||||
"source_status" ('ok', 'failed', or 'disabled')
|
"source_status" ('ok', 'failed', or 'disabled')
|
||||||
"""
|
"""
|
||||||
tasks = {}
|
tasks = {}
|
||||||
timing = {}
|
|
||||||
|
|
||||||
# Vector search
|
# Vector search
|
||||||
if config.enable_vector:
|
if config.enable_vector:
|
||||||
|
|||||||
@@ -14,16 +14,13 @@ This service is called by:
|
|||||||
import logging
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from datetime import datetime
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from src.services.vector_service import VectorService
|
from src.services.vector_service import VectorService
|
||||||
from src.services.graph_service import GraphService
|
from src.services.graph_service import GraphService
|
||||||
from src.clients.wikijs_client import WikiJSClient
|
from src.clients.wikijs_client import WikiJSClient
|
||||||
from src.models.ingestion import (
|
from src.models.ingestion import (
|
||||||
IngestionRequest,
|
|
||||||
IngestionResult,
|
IngestionResult,
|
||||||
BatchIngestionRequest,
|
|
||||||
BatchIngestionResult
|
BatchIngestionResult
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ from src.clients.content_extractor import ContentExtractor
|
|||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
from src.models.rag_search import (
|
from src.models.rag_search import (
|
||||||
SearchType,
|
SearchType,
|
||||||
RAGSearchRequest,
|
|
||||||
RAGSearchResult,
|
RAGSearchResult,
|
||||||
RAGSearchResponse,
|
RAGSearchResponse,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,9 +6,8 @@ Handles semantic search, document chunking, and embeddings.
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import hashlib
|
|
||||||
import uuid
|
import uuid
|
||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from src.clients.qdrant_client import QdrantClientWrapper
|
from src.clients.qdrant_client import QdrantClientWrapper
|
||||||
@@ -17,7 +16,7 @@ from src.clients.ollama_client import OllamaClient
|
|||||||
from src.core.multi_tenancy import get_qdrant_collection_name, is_path_in_user_namespace
|
from src.core.multi_tenancy import get_qdrant_collection_name, is_path_in_user_namespace
|
||||||
from src.models.vector import (
|
from src.models.vector import (
|
||||||
SearchResult, SearchResponse, VectorUpdateSummary,
|
SearchResult, SearchResponse, VectorUpdateSummary,
|
||||||
DocumentChunk, CollectionInfo, CollectionListResponse
|
CollectionInfo, CollectionListResponse
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -14,12 +14,6 @@ from src.apis import (
|
|||||||
OpenMeteoProvider,
|
OpenMeteoProvider,
|
||||||
AggregatedNewsProvider,
|
AggregatedNewsProvider,
|
||||||
AlphaVantageProvider,
|
AlphaVantageProvider,
|
||||||
CurrentWeather,
|
|
||||||
WeatherForecast,
|
|
||||||
SunTimes,
|
|
||||||
AirQuality,
|
|
||||||
NewsFeed,
|
|
||||||
StockQuote,
|
|
||||||
)
|
)
|
||||||
from src.services.volatile_service import VolatileCacheService
|
from src.services.volatile_service import VolatileCacheService
|
||||||
from src.models.volatile import VolatileRecordResponse, VolatileNamespace
|
from src.models.volatile import VolatileRecordResponse, VolatileNamespace
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from datetime import datetime
|
|||||||
|
|
||||||
from src.config import get_settings
|
from src.config import get_settings
|
||||||
from src.core.dependencies import get_ingestion_service
|
from src.core.dependencies import get_ingestion_service
|
||||||
from src.services.consolidation_service import ConsolidationService
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -27,21 +26,59 @@ class WikiChangeListener:
|
|||||||
NOTIFY events on INSERT/UPDATE/DELETE to the pages table.
|
NOTIFY events on INSERT/UPDATE/DELETE to the pages table.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
CHANNEL = 'wiki_page_changes'
|
||||||
|
|
||||||
|
# Bounded exponential backoff between reconnect attempts.
|
||||||
|
BACKOFF_INITIAL_SECONDS = 1.0
|
||||||
|
BACKOFF_MAX_SECONDS = 60.0
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.settings = get_settings()
|
self.settings = get_settings()
|
||||||
self.connection: Optional[asyncpg.Connection] = None
|
self.connection: Optional[asyncpg.Connection] = None
|
||||||
self.running = False
|
|
||||||
|
|
||||||
# Loop prevention: Track recently processed pages
|
# Loop prevention: Track recently processed pages
|
||||||
# Key: page_id, Value: timestamp of last processing
|
# Key: page_id, Value: timestamp of last processing
|
||||||
self._recent_notifications = {}
|
self._recent_notifications = {}
|
||||||
self._debounce_seconds = self.settings.wikijs_change_listener_debounce_seconds
|
self._debounce_seconds = self.settings.wikijs_change_listener_debounce_seconds
|
||||||
|
|
||||||
async def start(self):
|
# Supervision state. A single LISTEN connection does not heal itself the
|
||||||
"""Start listening to database changes."""
|
# way an asyncpg pool does, so the drop has to be detected and repaired
|
||||||
logger.info("Starting Wiki.js database change listener")
|
# explicitly — see _supervise().
|
||||||
|
self._stopping = False
|
||||||
|
self._disconnected = asyncio.Event()
|
||||||
|
self._supervisor_task: Optional[asyncio.Task] = None
|
||||||
|
self._disconnected_at: Optional[datetime] = None
|
||||||
|
self.reconnects = 0
|
||||||
|
self.last_gap_seconds: Optional[float] = None
|
||||||
|
|
||||||
# Connect to Wiki.js PostgreSQL database
|
@property
|
||||||
|
def running(self) -> bool:
|
||||||
|
"""
|
||||||
|
Whether a live subscription actually exists.
|
||||||
|
|
||||||
|
Derived rather than assigned. The previous implementation set a flag once
|
||||||
|
in start() and never revisited it, so after the connection dropped the
|
||||||
|
listener reported itself as running while being deaf to every event.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
not self._stopping
|
||||||
|
and self.connection is not None
|
||||||
|
and not self.connection.is_closed()
|
||||||
|
)
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
"""Start listening to database changes, and keep listening."""
|
||||||
|
logger.info("Starting Wiki.js database change listener")
|
||||||
|
self._stopping = False
|
||||||
|
self._disconnected.clear()
|
||||||
|
|
||||||
|
await self._connect()
|
||||||
|
|
||||||
|
self._supervisor_task = asyncio.create_task(self._supervise())
|
||||||
|
logger.info("Listening for Wiki.js page changes via PostgreSQL NOTIFY")
|
||||||
|
|
||||||
|
async def _connect(self):
|
||||||
|
"""Open a connection and subscribe. Raises if the database is unreachable."""
|
||||||
self.connection = await asyncpg.connect(
|
self.connection = await asyncpg.connect(
|
||||||
host=self.settings.wikijs_db_host,
|
host=self.settings.wikijs_db_host,
|
||||||
port=self.settings.wikijs_db_port,
|
port=self.settings.wikijs_db_port,
|
||||||
@@ -50,18 +87,136 @@ class WikiChangeListener:
|
|||||||
database=self.settings.wikijs_db_name
|
database=self.settings.wikijs_db_name
|
||||||
)
|
)
|
||||||
|
|
||||||
# Listen to the wiki_page_changes channel
|
await self.connection.add_listener(self.CHANNEL, self._handle_notification)
|
||||||
await self.connection.add_listener('wiki_page_changes', self._handle_notification)
|
|
||||||
|
|
||||||
self.running = True
|
# Must be re-registered on every connection: asyncpg clears its
|
||||||
logger.info("Listening for Wiki.js page changes via PostgreSQL NOTIFY")
|
# termination listeners as soon as it fires them, so this is one-shot.
|
||||||
|
self.connection.add_termination_listener(self._on_connection_lost)
|
||||||
|
|
||||||
|
def _on_connection_lost(self, connection):
|
||||||
|
"""
|
||||||
|
Called by asyncpg when the connection terminates.
|
||||||
|
|
||||||
|
Dispatched through loop.call_soon, so it must stay synchronous — the work
|
||||||
|
of reconnecting belongs to _supervise(), which this only wakes.
|
||||||
|
"""
|
||||||
|
if self._stopping:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._disconnected_at = datetime.now()
|
||||||
|
logger.error(
|
||||||
|
"Wiki.js change listener lost its database connection — "
|
||||||
|
"page changes are NOT being processed until it reconnects"
|
||||||
|
)
|
||||||
|
self._disconnected.set()
|
||||||
|
|
||||||
|
async def _supervise(self, max_iterations: Optional[int] = None) -> int:
|
||||||
|
"""
|
||||||
|
Reconnect whenever the subscription drops.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
max_iterations: Stop after N reconnect cycles (None = run forever;
|
||||||
|
used by tests)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Number of completed reconnect cycles
|
||||||
|
"""
|
||||||
|
iterations = 0
|
||||||
|
while max_iterations is None or iterations < max_iterations:
|
||||||
|
await self._disconnected.wait()
|
||||||
|
if self._stopping:
|
||||||
|
break
|
||||||
|
|
||||||
|
self._disconnected.clear()
|
||||||
|
await self._reconnect_with_backoff()
|
||||||
|
iterations += 1
|
||||||
|
|
||||||
|
return iterations
|
||||||
|
|
||||||
|
async def _reconnect_with_backoff(self, max_attempts: Optional[int] = None) -> bool:
|
||||||
|
"""
|
||||||
|
Re-establish the subscription, backing off between failures.
|
||||||
|
|
||||||
|
Keeps trying indefinitely by default: a database that is down for
|
||||||
|
maintenance will come back, and giving up would recreate exactly the
|
||||||
|
silent-deafness this supervision exists to prevent.
|
||||||
|
"""
|
||||||
|
delay = self.BACKOFF_INITIAL_SECONDS
|
||||||
|
attempts = 0
|
||||||
|
|
||||||
|
while not self._stopping and (max_attempts is None or attempts < max_attempts):
|
||||||
|
attempts += 1
|
||||||
|
await self._close_connection()
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self._connect()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
f"Wiki.js change listener reconnect attempt {attempts} failed: {e}; "
|
||||||
|
f"retrying in {delay:.0f}s"
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
delay = min(delay * 2, self.BACKOFF_MAX_SECONDS)
|
||||||
|
continue
|
||||||
|
|
||||||
|
self.reconnects += 1
|
||||||
|
gap = None
|
||||||
|
if self._disconnected_at is not None:
|
||||||
|
gap = (datetime.now() - self._disconnected_at).total_seconds()
|
||||||
|
self.last_gap_seconds = gap
|
||||||
|
self._disconnected_at = None
|
||||||
|
|
||||||
|
# NOTIFY is fire-and-forget: anything emitted while we were gone was
|
||||||
|
# delivered to nobody and cannot be replayed. Say so, and say what
|
||||||
|
# closes the gap, rather than reporting a clean recovery.
|
||||||
|
outage = f" after {gap:.0f}s" if gap is not None else ""
|
||||||
|
logger.warning(
|
||||||
|
f"Wiki.js change listener reconnected{outage} "
|
||||||
|
f"(reconnect #{self.reconnects}). NOTIFY events emitted during the "
|
||||||
|
f"outage were lost and cannot be replayed — run "
|
||||||
|
f"POST /maintenance/integrity-check to reconcile pages that "
|
||||||
|
f"changed while the listener was down."
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def _close_connection(self):
|
||||||
|
"""Drop the current connection, tolerating one that is already dead."""
|
||||||
|
if not self.connection:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not self.connection.is_closed():
|
||||||
|
await self.connection.remove_listener(
|
||||||
|
self.CHANNEL, self._handle_notification
|
||||||
|
)
|
||||||
|
await self.connection.close()
|
||||||
|
except Exception as e:
|
||||||
|
# A terminated connection raises on both calls; that is expected here.
|
||||||
|
logger.debug(f"Error closing Wiki.js listener connection: {e}")
|
||||||
|
finally:
|
||||||
|
self.connection = None
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
"""Stop listening and close connection."""
|
"""Stop listening and close connection."""
|
||||||
if self.connection:
|
self._stopping = True
|
||||||
await self.connection.remove_listener('wiki_page_changes', self._handle_notification)
|
|
||||||
await self.connection.close()
|
# Wake the supervisor so it observes _stopping and exits rather than
|
||||||
self.running = False
|
# racing us to reconnect the connection we are about to close.
|
||||||
|
self._disconnected.set()
|
||||||
|
|
||||||
|
if self._supervisor_task:
|
||||||
|
self._supervisor_task.cancel()
|
||||||
|
try:
|
||||||
|
await self._supervisor_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Wiki.js listener supervisor ended with: {e}")
|
||||||
|
self._supervisor_task = None
|
||||||
|
|
||||||
|
await self._close_connection()
|
||||||
logger.info("Stopped Wiki.js change listener")
|
logger.info("Stopped Wiki.js change listener")
|
||||||
|
|
||||||
async def _handle_notification(self, connection, pid, channel, payload):
|
async def _handle_notification(self, connection, pid, channel, payload):
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Handles business logic for wiki operations with:
|
|||||||
- Search functionality
|
- Search functionality
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import List, Optional, Dict, Any
|
from typing import TYPE_CHECKING, List, Optional, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from src.clients.wikijs_client import WikiJSClient
|
from src.clients.wikijs_client import WikiJSClient
|
||||||
@@ -19,6 +19,14 @@ from src.models.wiki import (
|
|||||||
DossierInfo, DossierList
|
DossierInfo, DossierList
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Imported for annotations only. The real imports live inside the functions
|
||||||
|
# that use them, to break an import cycle; a quoted annotation is never
|
||||||
|
# evaluated at runtime, so the names were unresolvable to any checker. This
|
||||||
|
# block costs nothing at import time and makes them resolvable again.
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from src.services.hybrid_rag_service import HybridRAGService
|
||||||
|
from src.services.wiki_page_writer import WikiPageWriter
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,21 +2,28 @@
|
|||||||
* Library Desk Integration for Wiki.js
|
* Library Desk Integration for Wiki.js
|
||||||
* Combined re-index and entity linking buttons
|
* Combined re-index and entity linking buttons
|
||||||
*
|
*
|
||||||
* Usage: Add to Wiki.js Code Injection:
|
* Usage: Add to Wiki.js Code Injection (served same-origin behind Authentik):
|
||||||
* <script src="http://192.168.86.149:8089/static/wikijs-integration.js"></script>
|
* <script src="/library-desk/static/wikijs-integration.js"></script>
|
||||||
|
*
|
||||||
|
* Auth: none in the browser. Requests go same-origin through the NPM
|
||||||
|
* /library-desk/ location, which is gated by Authentik forward-auth with the
|
||||||
|
* LAN bypass — external users are authenticated, LAN users pass through, and
|
||||||
|
* library-desk trusts the proxy marker header. No API key is embedded here.
|
||||||
*/
|
*/
|
||||||
(function() {
|
(function() {
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
// Auto-detect Library Desk URL
|
// Same-origin base: the script is served from <origin>/library-desk/static/...,
|
||||||
|
// so strip '/static/...' to get the library-desk mount point on this origin.
|
||||||
const scriptTag = document.currentScript;
|
const scriptTag = document.currentScript;
|
||||||
const scriptUrl = scriptTag ? scriptTag.src : '';
|
const scriptUrl = scriptTag ? scriptTag.src : '';
|
||||||
const libraryDeskUrl = scriptUrl ? scriptUrl.split('/static/')[0] : 'http://192.168.86.149:8089';
|
const libraryDeskUrl = scriptUrl
|
||||||
|
? scriptUrl.replace(/^https?:\/\/[^/]+/, '').split('/static/')[0]
|
||||||
|
: '/library-desk';
|
||||||
|
|
||||||
// Shared configuration
|
// Shared configuration
|
||||||
const CONFIG = window.LIBRARY_DESK_CONFIG || {
|
const CONFIG = window.LIBRARY_DESK_CONFIG || {
|
||||||
libraryDeskUrl: libraryDeskUrl,
|
libraryDeskUrl: libraryDeskUrl,
|
||||||
apiKey: 'af88ed8f44bed81bdb20d0534f1c4547340b29e2aba4963f61a71b993d7eb6e5',
|
|
||||||
user: 'jpmschweitzer',
|
user: 'jpmschweitzer',
|
||||||
buttonPosition: 'toolbar', // 'toolbar' or 'floating'
|
buttonPosition: 'toolbar', // 'toolbar' or 'floating'
|
||||||
debug: true
|
debug: true
|
||||||
@@ -229,8 +236,8 @@
|
|||||||
// Re-index directly
|
// Re-index directly
|
||||||
const response = await fetch(CONFIG.libraryDeskUrl + '/ingest/page', {
|
const response = await fetch(CONFIG.libraryDeskUrl + '/ingest/page', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
credentials: 'same-origin',
|
||||||
headers: {
|
headers: {
|
||||||
'Authorization': 'Bearer ' + CONFIG.apiKey,
|
|
||||||
'Content-Type': 'application/json'
|
'Content-Type': 'application/json'
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
@@ -334,8 +341,8 @@
|
|||||||
// Call entity linking endpoint
|
// Call entity linking endpoint
|
||||||
const response = await fetch(CONFIG.libraryDeskUrl + '/entity-linking/link-page', {
|
const response = await fetch(CONFIG.libraryDeskUrl + '/entity-linking/link-page', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
credentials: 'same-origin',
|
||||||
headers: {
|
headers: {
|
||||||
'Authorization': 'Bearer ' + CONFIG.apiKey,
|
|
||||||
'Content-Type': 'application/json'
|
'Content-Type': 'application/json'
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""
|
||||||
|
Tests for verify_browser_request — the session/proxy auth used by the
|
||||||
|
Wiki.js integration endpoints (no secret in the browser).
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from fastapi import HTTPException
|
||||||
|
from starlette.requests import Request
|
||||||
|
|
||||||
|
from src.core.dependencies import verify_browser_request
|
||||||
|
|
||||||
|
|
||||||
|
def _request(headers: dict) -> Request:
|
||||||
|
raw = [(k.lower().encode(), v.encode()) for k, v in headers.items()]
|
||||||
|
return Request({"type": "http", "method": "POST", "path": "/ingest/page", "headers": raw})
|
||||||
|
|
||||||
|
|
||||||
|
SETTINGS = SimpleNamespace(library_api_key="server-secret-key")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_proxy_marker_with_authentik_identity_is_accepted():
|
||||||
|
req = _request({"X-Library-Desk-Proxy": "1", "X-Authentik-Email": "user@example.com"})
|
||||||
|
assert await verify_browser_request(req, SETTINGS) == "user@example.com"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_proxy_marker_on_lan_bypass_falls_back_to_lan():
|
||||||
|
# LAN bypass: proxy marker present, no Authentik identity headers.
|
||||||
|
req = _request({"X-Library-Desk-Proxy": "1"})
|
||||||
|
assert await verify_browser_request(req, SETTINGS) == "lan"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_valid_api_key_is_accepted_for_machine_callers():
|
||||||
|
req = _request({"Authorization": "Bearer server-secret-key"})
|
||||||
|
assert await verify_browser_request(req, SETTINGS) == "server-secret-key"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no_marker_and_no_key_is_rejected():
|
||||||
|
with pytest.raises(HTTPException) as exc:
|
||||||
|
await verify_browser_request(_request({}), SETTINGS)
|
||||||
|
assert exc.value.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_forged_marker_value_is_rejected():
|
||||||
|
# Only the exact NPM-set value "1" is trusted.
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
await verify_browser_request(_request({"X-Library-Desk-Proxy": "yes"}), SETTINGS)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_wrong_api_key_is_rejected():
|
||||||
|
with pytest.raises(HTTPException):
|
||||||
|
await verify_browser_request(_request({"Authorization": "Bearer wrong"}), SETTINGS)
|
||||||
@@ -12,9 +12,7 @@ Run with: pytest tests/test_consolidation.py -v -s
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
from typing import AsyncGenerator
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import json
|
import json
|
||||||
|
|
||||||
@@ -22,8 +20,7 @@ from src.services.consolidation_service import ConsolidationService
|
|||||||
from src.models.consolidation import (
|
from src.models.consolidation import (
|
||||||
ConsolidationRequest,
|
ConsolidationRequest,
|
||||||
ConsolidationResponse,
|
ConsolidationResponse,
|
||||||
ConsolidationResult,
|
ConsolidationResult
|
||||||
SearchQueryInfo
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Test constants
|
# Test constants
|
||||||
|
|||||||
@@ -285,6 +285,7 @@ class TestAddEntityLinksToContent:
|
|||||||
# Integration Tests - Full Entity Linking Flow
|
# Integration Tests - Full Entity Linking Flow
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
class TestEntityLinkingIntegration:
|
class TestEntityLinkingIntegration:
|
||||||
"""Test full entity linking flow."""
|
"""Test full entity linking flow."""
|
||||||
|
|
||||||
@@ -421,6 +422,7 @@ class TestEntityLinkingIntegration:
|
|||||||
# Multi-Tenancy Tests
|
# Multi-Tenancy Tests
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
class TestEntityLinkingMultiTenancy:
|
class TestEntityLinkingMultiTenancy:
|
||||||
"""Test multi-tenancy isolation in entity linking."""
|
"""Test multi-tenancy isolation in entity linking."""
|
||||||
|
|
||||||
@@ -464,6 +466,7 @@ class TestEntityLinkingMultiTenancy:
|
|||||||
# Cleanup
|
# Cleanup
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cleanup_entity_linking_test_data(neo4j_client):
|
async def test_cleanup_entity_linking_test_data(neo4j_client):
|
||||||
"""Clean up all test data created by entity linking tests."""
|
"""Clean up all test data created by entity linking tests."""
|
||||||
@@ -480,4 +483,4 @@ async def test_cleanup_entity_linking_test_data(neo4j_client):
|
|||||||
"""
|
"""
|
||||||
await neo4j_client.execute_query(cleanup_query)
|
await neo4j_client.execute_query(cleanup_query)
|
||||||
|
|
||||||
print(f"\n✓ Cleaned up entity linking test data")
|
print("\n✓ Cleaned up entity linking test data")
|
||||||
|
|||||||
@@ -10,9 +10,7 @@ Run with: pytest tests/test_graph_service.py -v -s
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
from unittest.mock import AsyncMock
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
from typing import AsyncGenerator
|
|
||||||
|
|
||||||
from src.services.graph_service import GraphService
|
from src.services.graph_service import GraphService
|
||||||
|
|
||||||
|
|||||||
@@ -29,9 +29,16 @@ from src.clients.content_extractor import ContentExtractor
|
|||||||
from src.services.hybrid_rag_service import HybridRAGService
|
from src.services.hybrid_rag_service import HybridRAGService
|
||||||
from src.services.vector_service import VectorService
|
from src.services.vector_service import VectorService
|
||||||
from src.services.graph_service import GraphService
|
from src.services.graph_service import GraphService
|
||||||
from src.models.hybrid_rag import HybridRAGConfig, HybridRAGRequest
|
from src.models.hybrid_rag import HybridRAGConfig
|
||||||
from src.config import get_settings
|
from src.config import get_settings
|
||||||
|
|
||||||
|
# Every test in this module goes through hybrid_rag_service -> graph_service ->
|
||||||
|
# neo4j_client, which opens a real Bolt connection at fixture setup (T-55/D-26).
|
||||||
|
# There is no unit/integration split within the file: even the fusion/formatting
|
||||||
|
# classes that look like pure logic still resolve the full fixture chain, so the
|
||||||
|
# whole module is marked rather than picking classes apart from underneath.
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
# Test user to isolate test data
|
# Test user to isolate test data
|
||||||
TEST_USER = "llm-tester"
|
TEST_USER = "llm-tester"
|
||||||
|
|
||||||
@@ -176,7 +183,7 @@ We deploy microservices using Helm charts and manage them with kubectl.
|
|||||||
# Cleanup
|
# Cleanup
|
||||||
try:
|
try:
|
||||||
await wiki_client.delete_page(page["id"])
|
await wiki_client.delete_page(page["id"])
|
||||||
except:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
pytest.skip(f"Could not create test page: {e}")
|
pytest.skip(f"Could not create test page: {e}")
|
||||||
@@ -531,7 +538,7 @@ class TestPhase6_Persistence:
|
|||||||
result = await neo4j_client.execute_query(query, {"search_id": search_id})
|
result = await neo4j_client.execute_query(query, {"search_id": search_id})
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert result[0]["query"] == "test query"
|
assert result[0]["query"] == "test query"
|
||||||
assert result[0]["processed"] == False
|
assert not result[0]["processed"]
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
cleanup_query = f"""
|
cleanup_query = f"""
|
||||||
@@ -614,7 +621,7 @@ class TestHybridRAG_EndToEnd:
|
|||||||
assert len(response.context) > 0
|
assert len(response.context) > 0
|
||||||
|
|
||||||
# Log results for inspection
|
# Log results for inspection
|
||||||
print(f"\n=== HybridRAG E2E Test Results ===")
|
print("\n=== HybridRAG E2E Test Results ===")
|
||||||
print(f"Query: {response.query}")
|
print(f"Query: {response.query}")
|
||||||
print(f"Total Results: {response.total_results}")
|
print(f"Total Results: {response.total_results}")
|
||||||
print(f"Source Counts: {response.source_counts}")
|
print(f"Source Counts: {response.source_counts}")
|
||||||
@@ -623,7 +630,7 @@ class TestHybridRAG_EndToEnd:
|
|||||||
print(f"Search ID: {response.search_id}")
|
print(f"Search ID: {response.search_id}")
|
||||||
|
|
||||||
if response.results:
|
if response.results:
|
||||||
print(f"\nTop Result:")
|
print("\nTop Result:")
|
||||||
top = response.results[0]
|
top = response.results[0]
|
||||||
print(f" Title: {top.title}")
|
print(f" Title: {top.title}")
|
||||||
print(f" Source: {top.source_type}")
|
print(f" Source: {top.source_type}")
|
||||||
@@ -674,7 +681,7 @@ class TestHybridRAG_EndToEnd:
|
|||||||
config = HybridRAGConfig()
|
config = HybridRAGConfig()
|
||||||
|
|
||||||
start = time.time()
|
start = time.time()
|
||||||
response = await hybrid_rag_service.search(
|
await hybrid_rag_service.search(
|
||||||
query="kubernetes orchestration",
|
query="kubernetes orchestration",
|
||||||
user=TEST_USER,
|
user=TEST_USER,
|
||||||
config=config
|
config=config
|
||||||
@@ -722,7 +729,7 @@ async def test_cleanup_test_data(neo4j_client, qdrant_client):
|
|||||||
collection_name = get_qdrant_collection_name(TEST_USER)
|
collection_name = get_qdrant_collection_name(TEST_USER)
|
||||||
try:
|
try:
|
||||||
await qdrant_client.delete_collection(collection_name)
|
await qdrant_client.delete_collection(collection_name)
|
||||||
except:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
print(f"\n✓ Cleaned up test data for user: {TEST_USER}")
|
print(f"\n✓ Cleaned up test data for user: {TEST_USER}")
|
||||||
|
|||||||
@@ -11,16 +11,12 @@ Run with: pytest tests/test_ingestion.py -v -s
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
from typing import AsyncGenerator
|
|
||||||
|
|
||||||
from src.services.ingestion_service import IngestionService
|
from src.services.ingestion_service import IngestionService
|
||||||
from src.models.ingestion import (
|
from src.models.ingestion import (
|
||||||
IngestionRequest,
|
IngestionRequest,
|
||||||
IngestionResult,
|
BatchIngestionRequest
|
||||||
BatchIngestionRequest,
|
|
||||||
BatchIngestionResult
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Tests cleanup of:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
from src.routers.maintenance import (
|
from src.routers.maintenance import (
|
||||||
cleanup_vectors,
|
cleanup_vectors,
|
||||||
cleanup_graph,
|
cleanup_graph,
|
||||||
@@ -18,9 +18,7 @@ from src.routers.maintenance import (
|
|||||||
CleanupResult,
|
CleanupResult,
|
||||||
VectorCleanupResponse,
|
VectorCleanupResponse,
|
||||||
GraphCleanupResponse,
|
GraphCleanupResponse,
|
||||||
FullCleanupResponse,
|
HealthCheckResponse
|
||||||
HealthCheckResponse,
|
|
||||||
ReindexResponse
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Tests for RAG search service and endpoints."""
|
"""Tests for RAG search service and endpoints."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
from src.models.rag_search import (
|
from src.models.rag_search import (
|
||||||
SearchType,
|
SearchType,
|
||||||
|
|||||||
@@ -12,19 +12,21 @@ import pytest
|
|||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from src.main import app
|
from src.main import app
|
||||||
from src.core.dependencies import verify_api_key
|
from src.core.dependencies import verify_api_key, verify_browser_request
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def client():
|
def client():
|
||||||
"""TestClient with API-key auth stubbed out (no lifespan startup)."""
|
"""TestClient with auth stubbed out (no lifespan startup)."""
|
||||||
app.dependency_overrides[verify_api_key] = lambda: "test-key"
|
app.dependency_overrides[verify_api_key] = lambda: "test-key"
|
||||||
|
app.dependency_overrides[verify_browser_request] = lambda: "test-user"
|
||||||
try:
|
try:
|
||||||
# No context manager: startup/lifespan events are NOT triggered,
|
# No context manager: startup/lifespan events are NOT triggered,
|
||||||
# so no connections to external services are attempted.
|
# so no connections to external services are attempted.
|
||||||
yield TestClient(app)
|
yield TestClient(app)
|
||||||
finally:
|
finally:
|
||||||
app.dependency_overrides.pop(verify_api_key, None)
|
app.dependency_overrides.pop(verify_api_key, None)
|
||||||
|
app.dependency_overrides.pop(verify_browser_request, None)
|
||||||
|
|
||||||
|
|
||||||
QUERY_PARAM_ENDPOINTS = [
|
QUERY_PARAM_ENDPOINTS = [
|
||||||
|
|||||||
@@ -538,8 +538,6 @@ class TestSmartCreateEndpoint:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_endpoint_returns_201(self, mock_clients):
|
async def test_endpoint_returns_201(self, mock_clients):
|
||||||
"""Test that successful creation returns 201 status."""
|
"""Test that successful creation returns 201 status."""
|
||||||
from fastapi.testclient import TestClient
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
# This test would require more setup with FastAPI TestClient
|
# This test would require more setup with FastAPI TestClient
|
||||||
# For now, we test the model validation
|
# For now, we test the model validation
|
||||||
|
|||||||
@@ -539,7 +539,7 @@ class TestVolatileCleanupEndpoint:
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cleanup_volatile(self):
|
async def test_cleanup_volatile(self):
|
||||||
"""Test volatile cleanup endpoint."""
|
"""Test volatile cleanup endpoint."""
|
||||||
from src.routers.maintenance import cleanup_volatile, VolatileCleanupResponse
|
from src.routers.maintenance import cleanup_volatile
|
||||||
|
|
||||||
mock_qdrant = AsyncMock()
|
mock_qdrant = AsyncMock()
|
||||||
mock_qdrant.get_volatile_collections = AsyncMock(return_value=[
|
mock_qdrant.get_volatile_collections = AsyncMock(return_value=[
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ Tests the PostgreSQL NOTIFY/LISTEN change detection system including:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import asyncio
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
@@ -238,7 +237,7 @@ class TestWikiChangeListener:
|
|||||||
async def test_process_page_delete_calls_cleanup(self, listener):
|
async def test_process_page_delete_calls_cleanup(self, listener):
|
||||||
"""Test that DELETE events call cleanup_deleted_page."""
|
"""Test that DELETE events call cleanup_deleted_page."""
|
||||||
with patch('src.routers.webhooks.cleanup_deleted_page', new_callable=AsyncMock) as mock_cleanup, \
|
with patch('src.routers.webhooks.cleanup_deleted_page', new_callable=AsyncMock) as mock_cleanup, \
|
||||||
patch('src.services.wiki_change_listener.get_ingestion_service') as mock_service:
|
patch('src.services.wiki_change_listener.get_ingestion_service'):
|
||||||
|
|
||||||
await listener._process_page_change(
|
await listener._process_page_change(
|
||||||
page_id=123,
|
page_id=123,
|
||||||
@@ -308,6 +307,10 @@ class TestWikiChangeListener:
|
|||||||
async def test_connection_lifecycle(self, listener, mock_settings):
|
async def test_connection_lifecycle(self, listener, mock_settings):
|
||||||
"""Test listener connection start and stop lifecycle."""
|
"""Test listener connection start and stop lifecycle."""
|
||||||
mock_connection = AsyncMock()
|
mock_connection = AsyncMock()
|
||||||
|
# asyncpg's is_closed() is synchronous. Left as an AsyncMock it returns a
|
||||||
|
# coroutine, which is always truthy, so the connection would read as
|
||||||
|
# closed the moment `running` started deriving from it.
|
||||||
|
mock_connection.is_closed = MagicMock(return_value=False)
|
||||||
|
|
||||||
with patch('src.services.wiki_change_listener.asyncpg.connect', return_value=mock_connection) as mock_connect:
|
with patch('src.services.wiki_change_listener.asyncpg.connect', return_value=mock_connection) as mock_connect:
|
||||||
# Start listener
|
# Start listener
|
||||||
@@ -426,3 +429,168 @@ class TestWikiChangeListenerIntegration:
|
|||||||
|
|
||||||
# This would test actual pg_notify() calls from triggers
|
# This would test actual pg_notify() calls from triggers
|
||||||
# and verify the listener receives and processes them
|
# and verify the listener receives and processes them
|
||||||
|
|
||||||
|
|
||||||
|
class TestWikiChangeListenerReconnect:
|
||||||
|
"""
|
||||||
|
Supervision and reconnect behaviour.
|
||||||
|
|
||||||
|
Regression cover for 2026-08-08: redeploying postgres-shared dropped the
|
||||||
|
listener's connection and it never came back. The container kept reporting
|
||||||
|
healthy because nothing observed the subscription, so Wiki.js page edits
|
||||||
|
silently stopped being indexed until someone restarted the service.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_running_is_false_when_connection_closed(self, listener):
|
||||||
|
"""running reflects the live connection, not a flag set once at startup."""
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.is_closed.return_value = False
|
||||||
|
listener.connection = connection
|
||||||
|
assert listener.running is True
|
||||||
|
|
||||||
|
# The exact failure mode: connection dies, nothing reassigns a flag.
|
||||||
|
connection.is_closed.return_value = True
|
||||||
|
assert listener.running is False
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_running_is_false_while_stopping(self, listener):
|
||||||
|
"""A listener being torn down does not advertise itself as subscribed."""
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.is_closed.return_value = False
|
||||||
|
listener.connection = connection
|
||||||
|
listener._stopping = True
|
||||||
|
assert listener.running is False
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_termination_callback_wakes_supervisor(self, listener):
|
||||||
|
"""asyncpg's termination callback signals the supervisor and records the time."""
|
||||||
|
assert not listener._disconnected.is_set()
|
||||||
|
|
||||||
|
listener._on_connection_lost(MagicMock())
|
||||||
|
|
||||||
|
assert listener._disconnected.is_set()
|
||||||
|
assert listener._disconnected_at is not None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_termination_callback_ignored_while_stopping(self, listener):
|
||||||
|
"""A deliberate shutdown must not trigger a reconnect."""
|
||||||
|
listener._stopping = True
|
||||||
|
|
||||||
|
listener._on_connection_lost(MagicMock())
|
||||||
|
|
||||||
|
assert not listener._disconnected.is_set()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_supervisor_reconnects_after_drop(self, listener):
|
||||||
|
"""One drop produces one reconnect cycle."""
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.is_closed.return_value = True
|
||||||
|
listener.connection = connection
|
||||||
|
|
||||||
|
with patch.object(listener, '_connect', new=AsyncMock()) as connect:
|
||||||
|
listener._disconnected.set()
|
||||||
|
cycles = await listener._supervise(max_iterations=1)
|
||||||
|
|
||||||
|
assert cycles == 1
|
||||||
|
connect.assert_awaited_once()
|
||||||
|
assert listener.reconnects == 1
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_supervisor_exits_without_reconnecting_when_stopping(self, listener):
|
||||||
|
"""stop() wakes the supervisor to exit, not to re-establish the connection."""
|
||||||
|
with patch.object(listener, '_connect', new=AsyncMock()) as connect:
|
||||||
|
listener._stopping = True
|
||||||
|
listener._disconnected.set()
|
||||||
|
cycles = await listener._supervise(max_iterations=1)
|
||||||
|
|
||||||
|
assert cycles == 0
|
||||||
|
connect.assert_not_awaited()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reconnect_backs_off_and_retries(self, listener):
|
||||||
|
"""A database that is still down is retried, with growing delay."""
|
||||||
|
attempts = []
|
||||||
|
|
||||||
|
async def fail_twice_then_succeed():
|
||||||
|
attempts.append(1)
|
||||||
|
if len(attempts) < 3:
|
||||||
|
raise OSError("connection refused")
|
||||||
|
|
||||||
|
sleeps = []
|
||||||
|
|
||||||
|
async def fake_sleep(seconds):
|
||||||
|
sleeps.append(seconds)
|
||||||
|
|
||||||
|
with patch.object(listener, '_connect', new=AsyncMock(side_effect=fail_twice_then_succeed)), \
|
||||||
|
patch.object(listener, '_close_connection', new=AsyncMock()), \
|
||||||
|
patch('asyncio.sleep', new=fake_sleep):
|
||||||
|
ok = await listener._reconnect_with_backoff()
|
||||||
|
|
||||||
|
assert ok is True
|
||||||
|
assert len(attempts) == 3
|
||||||
|
assert sleeps == [1.0, 2.0] # doubling
|
||||||
|
assert listener.reconnects == 1
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reconnect_backoff_is_capped(self, listener):
|
||||||
|
"""Backoff does not grow without bound during a long outage."""
|
||||||
|
sleeps = []
|
||||||
|
|
||||||
|
async def fake_sleep(seconds):
|
||||||
|
sleeps.append(seconds)
|
||||||
|
|
||||||
|
with patch.object(listener, '_connect', new=AsyncMock(side_effect=OSError("down"))), \
|
||||||
|
patch.object(listener, '_close_connection', new=AsyncMock()), \
|
||||||
|
patch('asyncio.sleep', new=fake_sleep):
|
||||||
|
ok = await listener._reconnect_with_backoff(max_attempts=12)
|
||||||
|
|
||||||
|
assert ok is False
|
||||||
|
assert max(sleeps) == listener.BACKOFF_MAX_SECONDS
|
||||||
|
assert listener.reconnects == 0
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reconnect_records_outage_duration(self, listener):
|
||||||
|
"""The gap is measured so the lost-notification window is reportable."""
|
||||||
|
listener._disconnected_at = datetime.now() - timedelta(seconds=30)
|
||||||
|
|
||||||
|
with patch.object(listener, '_connect', new=AsyncMock()), \
|
||||||
|
patch.object(listener, '_close_connection', new=AsyncMock()):
|
||||||
|
await listener._reconnect_with_backoff()
|
||||||
|
|
||||||
|
assert listener.last_gap_seconds is not None
|
||||||
|
assert 29 <= listener.last_gap_seconds <= 32
|
||||||
|
assert listener._disconnected_at is None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_close_connection_tolerates_dead_connection(self, listener):
|
||||||
|
"""Cleaning up an already-terminated connection must not raise."""
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.is_closed.return_value = False
|
||||||
|
connection.remove_listener = AsyncMock(side_effect=Exception("connection is closed"))
|
||||||
|
connection.close = AsyncMock()
|
||||||
|
listener.connection = connection
|
||||||
|
|
||||||
|
await listener._close_connection()
|
||||||
|
|
||||||
|
assert listener.connection is None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_registers_termination_listener(self, listener):
|
||||||
|
"""
|
||||||
|
The termination listener is re-registered on every connection.
|
||||||
|
|
||||||
|
asyncpg clears its termination listeners as soon as it fires them, so a
|
||||||
|
registration that happened only once would survive exactly one drop.
|
||||||
|
"""
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.add_listener = AsyncMock()
|
||||||
|
connection.is_closed.return_value = False
|
||||||
|
|
||||||
|
with patch('asyncpg.connect', new=AsyncMock(return_value=connection)):
|
||||||
|
await listener._connect()
|
||||||
|
|
||||||
|
connection.add_listener.assert_awaited_once()
|
||||||
|
connection.add_termination_listener.assert_called_once_with(
|
||||||
|
listener._on_connection_lost
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user