jpmschweitzer/library-desk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Wiki Activity

feat: implement check-updates, job-backed ingest status, and dedup scan

Browse Source 
Replace the four stub endpoints with real implementations, all requiring
an explicit tenant user (Phase B rule):

- /ingest/check-updates: GraphService now records a SHA-256 content_hash
  on every Document node at ingestion time; the endpoint compares those
  stored hashes against current Wiki.js page content in one UNWIND Cypher
  query per tenant and returns changed/new/deleted page lists (entity-stub
  pages excluded, pre-hash-tracking documents flagged stored_hash_missing).
- /ingest/status/{job_id}: backed by the Redis JobManager; jobs are
  tenant-scoped (foreign jobs 404). /ingest/page, /ingest/batch and
  /ingest/all now create job records and return job_id.
- /ingest/repo-status/{repository}: wiki page count vs indexed Document
  nodes under users/{tenant}/{repository} plus tenant job stats.
- /deduplicate/check: tenant-scoped Qdrant similarity scan; chunk pairs
  above ~0.9 cosine from different pages grouped per page pair with best
  score and page references (read-only).

Supporting changes: get_job_manager dependency (+ shutdown close),
scroll_all_points can return vectors, VectorService.find_duplicate_pairs,
src/core/hashing.compute_content_hash. 13 new offline unit tests.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
This commit is contained in:
jpmschweitzer
2026-07-14 12:06:21 +02:00
co-authored by Claude Fable 5
parent eae39aff3e
commit 86051d8022
10 changed files with 857 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.py
+243 -53
View File
@@ -11,7 +11,7 @@ Following best practices:
from fastapi import FastAPI, HTTPException, Depends, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from pydantic import BaseModel, Field
from typing import Dict, Any
import logging
from pathlib import Path
@@ -19,8 +19,9 @@ from pathlib import Path
from src.config import Settings, get_settings, __version__
from src.core.dependencies import (
verify_api_key, QdrantDep, WikiJSDep, OllamaDep, Neo4jDep, PaperlessDep,
RequiredUserQuery
RequiredUserQuery, JobManagerDep
)
from src.core.multi_tenancy import RequiredUser
# Configure logging
logging.basicConfig(
@@ -228,61 +229,227 @@ async def stats(
)
class CheckUpdatesRequest(BaseModel):
"""Request body for /ingest/check-updates."""
user: RequiredUser = Field(
...,
description="User identifier (tenant). Required — only this tenant's namespace is compared."
)
path_prefix: str | None = Field(
default=None,
description="Optional sub-path inside the tenant namespace (e.g. 'technology')"
)
@app.post("/ingest/check-updates", tags=["Ingestion"])
async def check_updates(
documents: Dict[str, Any],
request: CheckUpdatesRequest,
neo4j: Neo4jDep = None,
wikijs: WikiJSDep = None,
api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
"""
Check which documents need updating based on content hashes.
Used by Scheduler to determine what changed since last sync.
Check which wiki pages need (re-)ingestion based on content hashes.
TODO: Implement update detection:
1. Query existing documents by path
2. Compare content hashes
3. Return list of updates needed
Compares the `content_hash` stored on the tenant's Neo4j Document nodes
(recorded at ingestion time) against the SHA-256 of the current Wiki.js
page content, in a single UNWIND Cypher query. Used by the Scheduler to
determine what changed since the last sync. Read-only.
Returns per-tenant lists:
- `changed`: page exists in wiki AND graph, but hashes differ (or the
stored hash predates hash tracking — flagged `stored_hash_missing`)
- `new`: wiki page with no Document node yet
- `deleted`: Document node whose wiki page no longer exists
"""
return {
"message": "Update checking not yet implemented",
"updates_needed": [],
"up_to_date": [],
"new_documents": []
}
import time as _time
from src.core.hashing import compute_content_hash
from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id
start_time = _time.time()
user = request.user
tenant_prefix = f"users/{sanitize_user_id(user)}"
if request.path_prefix:
tenant_prefix = f"{tenant_prefix}/{request.path_prefix.strip('/')}"
try:
pages = await wikijs.list_all_pages(path_prefix=tenant_prefix)
# Auto-generated entity stubs are intentionally never ingested into
# the graph (see GraphService.update_from_page), so they would show
# up as perpetually "new". Exclude them.
pages = [
p for p in pages
if not ({"entity-stub", "auto-generated"} & set(p.get("tags") or []))
]
page_hashes = []
for p in pages:
full_page = await wikijs.get_page(p["id"])
content = (full_page or {}).get("content", "")
page_hashes.append({
"page_id": p["id"],
"path": p.get("path", ""),
"title": p.get("title", ""),
"hash": compute_content_hash(content)
})
user_doc_label = get_neo4j_user_label(user)
if page_hashes:
# Single UNWIND query: compare every current page hash against the
# stored Document hash AND collect stale Document nodes whose wiki
# page is gone.
cypher = f"""
UNWIND $pages AS p
OPTIONAL MATCH (d:{user_doc_label}:Document {{page_id: p.page_id}})
WITH collect({{
page_id: p.page_id,
path: p.path,
title: p.title,
is_new: d IS NULL,
changed: d IS NOT NULL AND (d.content_hash IS NULL OR d.content_hash <> p.hash),
stored_hash_missing: d IS NOT NULL AND d.content_hash IS NULL
}}) AS checked,
collect(p.page_id) AS current_ids
OPTIONAL MATCH (stale:{user_doc_label}:Document)
WHERE stale.page_id IS NOT NULL AND NOT stale.page_id IN current_ids
RETURN checked,
collect(CASE WHEN stale IS NULL THEN NULL ELSE {{
page_id: stale.page_id, path: stale.path, title: stale.title
}} END) AS deleted
"""
rows = await neo4j.execute_query(cypher, {"pages": page_hashes})
checked = rows[0]["checked"] if rows else []
deleted = rows[0]["deleted"] if rows else []
else:
# No wiki pages under the prefix: every Document node is stale.
cypher = f"""
MATCH (stale:{user_doc_label}:Document)
WHERE stale.page_id IS NOT NULL
RETURN collect({{page_id: stale.page_id, path: stale.path, title: stale.title}}) AS deleted
"""
rows = await neo4j.execute_query(cypher, {})
checked = []
deleted = rows[0]["deleted"] if rows else []
# Deleted detection is namespace-wide only for full-tenant scans; a
# sub-path scan must not flag documents outside its prefix.
if request.path_prefix:
deleted = [
d for d in deleted
if str(d.get("path", "")).lstrip("/").startswith(tenant_prefix)
]
new_pages = [c for c in checked if c["is_new"]]
changed_pages = [c for c in checked if c["changed"]]
up_to_date = len(checked) - len(new_pages) - len(changed_pages)
duration_ms = (_time.time() - start_time) * 1000
return {
"user": user,
"path_prefix": tenant_prefix,
"total_wiki_pages": len(page_hashes),
"changed": [
{k: c[k] for k in ("page_id", "path", "title", "stored_hash_missing")}
for c in changed_pages
],
"new": [
{k: c[k] for k in ("page_id", "path", "title")} for c in new_pages
],
"deleted": deleted,
"counts": {
"changed": len(changed_pages),
"new": len(new_pages),
"deleted": len(deleted),
"up_to_date": up_to_date
},
"duration_ms": duration_ms
}
except Exception as e:
logger.error(f"check-updates failed for {user}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Update check failed")
@app.get("/ingest/status/{document_id}", tags=["Ingestion"])
@app.get("/ingest/status/{job_id}", tags=["Ingestion"])
async def get_ingestion_status(
document_id: str,
job_id: str,
user: RequiredUserQuery,
job_manager: JobManagerDep = None,
api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
"""
Get processing status for a document.
Get processing status for an ingestion job.
TODO: Implement status tracking
Backed by the Redis job store (`library:job:{job_id}`, 24h TTL). Job IDs
are returned by /ingest/page, /ingest/batch and /ingest/all. Jobs are
tenant-scoped: requesting another tenant's job returns 404.
"""
return {
"message": "Status tracking not yet implemented",
"document_id": document_id,
"status": "unknown"
}
job = await job_manager.get_job(job_id)
if not job or job.get("user") != user:
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
return job
@app.get("/ingest/repo-status/{repository}", tags=["Ingestion"])
async def get_repo_status(
repository: str,
user: RequiredUserQuery,
neo4j: Neo4jDep = None,
wikijs: WikiJSDep = None,
job_manager: JobManagerDep = None,
api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
"""
Get indexing status for an entire repository.
Get indexing status for a repository (a sub-path of the tenant namespace).
TODO: Implement repository-level statistics
`repository` is resolved as `users/{tenant}/{repository}`; use `_all` for
the whole tenant namespace. Reports how many wiki pages exist under the
path, how many have graph Document nodes (i.e. are indexed), and the
tenant's recent job statistics from the Redis job store.
"""
return {
"message": "Repository status not yet implemented",
"repository": repository,
"total_documents": 0,
"indexed_documents": 0
}
import time as _time
from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id
start_time = _time.time()
tenant_root = f"users/{sanitize_user_id(user)}"
prefix = tenant_root if repository in ("_all", "all", "") else f"{tenant_root}/{repository.strip('/')}"
try:
pages = await wikijs.list_all_pages(path_prefix=prefix)
page_ids = [p["id"] for p in pages if p.get("id")]
indexed = 0
if page_ids:
user_doc_label = get_neo4j_user_label(user)
rows = await neo4j.execute_query(
f"""
MATCH (d:{user_doc_label}:Document)
WHERE d.page_id IN $page_ids
RETURN count(DISTINCT d.page_id) AS indexed
""",
{"page_ids": page_ids}
)
indexed = rows[0]["indexed"] if rows else 0
job_stats = await job_manager.get_job_stats(user=user)
return {
"repository": repository,
"user": user,
"path_prefix": prefix,
"total_documents": len(page_ids),
"indexed_documents": indexed,
"unindexed_documents": len(page_ids) - indexed,
"jobs": job_stats,
"duration_ms": (_time.time() - start_time) * 1000
}
except Exception as e:
logger.error(f"repo-status failed for {user}/{repository}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Repository status failed")
# Query endpoints
@@ -373,35 +540,58 @@ async def graph_query(
# Deduplication endpoints
class DeduplicateCheckRequest(BaseModel):
"""Request body for /deduplicate/check."""
user: RequiredUser = Field(
...,
description="User identifier (tenant). Required — only this tenant's collection is scanned."
)
similarity_threshold: float = Field(
default=0.9, ge=0.5, le=1.0,
description="Minimum cosine similarity for a chunk pair to count as duplicate"
)
max_pairs: int = Field(default=100, ge=1, le=500, description="Maximum page pairs returned")
@app.post("/deduplicate/check", tags=["Deduplication"])
async def check_duplicates(
request: Dict[str, Any],
request: DeduplicateCheckRequest,
qdrant_client: QdrantDep = None,
wiki_client: WikiJSDep = None,
ollama_client: OllamaDep = None,
api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
"""
Check for duplicate or highly similar documents.
Uses vector similarity and graph analysis.
Check for duplicate or highly similar wiki pages (tenant-scoped, read-only).
Expected fields:
- document_id: str
- similarity_threshold: float (default 0.85)
TODO: Implement deduplication:
1. Get document embedding from Qdrant
2. Find similar vectors above threshold
3. Check graph relationships
4. Return candidates with similarity scores
Scans the tenant's own Qdrant collection: every wiki chunk vector is
queried against the same collection, and chunk pairs from different
pages scoring above the threshold (default 0.9 cosine) are grouped per
page pair with the best similarity and matching chunk-pair count.
"""
document_id = request.get("document_id")
threshold = request.get("similarity_threshold", 0.85)
import time as _time
from src.services.vector_service import VectorService
return {
"message": "Deduplication not yet implemented",
"document_id": document_id,
"threshold": threshold,
"duplicates": [],
"suggestions": None
}
start_time = _time.time()
vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
try:
scan = await vector_service.find_duplicate_pairs(
user=request.user,
similarity_threshold=request.similarity_threshold,
max_pairs=request.max_pairs
)
return {
"user": request.user,
"similarity_threshold": request.similarity_threshold,
"chunks_scanned": scan["chunks_scanned"],
"duplicate_groups": scan["duplicate_groups"],
"duplicate_group_count": len(scan["duplicate_groups"]),
"duration_ms": (_time.time() - start_time) * 1000
}
except Exception as e:
logger.error(f"Deduplication check failed for {request.user}: {e}", exc_info=True)
raise HTTPException(status_code=500, detail="Deduplication check failed")
# Application lifecycle