mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-14 20:22:22 +02:00
fix(stabilization): harden attachment lifecycle and agent guard signals (#5420)
* fix: harden stabilization attachment and agent guards * fix(uploads): preserve durable references during cleanup * fix(uploads): close cleanup and compaction races
This commit is contained in:
falabellamichael
+652
-109
@@ -35,11 +35,34 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UploadCleanupSafetyError(RuntimeError):
|
||||
"""Raised when cleanup cannot prove that destructive work is safe."""
|
||||
|
||||
# The extension is optional: save_upload builds the id as `{uuid.hex}{ext}`,
|
||||
# and a file with no extension (Dockerfile, README, ...) yields a bare 32-hex
|
||||
# id. Requiring `.ext` made those ids fail validation, so the stored file
|
||||
# could never be resolved or downloaded again.
|
||||
UPLOAD_ID_RE = re.compile(r"^[0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?$")
|
||||
UPLOAD_ID_TOKEN_RE = re.compile(
|
||||
r"(?<![0-9a-fA-F])([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)(?![A-Za-z0-9])"
|
||||
)
|
||||
INTERNAL_UPLOAD_URL_RE = re.compile(
|
||||
r"(?:odysseus://attachment/|/api/upload/)"
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?=$|[\s\"'<>\[\](){},;!?:&#]|\.(?![A-Za-z0-9]))"
|
||||
)
|
||||
PDF_SOURCE_UPLOAD_RE = re.compile(
|
||||
r"<!--\s*pdf(?:_form)?_source\b[^>]*\bupload_id="
|
||||
r"[\"']([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)[\"'][^>]*-->",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
ATTACHMENT_REFERENCE_LINE_RE = re.compile(
|
||||
r"\[Attachment:[^\]\r\n]*\|\s*id="
|
||||
r"([0-9a-fA-F]{32}(?:\.[A-Za-z0-9]+)?)"
|
||||
r"(?:\s*\||\s*\])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def is_valid_upload_id(upload_id: str) -> bool:
|
||||
@@ -47,6 +70,110 @@ def is_valid_upload_id(upload_id: str) -> bool:
|
||||
return UPLOAD_ID_RE.fullmatch(upload_id or "") is not None
|
||||
|
||||
|
||||
def extract_upload_ids(value: Any) -> set[str]:
|
||||
"""Return canonical upload IDs embedded in a persisted URL/text value."""
|
||||
if not isinstance(value, str) or not value:
|
||||
return set()
|
||||
return set(UPLOAD_ID_TOKEN_RE.findall(value))
|
||||
|
||||
|
||||
def extract_internal_upload_ids(value: Any) -> set[str]:
|
||||
"""Return IDs from explicit internal upload references only.
|
||||
|
||||
Cleanup intentionally uses :func:`extract_upload_ids` conservatively, but
|
||||
write-time reservation must not treat an arbitrary 32-hex checksum in note
|
||||
or calendar text as an upload reference. Nested JSON-like values are
|
||||
supported because note checklist items are persisted as structured data.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
found: set[str] = set()
|
||||
for nested in value.values():
|
||||
found.update(extract_internal_upload_ids(nested))
|
||||
return found
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
found: set[str] = set()
|
||||
for nested in value:
|
||||
found.update(extract_internal_upload_ids(nested))
|
||||
return found
|
||||
if not isinstance(value, str) or not value:
|
||||
return set()
|
||||
return (
|
||||
set(INTERNAL_UPLOAD_URL_RE.findall(value))
|
||||
| set(PDF_SOURCE_UPLOAD_RE.findall(value))
|
||||
| set(ATTACHMENT_REFERENCE_LINE_RE.findall(value))
|
||||
)
|
||||
|
||||
|
||||
def reserve_upload_references(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
*values: Any,
|
||||
) -> Optional[str]:
|
||||
"""Reserve upload IDs in values before a caller persists references.
|
||||
|
||||
Returns the first ID that cannot be owner-checked/reserved, otherwise
|
||||
``None``. A missing handler is treated as no-op for backward-compatible
|
||||
route factories; production wires the shared UploadHandler instance.
|
||||
"""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
upload_ids: set[str] = set()
|
||||
for value in values:
|
||||
upload_ids.update(extract_internal_upload_ids(value))
|
||||
return reserve_upload_ids(upload_handler, owner, upload_ids)
|
||||
|
||||
|
||||
def reserve_upload_ids(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
upload_ids: Any,
|
||||
) -> Optional[str]:
|
||||
"""Owner-reserve canonical IDs from a trusted structured reference field."""
|
||||
if upload_handler is None:
|
||||
return None
|
||||
canonical_ids = {
|
||||
str(upload_id).strip()
|
||||
for upload_id in (upload_ids or [])
|
||||
if is_valid_upload_id(str(upload_id).strip())
|
||||
}
|
||||
for upload_id in sorted(canonical_ids):
|
||||
try:
|
||||
resolved = upload_handler.reserve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
allow_admin=False,
|
||||
)
|
||||
except Exception:
|
||||
resolved = None
|
||||
if not resolved:
|
||||
return upload_id
|
||||
return None
|
||||
|
||||
|
||||
def reserve_message_upload_references(
|
||||
upload_handler: Any,
|
||||
owner: Optional[str],
|
||||
content: Any,
|
||||
metadata: Any = None,
|
||||
) -> Optional[str]:
|
||||
"""Reserve explicit chat references, including structured attachment IDs."""
|
||||
upload_ids = extract_internal_upload_ids(content)
|
||||
if metadata not in (None, ""):
|
||||
if isinstance(metadata, str):
|
||||
metadata = json.loads(metadata)
|
||||
if not isinstance(metadata, dict):
|
||||
raise ValueError("message metadata must be a JSON object")
|
||||
upload_ids.update(extract_internal_upload_ids(metadata))
|
||||
from src.attachment_refs import attachment_refs_from_metadata
|
||||
|
||||
upload_ids.update(
|
||||
str(ref.get("attachment_id") or "").strip()
|
||||
for ref in attachment_refs_from_metadata(metadata)
|
||||
if ref.get("attachment_id")
|
||||
)
|
||||
return reserve_upload_ids(upload_handler, owner, upload_ids)
|
||||
|
||||
|
||||
def _build_upload_id(safe_filename: str) -> str:
|
||||
"""Build a unique upload id whose extension matches UPLOAD_ID_RE.
|
||||
|
||||
@@ -249,43 +376,295 @@ class UploadHandler:
|
||||
|
||||
return True
|
||||
|
||||
def cleanup_old_uploads(self):
|
||||
"""Remove uploaded files older than CLEANUP_DAYS days."""
|
||||
@staticmethod
|
||||
def _parse_upload_timestamp(value: Any) -> Optional[datetime]:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
cutoff_date = datetime.now() - timedelta(days=self.cleanup_days)
|
||||
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is not None:
|
||||
parsed = parsed.astimezone().replace(tzinfo=None)
|
||||
return parsed
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _upload_metadata_is_recent(cls, info: Dict[str, Any], cutoff_date: datetime) -> bool:
|
||||
"""Return True when upload metadata records activity inside retention."""
|
||||
for field in ("last_accessed", "created_at", "uploaded_at"):
|
||||
parsed = cls._parse_upload_timestamp(info.get(field))
|
||||
if parsed is None:
|
||||
continue
|
||||
if parsed >= cutoff_date:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _upload_index_keys_for_file(
|
||||
cls,
|
||||
upload_index: Dict[str, Any],
|
||||
upload_id: str,
|
||||
file_path: str,
|
||||
) -> list[str]:
|
||||
"""Find a coherent set of index rows for one physical upload.
|
||||
|
||||
Every related row must agree on ID, canonical path, owner, and a
|
||||
non-empty checksum. Each row must also contain the complete lifecycle
|
||||
timestamps written for new uploads. Ambiguous or incomplete index
|
||||
state cannot authorize destructive cleanup.
|
||||
"""
|
||||
target_path = os.path.normcase(os.path.realpath(file_path))
|
||||
matches: list[str] = []
|
||||
owners: set[str] = set()
|
||||
checksums: set[str] = set()
|
||||
for key, info in upload_index.items():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
stored_path = info.get("path")
|
||||
stored_real_path = (
|
||||
os.path.normcase(os.path.realpath(stored_path))
|
||||
if isinstance(stored_path, str) and stored_path
|
||||
else None
|
||||
)
|
||||
same_id = info.get("id") == upload_id
|
||||
same_path = stored_real_path == target_path
|
||||
if not same_id and not same_path:
|
||||
continue
|
||||
if not same_id or not same_path:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: related row has id=%r path=%r",
|
||||
file_path,
|
||||
info.get("id"),
|
||||
stored_path,
|
||||
)
|
||||
return []
|
||||
|
||||
owner = info.get("owner")
|
||||
if not isinstance(owner, str) or not owner.strip():
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row has no owner",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
row_checksums = {
|
||||
str(info.get(field)).strip().lower()
|
||||
for field in ("hash", "checksum_sha256")
|
||||
if info.get(field) is not None and str(info.get(field)).strip()
|
||||
}
|
||||
if not row_checksums:
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row has no checksum",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
if len(row_checksums) != 1:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: matching row has conflicting checksums",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
lifecycle_fields = ("uploaded_at", "created_at", "last_accessed")
|
||||
if any(
|
||||
cls._parse_upload_timestamp(info.get(field)) is None
|
||||
for field in lifecycle_fields
|
||||
):
|
||||
logger.warning(
|
||||
"Skipping incomplete cleanup candidate %s: matching row lacks lifecycle timestamps",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
|
||||
matches.append(key)
|
||||
owners.add(owner)
|
||||
checksums.update(row_checksums)
|
||||
|
||||
if len(owners) > 1 or len(checksums) > 1:
|
||||
logger.warning(
|
||||
"Skipping ambiguous cleanup candidate %s: matching rows disagree on owner or checksum",
|
||||
file_path,
|
||||
)
|
||||
return []
|
||||
return matches
|
||||
|
||||
def cleanup_old_uploads(
|
||||
self,
|
||||
referenced_upload_ids: Optional[set[str]] = None,
|
||||
referenced_upload_hashes: Optional[set[str]] = None,
|
||||
):
|
||||
"""Remove expired uploads proven unreferenced by a complete snapshot.
|
||||
|
||||
``None`` means reference discovery was not completed, so cleanup fails
|
||||
closed and removes nothing. The admin route supplies both sets after
|
||||
scanning persisted chats, documents, and gallery records.
|
||||
"""
|
||||
if referenced_upload_ids is None or referenced_upload_hashes is None:
|
||||
logger.warning("Upload cleanup skipped: persisted reference snapshot unavailable")
|
||||
return 0
|
||||
|
||||
try:
|
||||
cleanup_started_at = datetime.now()
|
||||
cutoff_date = cleanup_started_at - timedelta(days=self.cleanup_days)
|
||||
cleaned_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(self.upload_dir):
|
||||
if root == self.upload_dir:
|
||||
continue
|
||||
|
||||
path_parts = root.split(os.sep)
|
||||
if len(path_parts) >= 4:
|
||||
|
||||
referenced_ids = {str(value) for value in referenced_upload_ids}
|
||||
referenced_hashes = {str(value) for value in referenced_upload_hashes}
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
|
||||
# Keep index mutation and file removal serialized with upload writes.
|
||||
# Each row removal is atomically persisted before the bytes are
|
||||
# deleted; if deletion fails, the previous index is restored.
|
||||
with self._index_lock:
|
||||
current_index = dict(self._load_upload_index(fail_on_error=True))
|
||||
|
||||
for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
is_junction = getattr(os.path, "isjunction", lambda _path: False)
|
||||
dirs[:] = [
|
||||
directory
|
||||
for directory in dirs
|
||||
if not os.path.islink(os.path.join(root, directory))
|
||||
and not is_junction(os.path.join(root, directory))
|
||||
]
|
||||
if root == self.upload_dir:
|
||||
continue
|
||||
if not self._inside_upload_dir(root):
|
||||
dirs[:] = []
|
||||
continue
|
||||
|
||||
path_parts = root.split(os.sep)
|
||||
if len(path_parts) < 4:
|
||||
continue
|
||||
try:
|
||||
dir_date = datetime(int(path_parts[-3]), int(path_parts[-2]), int(path_parts[-1]))
|
||||
if dir_date < cutoff_date:
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
try:
|
||||
os.remove(file_path)
|
||||
cleaned_count += 1
|
||||
logger.info(f"Cleaned up old upload: {file_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove {file_path}: {e}")
|
||||
|
||||
try:
|
||||
os.rmdir(root)
|
||||
logger.info(f"Removed empty upload directory: {root}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to remove directory {root}: {e}")
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
if dir_date >= cutoff_date:
|
||||
continue
|
||||
|
||||
for file in files:
|
||||
# Reference discovery only understands canonical upload
|
||||
# IDs; unknown files fail closed instead of being swept.
|
||||
if not self.validate_upload_id(file):
|
||||
continue
|
||||
|
||||
file_path = os.path.join(root, file)
|
||||
if not self._inside_upload_dir(file_path):
|
||||
logger.warning(
|
||||
"Skipping cleanup candidate outside upload directory: %s",
|
||||
file_path,
|
||||
)
|
||||
continue
|
||||
matching_keys = self._upload_index_keys_for_file(
|
||||
current_index,
|
||||
file,
|
||||
file_path,
|
||||
)
|
||||
matching_rows = [
|
||||
current_index[key]
|
||||
for key in matching_keys
|
||||
if isinstance(current_index.get(key), dict)
|
||||
]
|
||||
|
||||
# Files without authoritative live index rows are not
|
||||
# eligible for destructive cleanup. Reference hashes,
|
||||
# recency, and ownership cannot be proven for them.
|
||||
if not matching_rows:
|
||||
continue
|
||||
|
||||
is_referenced = file in referenced_ids or any(
|
||||
str(info.get("id") or "") in referenced_ids
|
||||
or str(info.get("hash") or "") in referenced_hashes
|
||||
or str(info.get("checksum_sha256") or "") in referenced_hashes
|
||||
for info in matching_rows
|
||||
)
|
||||
metadata_is_recent = any(
|
||||
self._upload_metadata_is_recent(info, cutoff_date)
|
||||
for info in matching_rows
|
||||
)
|
||||
if is_referenced or metadata_is_recent:
|
||||
continue
|
||||
|
||||
reduced_index = {
|
||||
key: value
|
||||
for key, value in current_index.items()
|
||||
if key not in matching_keys
|
||||
}
|
||||
if matching_keys:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
reduced_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception as e:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload indexes after reconciliation failed for %s",
|
||||
file_path,
|
||||
)
|
||||
raise UploadCleanupSafetyError(
|
||||
"upload index rollback failed before file removal"
|
||||
) from e
|
||||
logger.warning(
|
||||
"Failed to reconcile upload index before removing %s: %s",
|
||||
file_path,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
# The bytes are already absent. Keep the reduced
|
||||
# lifecycle index instead of recreating a stale row.
|
||||
current_index = reduced_index
|
||||
logger.info(
|
||||
"Reconciled missing expired upload from index: %s",
|
||||
file_path,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
if matching_keys:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload index after removal failed for %s",
|
||||
file_path,
|
||||
)
|
||||
raise UploadCleanupSafetyError(
|
||||
"upload index rollback failed after file removal was refused"
|
||||
) from e
|
||||
logger.warning(f"Failed to remove {file_path}: {e}")
|
||||
continue
|
||||
|
||||
current_index = reduced_index
|
||||
cleaned_count += 1
|
||||
logger.info(f"Cleaned up old unreferenced upload: {file_path}")
|
||||
|
||||
try:
|
||||
if not os.listdir(root):
|
||||
os.rmdir(root)
|
||||
logger.info(f"Removed empty upload directory: {root}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to inspect/remove directory {root}: {e}")
|
||||
|
||||
logger.info(f"Upload cleanup completed: {cleaned_count} files removed")
|
||||
return cleaned_count
|
||||
except Exception as e:
|
||||
logger.error(f"Upload cleanup failed: {e}")
|
||||
return 0
|
||||
raise UploadCleanupSafetyError("upload cleanup safety checks failed") from e
|
||||
|
||||
def validate_upload_id(self, upload_id: str) -> bool:
|
||||
"""Validate that the upload ID matches the expected pattern."""
|
||||
@@ -293,71 +672,103 @@ class UploadHandler:
|
||||
|
||||
def _inside_upload_dir(self, path: str) -> bool:
|
||||
"""Check if path is inside the upload directory."""
|
||||
base = os.path.realpath(self.upload_dir)
|
||||
p = os.path.realpath(path)
|
||||
base = os.path.normcase(os.path.realpath(self.upload_dir))
|
||||
p = os.path.normcase(os.path.realpath(path))
|
||||
try:
|
||||
return os.path.commonpath([base, p]) == base
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _atomic_write_json(self, path: str, data: dict) -> None:
|
||||
def _atomic_write_json(
|
||||
self,
|
||||
path: str,
|
||||
data: dict,
|
||||
*,
|
||||
sync_backup: bool = False,
|
||||
) -> None:
|
||||
"""Write `data` to `path` atomically: write to a temp file in the
|
||||
same directory, then `os.replace` onto the target. The kernel
|
||||
guarantees `os.replace` is atomic on POSIX, so a reader either
|
||||
sees the old contents or the new contents, never a half-written
|
||||
file. Also keeps a `.bak` sibling of the previous good state.
|
||||
file. Normally `.bak` retains the previous good state. Destructive
|
||||
lifecycle transitions use ``sync_backup=True`` so recovery cannot
|
||||
resurrect metadata for bytes that were deliberately removed.
|
||||
"""
|
||||
directory = os.path.dirname(path) or "."
|
||||
fd, tmp = tempfile.mkstemp(prefix=".uploads-", suffix=".tmp", dir=directory)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
if os.path.exists(path):
|
||||
bak = path + ".bak"
|
||||
|
||||
def _replace_json(target: str) -> None:
|
||||
fd, tmp = tempfile.mkstemp(
|
||||
prefix=".uploads-",
|
||||
suffix=".tmp",
|
||||
dir=directory,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, target)
|
||||
except Exception:
|
||||
try:
|
||||
shutil.copy2(path, bak)
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, path)
|
||||
# Update cache if this is the main index
|
||||
if path.endswith("uploads.json"):
|
||||
self._index_cache = data
|
||||
try:
|
||||
self._index_mtime = os.path.getmtime(path)
|
||||
except OSError:
|
||||
self._index_mtime = time.time()
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
if sync_backup:
|
||||
_replace_json(path + ".bak")
|
||||
elif os.path.exists(path):
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
shutil.copy2(path, path + ".bak")
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _load_upload_index(self) -> Dict[str, Any]:
|
||||
_replace_json(path)
|
||||
# Update cache if this is the main index
|
||||
if path.endswith("uploads.json"):
|
||||
self._index_cache = data
|
||||
try:
|
||||
self._index_mtime = os.path.getmtime(path)
|
||||
except OSError:
|
||||
self._index_mtime = time.time()
|
||||
|
||||
def _load_upload_index(self, *, fail_on_error: bool = False) -> Dict[str, Any]:
|
||||
"""Load the upload index from disk/cache. Uses mtime-based validation
|
||||
to avoid redundant parsing on hot paths.
|
||||
to avoid redundant parsing on hot paths. When ``fail_on_error`` is
|
||||
true, a missing, malformed, or unreadable live index raises so
|
||||
destructive callers cannot mistake corruption for an empty store.
|
||||
"""
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
if not os.path.exists(uploads_db_path):
|
||||
candidates = (uploads_db_path, uploads_db_path + ".bak")
|
||||
if fail_on_error:
|
||||
# A backup is intentionally the previous snapshot. It is useful for
|
||||
# non-destructive reads, but cannot authorize deletion when the live
|
||||
# index is missing or corrupt.
|
||||
if not os.path.exists(uploads_db_path):
|
||||
raise ValueError("live uploads database is missing")
|
||||
existing_candidates = [uploads_db_path]
|
||||
else:
|
||||
existing_candidates = [path for path in candidates if os.path.exists(path)]
|
||||
if not existing_candidates:
|
||||
self._index_cache = {}
|
||||
self._index_mtime = 0.0
|
||||
return {}
|
||||
|
||||
# Check cache validity
|
||||
try:
|
||||
mtime = os.path.getmtime(uploads_db_path)
|
||||
if self._index_cache is not None and mtime <= self._index_mtime:
|
||||
mtime = max(os.path.getmtime(path) for path in existing_candidates)
|
||||
if (
|
||||
not fail_on_error
|
||||
and self._index_cache is not None
|
||||
and mtime <= self._index_mtime
|
||||
):
|
||||
return self._index_cache
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
|
||||
# Try the live file first, fall back to the .bak sibling if the
|
||||
# live file is truncated/corrupted.
|
||||
for candidate in (uploads_db_path, uploads_db_path + ".bak"):
|
||||
if not os.path.exists(candidate):
|
||||
continue
|
||||
for candidate in existing_candidates:
|
||||
try:
|
||||
with open(candidate, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
@@ -369,6 +780,8 @@ class UploadHandler:
|
||||
logger.warning(f"Failed to read uploads database ({candidate}): {e}")
|
||||
continue
|
||||
|
||||
if fail_on_error:
|
||||
raise ValueError("live uploads database is unreadable")
|
||||
self._index_cache = {}
|
||||
return {}
|
||||
|
||||
@@ -381,6 +794,144 @@ class UploadHandler:
|
||||
return dict(info)
|
||||
return None
|
||||
|
||||
def reserve_upload(
|
||||
self,
|
||||
upload_id: str,
|
||||
*,
|
||||
owner: Optional[str],
|
||||
auth_manager: Any = None,
|
||||
allow_admin: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Owner-check and reserve an indexed upload against cleanup.
|
||||
|
||||
The live index lookup, ownership/path validation, and access touch all
|
||||
occur under the cleanup lock. A durable-reference writer must not
|
||||
commit when this returns ``None``.
|
||||
"""
|
||||
if not self.validate_upload_id(upload_id):
|
||||
return None
|
||||
|
||||
auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
|
||||
if auth_configured and not owner:
|
||||
return None
|
||||
|
||||
uploads_db_path = os.path.join(self.upload_dir, "uploads.json")
|
||||
with self._index_lock:
|
||||
try:
|
||||
current = dict(self._load_upload_index(fail_on_error=True))
|
||||
except Exception:
|
||||
logger.warning("Cannot reserve upload %s without a valid live index", upload_id)
|
||||
return None
|
||||
matching_keys = [
|
||||
key
|
||||
for key, info in current.items()
|
||||
if isinstance(info, dict) and info.get("id") == upload_id
|
||||
]
|
||||
if not matching_keys:
|
||||
return None
|
||||
|
||||
matching_rows = [dict(current[key]) for key in matching_keys]
|
||||
row_owners = {
|
||||
str(row.get("owner")) if row.get("owner") is not None else None
|
||||
for row in matching_rows
|
||||
}
|
||||
row_hashes = {
|
||||
str(row.get("hash") or row.get("checksum_sha256"))
|
||||
for row in matching_rows
|
||||
if row.get("hash") or row.get("checksum_sha256")
|
||||
}
|
||||
if len(row_owners) != 1 or len(row_hashes) > 1:
|
||||
logger.warning(
|
||||
"Cannot reserve ambiguous upload index rows for %s",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
|
||||
is_admin = False
|
||||
if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
|
||||
try:
|
||||
is_admin = bool(auth_manager.is_admin(owner))
|
||||
except Exception:
|
||||
is_admin = False
|
||||
|
||||
now = datetime.now()
|
||||
current_info = matching_rows[0]
|
||||
if owner and not is_admin and current_info.get("owner") != owner:
|
||||
return None
|
||||
if not owner and current_info.get("owner") is not None:
|
||||
return None
|
||||
|
||||
existing_paths: set[str] = set()
|
||||
for row in matching_rows:
|
||||
stored_path = row.get("path")
|
||||
if not stored_path:
|
||||
continue
|
||||
if not self._inside_upload_dir(stored_path):
|
||||
logger.warning(
|
||||
"Cannot reserve upload %s with an out-of-root index path",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
if os.path.isfile(stored_path):
|
||||
if os.path.basename(stored_path) != upload_id:
|
||||
return None
|
||||
existing_paths.add(os.path.normcase(os.path.realpath(stored_path)))
|
||||
if len(existing_paths) > 1:
|
||||
logger.warning("Cannot reserve upload %s with multiple indexed paths", upload_id)
|
||||
return None
|
||||
path = next(iter(existing_paths), None) or self._find_upload_path(upload_id)
|
||||
if not path or not os.path.isfile(path) or not self._inside_upload_dir(path):
|
||||
return None
|
||||
|
||||
last_accessed = self._parse_upload_timestamp(current_info.get("last_accessed"))
|
||||
path_changed = current_info.get("path") != path
|
||||
needs_write = (
|
||||
path_changed
|
||||
or last_accessed is None
|
||||
or last_accessed < now - timedelta(minutes=5)
|
||||
)
|
||||
if needs_write:
|
||||
accessed_at = now.isoformat()
|
||||
updated_index = dict(current)
|
||||
for key in matching_keys:
|
||||
updated = dict(updated_index[key])
|
||||
updated["path"] = path
|
||||
updated["last_accessed"] = accessed_at
|
||||
updated_index[key] = updated
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
updated_index,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
self._atomic_write_json(
|
||||
uploads_db_path,
|
||||
current,
|
||||
sync_backup=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to restore upload indexes after reservation failed for %s",
|
||||
upload_id,
|
||||
)
|
||||
logger.exception("Failed to reserve upload %s against cleanup", upload_id)
|
||||
return None
|
||||
current_info = dict(updated_index[matching_keys[0]])
|
||||
|
||||
resolved = dict(current_info)
|
||||
resolved.setdefault("id", upload_id)
|
||||
resolved["path"] = path
|
||||
resolved.setdefault("name", os.path.basename(path))
|
||||
resolved.setdefault("original_name", resolved["name"])
|
||||
resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
|
||||
if resolved.get("hash") and not resolved.get("checksum_sha256"):
|
||||
resolved["checksum_sha256"] = resolved["hash"]
|
||||
if resolved.get("uploaded_at") and not resolved.get("created_at"):
|
||||
resolved["created_at"] = resolved["uploaded_at"]
|
||||
return resolved
|
||||
|
||||
def _renamed_upload_index_key(self, key: str, info: Dict[str, Any], old_owner: str, new_owner: str) -> str:
|
||||
"""Return the storage key to use after renaming an owned upload row.
|
||||
|
||||
@@ -475,16 +1026,32 @@ class UploadHandler:
|
||||
if not self.validate_upload_id(upload_id):
|
||||
return None
|
||||
|
||||
candidates: list[str] = []
|
||||
direct = os.path.join(self.upload_dir, upload_id)
|
||||
if os.path.exists(direct) and self._inside_upload_dir(direct):
|
||||
return direct
|
||||
if os.path.isfile(direct) and self._inside_upload_dir(direct):
|
||||
candidates.append(os.path.realpath(direct))
|
||||
|
||||
for root, _dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
for root, dirs, files in os.walk(self.upload_dir, followlinks=False):
|
||||
is_junction = getattr(os.path, "isjunction", lambda _path: False)
|
||||
dirs[:] = [
|
||||
directory
|
||||
for directory in dirs
|
||||
if not os.path.islink(os.path.join(root, directory))
|
||||
and not is_junction(os.path.join(root, directory))
|
||||
]
|
||||
if upload_id in files:
|
||||
path = os.path.join(root, upload_id)
|
||||
if self._inside_upload_dir(path):
|
||||
return path
|
||||
return None
|
||||
if os.path.isfile(path) and self._inside_upload_dir(path):
|
||||
real_path = os.path.realpath(path)
|
||||
if real_path not in candidates:
|
||||
candidates.append(real_path)
|
||||
if len(candidates) > 1:
|
||||
logger.warning(
|
||||
"Upload ID %s resolves to multiple physical files",
|
||||
upload_id,
|
||||
)
|
||||
return None
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
def resolve_upload(
|
||||
self,
|
||||
@@ -493,52 +1060,20 @@ class UploadHandler:
|
||||
auth_manager: Any = None,
|
||||
allow_admin: bool = True,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Resolve an upload ID to metadata only if the caller may read it.
|
||||
"""Resolve and reserve an upload only if the caller may read it.
|
||||
|
||||
This is the owner-aware lookup used by internal processors. Public
|
||||
download routes already perform owner checks; chat/document paths must
|
||||
do the same before reading file bytes server-side.
|
||||
do the same before reading file bytes server-side. Reservation shares
|
||||
cleanup's lifecycle lock and prevents a newly persisted reference from
|
||||
racing final deletion.
|
||||
"""
|
||||
if not self.validate_upload_id(upload_id):
|
||||
logger.warning(f"Invalid upload ID format: {upload_id}")
|
||||
return None
|
||||
|
||||
auth_configured = bool(auth_manager and getattr(auth_manager, "is_configured", False))
|
||||
if auth_configured and not owner:
|
||||
return None
|
||||
|
||||
info = self.get_upload_info(upload_id) or {}
|
||||
is_admin = False
|
||||
if allow_admin and owner and auth_manager and hasattr(auth_manager, "is_admin"):
|
||||
try:
|
||||
is_admin = bool(auth_manager.is_admin(owner))
|
||||
except Exception:
|
||||
is_admin = False
|
||||
|
||||
if owner and not is_admin:
|
||||
if info.get("owner") != owner:
|
||||
logger.warning("Upload %s denied for owner %s", upload_id, owner)
|
||||
return None
|
||||
if not owner and info.get("owner") is not None:
|
||||
logger.warning("Upload %s denied without an authenticated owner", upload_id)
|
||||
return None
|
||||
|
||||
path = info.get("path")
|
||||
if not path or not os.path.exists(path) or not self._inside_upload_dir(path):
|
||||
path = self._find_upload_path(upload_id)
|
||||
if not path:
|
||||
return None
|
||||
if not self._inside_upload_dir(path):
|
||||
logger.warning(f"Upload path outside upload directory: {path}")
|
||||
return None
|
||||
|
||||
resolved = dict(info)
|
||||
resolved.setdefault("id", upload_id)
|
||||
resolved["path"] = path
|
||||
resolved.setdefault("name", os.path.basename(path))
|
||||
resolved.setdefault("original_name", resolved["name"])
|
||||
resolved.setdefault("mime", mimetypes.guess_type(path)[0] or "application/octet-stream")
|
||||
return resolved
|
||||
return self.reserve_upload(
|
||||
upload_id,
|
||||
owner=owner,
|
||||
auth_manager=auth_manager,
|
||||
allow_admin=allow_admin,
|
||||
)
|
||||
|
||||
def cleanup_rate_limits(self):
|
||||
"""Remove stale entries from upload_rate_log."""
|
||||
@@ -706,6 +1241,9 @@ class UploadHandler:
|
||||
# fresh-insert path below; release the lock first.
|
||||
raise LookupError("upload entry vanished mid-dedupe")
|
||||
existing_file["last_accessed"] = datetime.now().isoformat()
|
||||
existing_file.setdefault("checksum_sha256", file_hash)
|
||||
if existing_file.get("uploaded_at"):
|
||||
existing_file.setdefault("created_at", existing_file["uploaded_at"])
|
||||
current[live_key] = existing_file
|
||||
self._atomic_write_json(uploads_db_path, current)
|
||||
except LookupError:
|
||||
@@ -721,7 +1259,9 @@ class UploadHandler:
|
||||
"size": existing_file["size"],
|
||||
"name": existing_file["original_name"],
|
||||
"hash": file_hash,
|
||||
"checksum_sha256": existing_file.get("checksum_sha256") or file_hash,
|
||||
"uploaded_at": existing_file["uploaded_at"],
|
||||
"created_at": existing_file.get("created_at") or existing_file["uploaded_at"],
|
||||
"owner": existing_file.get("owner"),
|
||||
"width": existing_file.get("width"),
|
||||
"height": existing_file.get("height"),
|
||||
@@ -744,6 +1284,7 @@ class UploadHandler:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to save file: {str(e)}")
|
||||
|
||||
# Create file metadata
|
||||
created_at = datetime.now().isoformat()
|
||||
file_metadata = {
|
||||
"id": file_id,
|
||||
"path": file_path,
|
||||
@@ -751,9 +1292,11 @@ class UploadHandler:
|
||||
"size": file_size,
|
||||
"name": safe_filename,
|
||||
"hash": file_hash,
|
||||
"checksum_sha256": file_hash,
|
||||
"original_name": original_filename,
|
||||
"uploaded_at": datetime.now().isoformat(),
|
||||
"last_accessed": datetime.now().isoformat(),
|
||||
"uploaded_at": created_at,
|
||||
"created_at": created_at,
|
||||
"last_accessed": created_at,
|
||||
"client_ip": client_ip,
|
||||
"owner": owner,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user