Make the governed RAG evidence path executable end to end
Some checks failed
verify / verify (push) Has been cancelled
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
This commit is contained in:
296
backend/app/adapters/local_storage.py
Normal file
296
backend/app/adapters/local_storage.py
Normal file
@@ -0,0 +1,296 @@
|
||||
"""Fail-closed local upload storage with bounded atomic writes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import uuid
|
||||
from collections.abc import AsyncIterable
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
_SHA256_PATTERN: Final = re.compile(r"^[0-9a-f]{64}$")
|
||||
_WRITE_MODE: Final = 0o600
|
||||
_FINAL_MODE: Final = 0o440
|
||||
|
||||
|
||||
class StorageErrorCode(StrEnum):
|
||||
INVALID_CONTRACT = "INVALID_CONTRACT"
|
||||
TOO_LARGE = "TOO_LARGE"
|
||||
SIZE_MISMATCH = "SIZE_MISMATCH"
|
||||
HASH_MISMATCH = "HASH_MISMATCH"
|
||||
OBJECT_CONFLICT = "OBJECT_CONFLICT"
|
||||
ROOT_UNSAFE = "ROOT_UNSAFE"
|
||||
IO_UNAVAILABLE = "IO_UNAVAILABLE"
|
||||
|
||||
|
||||
_SAFE_MESSAGES: Final[dict[StorageErrorCode, str]] = {
|
||||
StorageErrorCode.INVALID_CONTRACT: "The upload storage contract is invalid.",
|
||||
StorageErrorCode.TOO_LARGE: "The upload exceeds its configured size limit.",
|
||||
StorageErrorCode.SIZE_MISMATCH: "The uploaded byte count does not match its declaration.",
|
||||
StorageErrorCode.HASH_MISMATCH: "The uploaded content digest does not match its declaration.",
|
||||
StorageErrorCode.OBJECT_CONFLICT: "The upload object already exists with different content.",
|
||||
StorageErrorCode.ROOT_UNSAFE: "The configured upload storage root is not safe.",
|
||||
StorageErrorCode.IO_UNAVAILABLE: "The upload storage is temporarily unavailable.",
|
||||
}
|
||||
|
||||
|
||||
class LocalStorageError(RuntimeError):
|
||||
"""A sanitized storage error that never contains paths or upload bytes."""
|
||||
|
||||
def __init__(self, code: StorageErrorCode) -> None:
|
||||
self.code = code
|
||||
super().__init__(_SAFE_MESSAGES[code])
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{type(self).__name__}(code={self.code.value!r})"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StoredUpload:
|
||||
storage_key: uuid.UUID
|
||||
byte_size: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class LocalUploadStorage:
|
||||
"""Store UUID-keyed objects below one absolute, non-symlink root.
|
||||
|
||||
Client filenames are intentionally absent from this adapter. Writes use a
|
||||
same-directory temporary file, fsync, read-only permissions, and atomic
|
||||
replacement. A retry for an already stored identical object is idempotent.
|
||||
"""
|
||||
|
||||
def __init__(self, root: Path, *, max_bytes: int) -> None:
|
||||
if not root.is_absolute() or isinstance(max_bytes, bool) or max_bytes <= 0:
|
||||
raise LocalStorageError(StorageErrorCode.INVALID_CONTRACT)
|
||||
self._root = root
|
||||
self._max_bytes = max_bytes
|
||||
|
||||
async def store(
|
||||
self,
|
||||
*,
|
||||
storage_key: uuid.UUID,
|
||||
chunks: AsyncIterable[bytes],
|
||||
expected_size: int,
|
||||
expected_sha256: str,
|
||||
) -> StoredUpload:
|
||||
self._validate_expectation(expected_size, expected_sha256)
|
||||
try:
|
||||
directory, target = await asyncio.to_thread(self._prepare_target, storage_key)
|
||||
existing = await asyncio.to_thread(
|
||||
self._existing_object,
|
||||
target,
|
||||
storage_key,
|
||||
expected_size,
|
||||
expected_sha256,
|
||||
)
|
||||
except LocalStorageError:
|
||||
raise
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
temporary = directory / f".{storage_key.hex}.{uuid.uuid4().hex}.upload"
|
||||
descriptor: int | None = None
|
||||
total = 0
|
||||
digest = hashlib.sha256()
|
||||
installed = False
|
||||
try:
|
||||
descriptor = await asyncio.to_thread(self._open_temporary, temporary)
|
||||
async for chunk in chunks:
|
||||
if not isinstance(chunk, bytes):
|
||||
raise LocalStorageError(StorageErrorCode.INVALID_CONTRACT)
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > self._max_bytes or total > expected_size:
|
||||
raise LocalStorageError(StorageErrorCode.TOO_LARGE)
|
||||
digest.update(chunk)
|
||||
await asyncio.to_thread(_write_all, descriptor, chunk)
|
||||
|
||||
if total != expected_size:
|
||||
raise LocalStorageError(StorageErrorCode.SIZE_MISMATCH)
|
||||
actual_sha256 = digest.hexdigest()
|
||||
if actual_sha256 != expected_sha256:
|
||||
raise LocalStorageError(StorageErrorCode.HASH_MISMATCH)
|
||||
|
||||
await asyncio.to_thread(os.fsync, descriptor)
|
||||
await asyncio.to_thread(os.close, descriptor)
|
||||
descriptor = None
|
||||
await asyncio.to_thread(os.chmod, temporary, _FINAL_MODE, follow_symlinks=False)
|
||||
await asyncio.to_thread(os.replace, temporary, target)
|
||||
installed = True
|
||||
await asyncio.to_thread(_fsync_directory, directory)
|
||||
return StoredUpload(storage_key, total, actual_sha256)
|
||||
except LocalStorageError:
|
||||
raise
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
try:
|
||||
os.close(descriptor)
|
||||
except OSError:
|
||||
pass
|
||||
if not installed:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def remove(self, storage_key: uuid.UUID) -> None:
|
||||
"""Remove an unreferenced deduplicated upload without exposing its path."""
|
||||
|
||||
try:
|
||||
directory, target = await asyncio.to_thread(self._prepare_target, storage_key)
|
||||
await asyncio.to_thread(target.unlink, missing_ok=True)
|
||||
await asyncio.to_thread(_fsync_directory, directory)
|
||||
except LocalStorageError:
|
||||
raise
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
|
||||
async def read_verified(
|
||||
self,
|
||||
*,
|
||||
storage_key: uuid.UUID,
|
||||
expected_size: int,
|
||||
expected_sha256: str,
|
||||
) -> bytes:
|
||||
"""Read one UUID object while rechecking its immutable size and digest."""
|
||||
|
||||
self._validate_expectation(expected_size, expected_sha256)
|
||||
try:
|
||||
_, target = await asyncio.to_thread(self._prepare_target, storage_key)
|
||||
return await asyncio.to_thread(
|
||||
self._read_verified_object,
|
||||
target,
|
||||
expected_size,
|
||||
expected_sha256,
|
||||
)
|
||||
except LocalStorageError:
|
||||
raise
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
|
||||
def _validate_expectation(self, expected_size: int, expected_sha256: str) -> None:
|
||||
if (
|
||||
isinstance(expected_size, bool)
|
||||
or not 1 <= expected_size <= self._max_bytes
|
||||
or _SHA256_PATTERN.fullmatch(expected_sha256) is None
|
||||
):
|
||||
raise LocalStorageError(StorageErrorCode.INVALID_CONTRACT)
|
||||
|
||||
def _prepare_target(self, storage_key: uuid.UUID) -> tuple[Path, Path]:
|
||||
if not isinstance(storage_key, uuid.UUID):
|
||||
raise LocalStorageError(StorageErrorCode.INVALID_CONTRACT)
|
||||
if self._root.exists() and self._root.is_symlink():
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
self._root.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||
if self._root.is_symlink() or not self._root.is_dir():
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
directory = self._root / storage_key.hex[:2]
|
||||
if directory.exists() and directory.is_symlink():
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
directory.mkdir(mode=0o750, exist_ok=True)
|
||||
if directory.is_symlink() or not directory.is_dir():
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
target = directory / storage_key.hex
|
||||
if target.parent != directory or directory.parent != self._root:
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
return directory, target
|
||||
|
||||
@staticmethod
|
||||
def _open_temporary(path: Path) -> int:
|
||||
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
return os.open(path, flags, _WRITE_MODE)
|
||||
|
||||
@staticmethod
|
||||
def _existing_object(
|
||||
target: Path,
|
||||
storage_key: uuid.UUID,
|
||||
expected_size: int,
|
||||
expected_sha256: str,
|
||||
) -> StoredUpload | None:
|
||||
try:
|
||||
metadata = target.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
if metadata.st_size != expected_size:
|
||||
raise LocalStorageError(StorageErrorCode.OBJECT_CONFLICT)
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with target.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
if digest.hexdigest() != expected_sha256:
|
||||
raise LocalStorageError(StorageErrorCode.OBJECT_CONFLICT)
|
||||
return StoredUpload(storage_key, expected_size, expected_sha256)
|
||||
|
||||
@staticmethod
|
||||
def _read_verified_object(
|
||||
target: Path,
|
||||
expected_size: int,
|
||||
expected_sha256: str,
|
||||
) -> bytes:
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
flags |= os.O_NOFOLLOW
|
||||
try:
|
||||
descriptor = os.open(target, flags)
|
||||
except OSError:
|
||||
raise LocalStorageError(StorageErrorCode.IO_UNAVAILABLE) from None
|
||||
try:
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise LocalStorageError(StorageErrorCode.ROOT_UNSAFE)
|
||||
if metadata.st_size != expected_size:
|
||||
raise LocalStorageError(StorageErrorCode.OBJECT_CONFLICT)
|
||||
value = bytearray()
|
||||
digest = hashlib.sha256()
|
||||
while len(value) < expected_size:
|
||||
chunk = os.read(descriptor, min(1024 * 1024, expected_size - len(value)))
|
||||
if not chunk:
|
||||
break
|
||||
value.extend(chunk)
|
||||
digest.update(chunk)
|
||||
if len(value) != expected_size or os.read(descriptor, 1):
|
||||
raise LocalStorageError(StorageErrorCode.OBJECT_CONFLICT)
|
||||
if digest.hexdigest() != expected_sha256:
|
||||
raise LocalStorageError(StorageErrorCode.OBJECT_CONFLICT)
|
||||
return bytes(value)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _write_all(descriptor: int, value: bytes) -> None:
|
||||
view = memoryview(value)
|
||||
while view:
|
||||
written = os.write(descriptor, view)
|
||||
if written <= 0:
|
||||
raise OSError("bounded upload write made no progress")
|
||||
view = view[written:]
|
||||
|
||||
|
||||
def _fsync_directory(directory: Path) -> None:
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_DIRECTORY"):
|
||||
flags |= os.O_DIRECTORY
|
||||
descriptor = os.open(directory, flags)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
Reference in New Issue
Block a user