YoVinchen
ecdb10c37a
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
518 lines
17 KiB
Python
518 lines
17 KiB
Python
"""Optimistic, manifest-bound document review persistence."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from typing import Literal
|
|
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
|
|
from app.core.config import Settings
|
|
from app.core.secrets import SecretFileError
|
|
from app.persistence.documents import DocumentActor, SafeJob
|
|
|
|
type ReviewDecision = Literal["APPROVE", "REJECT"]
|
|
type ReviewReason = Literal[
|
|
"SYNTHETIC_REVIEW_APPROVED",
|
|
"RIGHTS_NOT_VERIFIED",
|
|
"CONTENT_QUALITY_REJECTED",
|
|
"CLOUD_PROCESSING_REJECTED",
|
|
]
|
|
|
|
_HASH = re.compile(r"^[0-9a-f]{64}$")
|
|
LOGGER = logging.getLogger("geological_rag.document_review")
|
|
|
|
|
|
class DocumentReviewError(RuntimeError):
|
|
"""Base class for safe review persistence failures."""
|
|
|
|
|
|
class DocumentReviewNotFoundError(DocumentReviewError):
|
|
pass
|
|
|
|
|
|
class DocumentReviewConflictError(DocumentReviewError):
|
|
pass
|
|
|
|
|
|
class DocumentReviewStateError(DocumentReviewError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class DocumentReviewResult:
|
|
document_id: uuid.UUID
|
|
document_version_id: uuid.UUID
|
|
decision: ReviewDecision
|
|
review_state: Literal["CLOUD_APPROVED", "REJECTED"]
|
|
review_revision: int
|
|
outbound_manifest_sha256: str | None
|
|
embedding_profile_hash: str | None
|
|
job: SafeJob | None
|
|
|
|
|
|
_LOCK_REVIEW = """
|
|
SELECT
|
|
document.id AS document_id,
|
|
version.id AS document_version_id,
|
|
version.review_state,
|
|
version.review_revision,
|
|
version.status AS version_status,
|
|
version.outbound_manifest_sha256,
|
|
version.expected_chunk_count,
|
|
knowledge_base.active_embedding_profile_hash,
|
|
profile.model AS profile_model,
|
|
profile.dimension AS profile_dimension,
|
|
profile.enabled AS profile_enabled
|
|
FROM rag.documents AS document
|
|
JOIN rag.document_versions AS version
|
|
ON version.id = (
|
|
SELECT candidate.id
|
|
FROM rag.document_versions AS candidate
|
|
WHERE candidate.document_id = document.id
|
|
ORDER BY candidate.created_at DESC, candidate.id DESC
|
|
LIMIT 1
|
|
)
|
|
JOIN rag.knowledge_bases AS knowledge_base
|
|
ON knowledge_base.id = document.knowledge_base_id
|
|
LEFT JOIN rag.model_profiles AS profile
|
|
ON profile.profile_hash = knowledge_base.active_embedding_profile_hash
|
|
AND profile.kind = 'embedding'
|
|
WHERE document.id = %s
|
|
AND document.knowledge_base_id = %s
|
|
AND document.access_scope_id = %s
|
|
AND document.deleted_at IS NULL
|
|
FOR UPDATE OF document, version
|
|
"""
|
|
|
|
_APPROVE_VERSION = """
|
|
UPDATE rag.document_versions
|
|
SET review_state = 'CLOUD_APPROVED',
|
|
embedding_profile_hash = %s,
|
|
cloud_approved_at = now(),
|
|
cloud_approved_by = %s,
|
|
review_revision = review_revision + 1
|
|
WHERE id = %s
|
|
AND review_revision = %s
|
|
AND review_state = 'LOCAL_PARSED_PENDING_CLOUD_REVIEW'
|
|
AND status = 'PROCESSING'
|
|
AND outbound_manifest_sha256 = %s
|
|
RETURNING review_revision
|
|
"""
|
|
|
|
_APPROVE_CHUNKS = """
|
|
UPDATE rag.chunks
|
|
SET approval_status = 'CLOUD_APPROVED',
|
|
outbound_manifest_sha256 = %s,
|
|
embedding_profile_hash = %s,
|
|
embedding_model = %s,
|
|
embedding_dimension = 1024,
|
|
index_status = 'PENDING',
|
|
searchable = false,
|
|
updated_at = now()
|
|
WHERE document_version_id = %s
|
|
AND approval_status = 'LOCAL_PARSED_PENDING_CLOUD_REVIEW'
|
|
RETURNING id, embedding_text_sha256
|
|
"""
|
|
|
|
_REJECT_VERSION = """
|
|
UPDATE rag.document_versions
|
|
SET review_state = 'REJECTED',
|
|
embedding_profile_hash = NULL,
|
|
cloud_approved_at = NULL,
|
|
cloud_approved_by = NULL,
|
|
review_revision = review_revision + 1
|
|
WHERE id = %s
|
|
AND review_revision = %s
|
|
AND review_state = 'LOCAL_PARSED_PENDING_CLOUD_REVIEW'
|
|
AND status = 'PROCESSING'
|
|
RETURNING review_revision
|
|
"""
|
|
|
|
_ENQUEUE_EMBED_JOB = """
|
|
INSERT INTO rag.background_jobs (
|
|
job_type, required_capability, resource_type, resource_id,
|
|
idempotency_key, payload, stage, status, max_attempts
|
|
) VALUES (
|
|
'EMBED_DOCUMENT', 'embedding', 'document_version', %s,
|
|
%s, jsonb_build_object('document_version_id', %s::text),
|
|
'PENDING', 'QUEUED', 3
|
|
)
|
|
ON CONFLICT (job_type, idempotency_key)
|
|
DO UPDATE SET updated_at = rag.background_jobs.updated_at
|
|
RETURNING id, job_type, stage, status, progress, attempt,
|
|
max_attempts, last_error_code, created_at, updated_at, finished_at
|
|
"""
|
|
|
|
|
|
class PostgresDocumentReviewRepository:
|
|
def __init__(self, settings: Settings, *, connect_timeout: int = 5) -> None:
|
|
self._settings = settings
|
|
self._connect_timeout = connect_timeout
|
|
|
|
def _dsn(self) -> str:
|
|
return (
|
|
self._settings.database_url()
|
|
.set(drivername="postgresql")
|
|
.render_as_string(hide_password=False)
|
|
)
|
|
|
|
def apply_decision(
|
|
self,
|
|
*,
|
|
actor: DocumentActor,
|
|
document_id: uuid.UUID,
|
|
decision: ReviewDecision,
|
|
reason_code: ReviewReason,
|
|
expected_revision: int,
|
|
outbound_manifest_sha256: str | None,
|
|
trace_id: uuid.UUID,
|
|
) -> DocumentReviewResult:
|
|
_validate_decision(
|
|
decision=decision,
|
|
reason_code=reason_code,
|
|
expected_revision=expected_revision,
|
|
outbound_manifest_sha256=outbound_manifest_sha256,
|
|
)
|
|
try:
|
|
with psycopg.connect(
|
|
self._dsn(),
|
|
connect_timeout=self._connect_timeout,
|
|
row_factory=dict_row,
|
|
application_name="geological-rag-document-review",
|
|
) as connection:
|
|
with connection.transaction():
|
|
row = connection.execute(
|
|
_LOCK_REVIEW,
|
|
(document_id, actor.knowledge_base_id, actor.access_scope_id),
|
|
).fetchone()
|
|
if row is None:
|
|
raise DocumentReviewNotFoundError
|
|
current_revision = int(row["review_revision"])
|
|
if current_revision != expected_revision:
|
|
raise DocumentReviewConflictError
|
|
version_id = _uuid_value(row["document_version_id"])
|
|
manifest = _optional_text(row["outbound_manifest_sha256"])
|
|
if decision == "APPROVE":
|
|
return self._approve(
|
|
connection=connection,
|
|
actor=actor,
|
|
document_id=document_id,
|
|
version_id=version_id,
|
|
current=row,
|
|
manifest=manifest,
|
|
supplied_manifest=outbound_manifest_sha256,
|
|
expected_revision=expected_revision,
|
|
reason_code=reason_code,
|
|
trace_id=trace_id,
|
|
)
|
|
return self._reject(
|
|
connection=connection,
|
|
actor=actor,
|
|
document_id=document_id,
|
|
version_id=version_id,
|
|
manifest=manifest,
|
|
expected_revision=expected_revision,
|
|
reason_code=reason_code,
|
|
trace_id=trace_id,
|
|
)
|
|
except DocumentReviewError:
|
|
raise
|
|
except psycopg.Error as exc:
|
|
LOGGER.error(
|
|
"document_review_database_error sqlstate=%s",
|
|
exc.sqlstate or "UNKNOWN",
|
|
)
|
|
raise DocumentReviewError from None
|
|
except (OSError, SecretFileError, KeyError, TypeError, ValueError):
|
|
raise DocumentReviewError from None
|
|
|
|
def _approve(
|
|
self,
|
|
*,
|
|
connection: psycopg.Connection[dict[str, object]],
|
|
actor: DocumentActor,
|
|
document_id: uuid.UUID,
|
|
version_id: uuid.UUID,
|
|
current: dict[str, object],
|
|
manifest: str | None,
|
|
supplied_manifest: str | None,
|
|
expected_revision: int,
|
|
reason_code: ReviewReason,
|
|
trace_id: uuid.UUID,
|
|
) -> DocumentReviewResult:
|
|
profile_hash = _optional_text(current.get("active_embedding_profile_hash"))
|
|
profile_model = _optional_text(current.get("profile_model"))
|
|
expected_count = current.get("expected_chunk_count")
|
|
if (
|
|
current.get("review_state") != "LOCAL_PARSED_PENDING_CLOUD_REVIEW"
|
|
or current.get("version_status") != "PROCESSING"
|
|
or manifest is None
|
|
or supplied_manifest != manifest
|
|
or profile_hash is None
|
|
or profile_model is None
|
|
or current.get("profile_enabled") is not True
|
|
or current.get("profile_dimension") != 1024
|
|
or not isinstance(expected_count, int)
|
|
or isinstance(expected_count, bool)
|
|
or expected_count < 1
|
|
):
|
|
raise DocumentReviewStateError
|
|
revision_row = connection.execute(
|
|
_APPROVE_VERSION,
|
|
(profile_hash, actor.subject, version_id, expected_revision, manifest),
|
|
).fetchone()
|
|
if revision_row is None:
|
|
raise DocumentReviewConflictError
|
|
chunks = list(
|
|
connection.execute(
|
|
_APPROVE_CHUNKS,
|
|
(manifest, profile_hash, profile_model, version_id),
|
|
).fetchall()
|
|
)
|
|
if len(chunks) != expected_count:
|
|
raise DocumentReviewStateError
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO rag.chunk_embedding_assignments (
|
|
chunk_id, profile_hash, embedding_text_sha256, status
|
|
)
|
|
SELECT id, %s, embedding_text_sha256, 'PENDING'
|
|
FROM rag.chunks
|
|
WHERE document_version_id = %s
|
|
ON CONFLICT (chunk_id, profile_hash) DO NOTHING
|
|
""",
|
|
(profile_hash, version_id),
|
|
)
|
|
assignment_count = connection.execute(
|
|
"""
|
|
SELECT count(*)
|
|
FROM rag.chunk_embedding_assignments AS assignment
|
|
JOIN rag.chunks AS chunk ON chunk.id = assignment.chunk_id
|
|
WHERE chunk.document_version_id = %s
|
|
AND assignment.profile_hash = %s
|
|
AND assignment.embedding_text_sha256 = chunk.embedding_text_sha256
|
|
AND assignment.status = 'PENDING'
|
|
""",
|
|
(version_id, profile_hash),
|
|
).fetchone()
|
|
if assignment_count is None or assignment_count["count"] != expected_count:
|
|
raise DocumentReviewStateError
|
|
updated_document = connection.execute(
|
|
"""
|
|
UPDATE rag.documents
|
|
SET status = 'CLOUD_APPROVED', updated_at = now()
|
|
WHERE id = %s AND knowledge_base_id = %s AND access_scope_id = %s
|
|
RETURNING id
|
|
""",
|
|
(document_id, actor.knowledge_base_id, actor.access_scope_id),
|
|
).fetchone()
|
|
if updated_document is None:
|
|
raise DocumentReviewConflictError
|
|
job = connection.execute(
|
|
_ENQUEUE_EMBED_JOB,
|
|
(
|
|
version_id,
|
|
f"embed-document:{version_id}:{profile_hash}",
|
|
str(version_id),
|
|
),
|
|
).fetchone()
|
|
if job is None:
|
|
raise DocumentReviewError
|
|
revision = _integer_value(revision_row["review_revision"])
|
|
self._audit(
|
|
connection=connection,
|
|
document_id=document_id,
|
|
version_id=version_id,
|
|
actor=actor,
|
|
decision="APPROVE",
|
|
reason_code=reason_code,
|
|
previous_revision=expected_revision,
|
|
resulting_revision=revision,
|
|
manifest=manifest,
|
|
profile_hash=profile_hash,
|
|
trace_id=trace_id,
|
|
)
|
|
return DocumentReviewResult(
|
|
document_id=document_id,
|
|
document_version_id=version_id,
|
|
decision="APPROVE",
|
|
review_state="CLOUD_APPROVED",
|
|
review_revision=revision,
|
|
outbound_manifest_sha256=manifest,
|
|
embedding_profile_hash=profile_hash,
|
|
job=_safe_job(job),
|
|
)
|
|
|
|
def _reject(
|
|
self,
|
|
*,
|
|
connection: psycopg.Connection[dict[str, object]],
|
|
actor: DocumentActor,
|
|
document_id: uuid.UUID,
|
|
version_id: uuid.UUID,
|
|
manifest: str | None,
|
|
expected_revision: int,
|
|
reason_code: ReviewReason,
|
|
trace_id: uuid.UUID,
|
|
) -> DocumentReviewResult:
|
|
revision_row = connection.execute(
|
|
_REJECT_VERSION,
|
|
(version_id, expected_revision),
|
|
).fetchone()
|
|
if revision_row is None:
|
|
raise DocumentReviewConflictError
|
|
connection.execute(
|
|
"""
|
|
UPDATE rag.chunks
|
|
SET approval_status = 'REJECTED', searchable = false,
|
|
index_status = 'PENDING', embedding = NULL,
|
|
embedded_text_sha256 = NULL, embedding_profile_hash = NULL,
|
|
updated_at = now()
|
|
WHERE document_version_id = %s
|
|
""",
|
|
(version_id,),
|
|
)
|
|
updated_document = connection.execute(
|
|
"""
|
|
UPDATE rag.documents
|
|
SET status = 'REJECTED', active_version_id = NULL, updated_at = now()
|
|
WHERE id = %s AND knowledge_base_id = %s AND access_scope_id = %s
|
|
RETURNING id
|
|
""",
|
|
(document_id, actor.knowledge_base_id, actor.access_scope_id),
|
|
).fetchone()
|
|
if updated_document is None:
|
|
raise DocumentReviewConflictError
|
|
revision = _integer_value(revision_row["review_revision"])
|
|
self._audit(
|
|
connection=connection,
|
|
document_id=document_id,
|
|
version_id=version_id,
|
|
actor=actor,
|
|
decision="REJECT",
|
|
reason_code=reason_code,
|
|
previous_revision=expected_revision,
|
|
resulting_revision=revision,
|
|
manifest=manifest,
|
|
profile_hash=None,
|
|
trace_id=trace_id,
|
|
)
|
|
return DocumentReviewResult(
|
|
document_id=document_id,
|
|
document_version_id=version_id,
|
|
decision="REJECT",
|
|
review_state="REJECTED",
|
|
review_revision=revision,
|
|
outbound_manifest_sha256=manifest,
|
|
embedding_profile_hash=None,
|
|
job=None,
|
|
)
|
|
|
|
@staticmethod
|
|
def _audit(
|
|
*,
|
|
connection: psycopg.Connection[dict[str, object]],
|
|
document_id: uuid.UUID,
|
|
version_id: uuid.UUID,
|
|
actor: DocumentActor,
|
|
decision: ReviewDecision,
|
|
reason_code: ReviewReason,
|
|
previous_revision: int,
|
|
resulting_revision: int,
|
|
manifest: str | None,
|
|
profile_hash: str | None,
|
|
trace_id: uuid.UUID,
|
|
) -> None:
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO rag.document_review_events (
|
|
document_id, document_version_id, actor_subject, decision,
|
|
reason_code, previous_revision, resulting_revision,
|
|
outbound_manifest_sha256, embedding_profile_hash, trace_id
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
""",
|
|
(
|
|
document_id,
|
|
version_id,
|
|
actor.subject,
|
|
decision,
|
|
reason_code,
|
|
previous_revision,
|
|
resulting_revision,
|
|
manifest,
|
|
profile_hash,
|
|
trace_id,
|
|
),
|
|
)
|
|
|
|
|
|
def _validate_decision(
|
|
*,
|
|
decision: ReviewDecision,
|
|
reason_code: ReviewReason,
|
|
expected_revision: int,
|
|
outbound_manifest_sha256: str | None,
|
|
) -> None:
|
|
if isinstance(expected_revision, bool) or expected_revision < 0:
|
|
raise ValueError("expected_revision must be non-negative")
|
|
if decision == "APPROVE":
|
|
if reason_code != "SYNTHETIC_REVIEW_APPROVED" or not (
|
|
outbound_manifest_sha256 and _HASH.fullmatch(outbound_manifest_sha256)
|
|
):
|
|
raise ValueError("approval requires the reviewed manifest")
|
|
elif decision == "REJECT":
|
|
if reason_code == "SYNTHETIC_REVIEW_APPROVED":
|
|
raise ValueError("rejection requires a rejection reason")
|
|
else:
|
|
raise ValueError("unsupported review decision")
|
|
|
|
|
|
def _uuid_value(value: object) -> uuid.UUID:
|
|
if not isinstance(value, uuid.UUID):
|
|
raise DocumentReviewError
|
|
return value
|
|
|
|
|
|
def _optional_text(value: object) -> str | None:
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
|
|
def _integer_value(value: object) -> int:
|
|
if not isinstance(value, int) or isinstance(value, bool):
|
|
raise DocumentReviewError
|
|
return value
|
|
|
|
|
|
def _datetime_value(value: object) -> datetime:
|
|
if not isinstance(value, datetime):
|
|
raise DocumentReviewError
|
|
return value
|
|
|
|
|
|
def _optional_datetime_value(value: object) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
return _datetime_value(value)
|
|
|
|
|
|
def _safe_job(row: dict[str, object]) -> SafeJob:
|
|
return SafeJob(
|
|
id=_uuid_value(row["id"]),
|
|
job_type=str(row["job_type"]),
|
|
stage=str(row["stage"]),
|
|
status=str(row["status"]),
|
|
progress=_integer_value(row["progress"]),
|
|
attempt=_integer_value(row["attempt"]),
|
|
max_attempts=_integer_value(row["max_attempts"]),
|
|
last_error_code=_optional_text(row.get("last_error_code")),
|
|
created_at=_datetime_value(row["created_at"]),
|
|
updated_at=_datetime_value(row["updated_at"]),
|
|
finished_at=_optional_datetime_value(row.get("finished_at")),
|
|
)
|