Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
This commit is contained in:
@@ -0,0 +1,294 @@
|
||||
"""Add governed document uploads and traceable parsed-page artifacts.
|
||||
|
||||
Revision ID: 0003_document_ingestion
|
||||
Revises: 0002_model_profiles
|
||||
Create Date: 2026-07-13
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0003_document_ingestion"
|
||||
down_revision: str | None = "0002_model_profiles"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE rag.document_uploads (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
actor_subject text NOT NULL,
|
||||
knowledge_base_id uuid NOT NULL,
|
||||
access_scope_id uuid NOT NULL,
|
||||
idempotency_key_hash char(64) NOT NULL,
|
||||
request_fingerprint char(64) NOT NULL,
|
||||
original_filename text NOT NULL,
|
||||
declared_mime_type text NOT NULL,
|
||||
expected_size bigint NOT NULL,
|
||||
expected_sha256 char(64) NOT NULL,
|
||||
storage_key uuid NOT NULL DEFAULT gen_random_uuid(),
|
||||
actual_size bigint,
|
||||
actual_sha256 char(64),
|
||||
status text NOT NULL DEFAULT 'CREATED',
|
||||
document_id uuid,
|
||||
parse_job_id uuid,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
completed_at timestamptz,
|
||||
CONSTRAINT document_uploads_knowledge_base_fk
|
||||
FOREIGN KEY (knowledge_base_id)
|
||||
REFERENCES rag.knowledge_bases (id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_uploads_access_scope_fk
|
||||
FOREIGN KEY (knowledge_base_id, access_scope_id)
|
||||
REFERENCES rag.access_scopes (knowledge_base_id, id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT document_uploads_document_fk
|
||||
FOREIGN KEY (knowledge_base_id, document_id)
|
||||
REFERENCES rag.documents (knowledge_base_id, id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT document_uploads_parse_job_fk
|
||||
FOREIGN KEY (parse_job_id)
|
||||
REFERENCES rag.background_jobs (id)
|
||||
ON DELETE RESTRICT,
|
||||
CONSTRAINT document_uploads_actor_nonempty
|
||||
CHECK (btrim(actor_subject) <> '' AND length(actor_subject) <= 200),
|
||||
CONSTRAINT document_uploads_hashes_valid
|
||||
CHECK (
|
||||
idempotency_key_hash ~ '^[0-9a-f]{64}$'
|
||||
AND request_fingerprint ~ '^[0-9a-f]{64}$'
|
||||
AND expected_sha256 ~ '^[0-9a-f]{64}$'
|
||||
AND (
|
||||
actual_sha256 IS NULL
|
||||
OR actual_sha256 ~ '^[0-9a-f]{64}$'
|
||||
)
|
||||
),
|
||||
CONSTRAINT document_uploads_filename_safe
|
||||
CHECK (
|
||||
btrim(original_filename) <> ''
|
||||
AND length(original_filename) <= 240
|
||||
AND original_filename !~ '[/\\\\]'
|
||||
),
|
||||
CONSTRAINT document_uploads_mime_valid
|
||||
CHECK (declared_mime_type IN (
|
||||
'text/plain',
|
||||
'text/markdown',
|
||||
'application/pdf',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
)),
|
||||
CONSTRAINT document_uploads_expected_size_valid
|
||||
CHECK (expected_size BETWEEN 1 AND 2147483648),
|
||||
CONSTRAINT document_uploads_actual_size_valid
|
||||
CHECK (actual_size IS NULL OR actual_size BETWEEN 1 AND 2147483648),
|
||||
CONSTRAINT document_uploads_status_valid
|
||||
CHECK (status IN ('CREATED', 'STORED', 'COMPLETED')),
|
||||
CONSTRAINT document_uploads_stored_content_exact
|
||||
CHECK (
|
||||
(
|
||||
status = 'CREATED'
|
||||
AND actual_size IS NULL
|
||||
AND actual_sha256 IS NULL
|
||||
)
|
||||
OR (
|
||||
status IN ('STORED', 'COMPLETED')
|
||||
AND actual_size = expected_size
|
||||
AND actual_sha256 = expected_sha256
|
||||
)
|
||||
),
|
||||
CONSTRAINT document_uploads_completion_consistent
|
||||
CHECK (
|
||||
(
|
||||
status = 'COMPLETED'
|
||||
AND document_id IS NOT NULL
|
||||
AND parse_job_id IS NOT NULL
|
||||
AND completed_at IS NOT NULL
|
||||
)
|
||||
OR (
|
||||
status <> 'COMPLETED'
|
||||
AND document_id IS NULL
|
||||
AND parse_job_id IS NULL
|
||||
AND completed_at IS NULL
|
||||
)
|
||||
),
|
||||
CONSTRAINT document_uploads_timestamps_valid
|
||||
CHECK (
|
||||
updated_at >= created_at
|
||||
AND (completed_at IS NULL OR completed_at >= created_at)
|
||||
),
|
||||
CONSTRAINT document_uploads_actor_idempotency_key
|
||||
UNIQUE (actor_subject, idempotency_key_hash),
|
||||
CONSTRAINT document_uploads_storage_key_key
|
||||
UNIQUE (storage_key)
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX document_uploads_actor_status_lookup
|
||||
ON rag.document_uploads (
|
||||
actor_subject,
|
||||
knowledge_base_id,
|
||||
access_scope_id,
|
||||
status,
|
||||
created_at DESC
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE rag.document_upload_events (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
upload_id uuid NOT NULL,
|
||||
actor_subject text NOT NULL,
|
||||
event_type text NOT NULL,
|
||||
trace_id uuid NOT NULL,
|
||||
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT document_upload_events_upload_fk
|
||||
FOREIGN KEY (upload_id)
|
||||
REFERENCES rag.document_uploads (id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_upload_events_actor_nonempty
|
||||
CHECK (btrim(actor_subject) <> '' AND length(actor_subject) <= 200),
|
||||
CONSTRAINT document_upload_events_type_valid
|
||||
CHECK (event_type IN ('CREATED', 'STORED', 'COMPLETED')),
|
||||
CONSTRAINT document_upload_events_metadata_object
|
||||
CHECK (jsonb_typeof(metadata) = 'object'),
|
||||
CONSTRAINT document_upload_events_metadata_has_no_credentials
|
||||
CHECK (
|
||||
metadata::text !~*
|
||||
'"[^"]*(api[_-]?key|secret|password|token|authorization|credential|storage[_-]?key|path)[^"]*"[[:space:]]*:'
|
||||
)
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX document_upload_events_upload_timeline
|
||||
ON rag.document_upload_events (upload_id, created_at, id);
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE rag.document_pages (
|
||||
id uuid PRIMARY KEY,
|
||||
document_version_id uuid NOT NULL,
|
||||
ordinal integer NOT NULL,
|
||||
page_number integer,
|
||||
display_text text NOT NULL,
|
||||
text_sha256 char(64) NOT NULL,
|
||||
line_start integer NOT NULL,
|
||||
line_end integer NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT document_pages_version_fk
|
||||
FOREIGN KEY (document_version_id)
|
||||
REFERENCES rag.document_versions (id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_pages_ordinal_valid
|
||||
CHECK (ordinal >= 0),
|
||||
CONSTRAINT document_pages_page_number_valid
|
||||
CHECK (page_number IS NULL OR page_number > 0),
|
||||
CONSTRAINT document_pages_text_nonempty
|
||||
CHECK (btrim(display_text) <> ''),
|
||||
CONSTRAINT document_pages_text_hash_valid
|
||||
CHECK (text_sha256 ~ '^[0-9a-f]{64}$'),
|
||||
CONSTRAINT document_pages_lines_valid
|
||||
CHECK (line_start > 0 AND line_end >= line_start),
|
||||
CONSTRAINT document_pages_version_ordinal_key
|
||||
UNIQUE (document_version_id, ordinal),
|
||||
CONSTRAINT document_pages_version_id_key
|
||||
UNIQUE (document_version_id, id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX document_pages_version_number_key
|
||||
ON rag.document_pages (document_version_id, page_number)
|
||||
WHERE page_number IS NOT NULL;
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE rag.document_blocks (
|
||||
id uuid PRIMARY KEY,
|
||||
document_version_id uuid NOT NULL,
|
||||
page_id uuid NOT NULL,
|
||||
ordinal integer NOT NULL,
|
||||
block_kind text NOT NULL,
|
||||
display_text text NOT NULL,
|
||||
text_sha256 char(64) NOT NULL,
|
||||
section_path jsonb NOT NULL DEFAULT '[]'::jsonb,
|
||||
anchor_id char(64) NOT NULL,
|
||||
normalized_text_sha256 char(64) NOT NULL,
|
||||
char_start integer NOT NULL,
|
||||
char_end integer NOT NULL,
|
||||
line_start integer NOT NULL,
|
||||
line_end integer NOT NULL,
|
||||
page_start integer,
|
||||
page_end integer,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT document_blocks_version_fk
|
||||
FOREIGN KEY (document_version_id)
|
||||
REFERENCES rag.document_versions (id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_blocks_page_fk
|
||||
FOREIGN KEY (document_version_id, page_id)
|
||||
REFERENCES rag.document_pages (document_version_id, id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_blocks_ordinal_valid
|
||||
CHECK (ordinal >= 0),
|
||||
CONSTRAINT document_blocks_kind_valid
|
||||
CHECK (block_kind IN ('HEADING', 'PARAGRAPH', 'TABLE_ROW')),
|
||||
CONSTRAINT document_blocks_text_nonempty
|
||||
CHECK (btrim(display_text) <> ''),
|
||||
CONSTRAINT document_blocks_hashes_valid
|
||||
CHECK (
|
||||
text_sha256 ~ '^[0-9a-f]{64}$'
|
||||
AND anchor_id ~ '^[0-9a-f]{64}$'
|
||||
AND normalized_text_sha256 ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT document_blocks_section_path_array
|
||||
CHECK (jsonb_typeof(section_path) = 'array'),
|
||||
CONSTRAINT document_blocks_anchor_ranges_valid
|
||||
CHECK (
|
||||
char_start >= 0
|
||||
AND char_end > char_start
|
||||
AND line_start > 0
|
||||
AND line_end >= line_start
|
||||
AND (
|
||||
(page_start IS NULL AND page_end IS NULL)
|
||||
OR (
|
||||
page_start IS NOT NULL
|
||||
AND page_end IS NOT NULL
|
||||
AND page_start > 0
|
||||
AND page_end >= page_start
|
||||
)
|
||||
)
|
||||
),
|
||||
CONSTRAINT document_blocks_version_ordinal_key
|
||||
UNIQUE (document_version_id, ordinal),
|
||||
CONSTRAINT document_blocks_version_anchor_key
|
||||
UNIQUE (document_version_id, anchor_id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX document_blocks_version_page_lookup
|
||||
ON rag.document_blocks (document_version_id, page_id, ordinal);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP TABLE IF EXISTS rag.document_blocks;")
|
||||
op.execute("DROP TABLE IF EXISTS rag.document_pages;")
|
||||
op.execute("DROP TABLE IF EXISTS rag.document_upload_events;")
|
||||
op.execute("DROP TABLE IF EXISTS rag.document_uploads;")
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Add optimistic document review decisions and immutable review audit.
|
||||
|
||||
Revision ID: 0004_document_review
|
||||
Revises: 0003_document_ingestion
|
||||
Create Date: 2026-07-13
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004_document_review"
|
||||
down_revision: str | None = "0003_document_ingestion"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE rag.document_versions
|
||||
ADD COLUMN review_revision integer NOT NULL DEFAULT 0,
|
||||
ADD CONSTRAINT document_versions_review_revision_valid
|
||||
CHECK (review_revision >= 0);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TABLE rag.document_review_events (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
document_id uuid NOT NULL,
|
||||
document_version_id uuid NOT NULL,
|
||||
actor_subject text NOT NULL,
|
||||
decision text NOT NULL,
|
||||
reason_code text NOT NULL,
|
||||
previous_revision integer NOT NULL,
|
||||
resulting_revision integer NOT NULL,
|
||||
outbound_manifest_sha256 char(64),
|
||||
embedding_profile_hash char(64),
|
||||
trace_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT document_review_events_document_fk
|
||||
FOREIGN KEY (document_id, document_version_id)
|
||||
REFERENCES rag.document_versions (document_id, id)
|
||||
ON DELETE CASCADE,
|
||||
CONSTRAINT document_review_events_actor_nonempty
|
||||
CHECK (btrim(actor_subject) <> '' AND length(actor_subject) <= 200),
|
||||
CONSTRAINT document_review_events_decision_valid
|
||||
CHECK (decision IN ('APPROVE', 'REJECT')),
|
||||
CONSTRAINT document_review_events_reason_valid
|
||||
CHECK (reason_code IN (
|
||||
'SYNTHETIC_REVIEW_APPROVED',
|
||||
'RIGHTS_NOT_VERIFIED',
|
||||
'CONTENT_QUALITY_REJECTED',
|
||||
'CLOUD_PROCESSING_REJECTED'
|
||||
)),
|
||||
CONSTRAINT document_review_events_revision_valid
|
||||
CHECK (
|
||||
previous_revision >= 0
|
||||
AND resulting_revision = previous_revision + 1
|
||||
),
|
||||
CONSTRAINT document_review_events_hashes_valid
|
||||
CHECK (
|
||||
(
|
||||
decision = 'APPROVE'
|
||||
AND outbound_manifest_sha256 ~ '^[0-9a-f]{64}$'
|
||||
AND embedding_profile_hash ~ '^[0-9a-f]{64}$'
|
||||
)
|
||||
OR (
|
||||
decision = 'REJECT'
|
||||
AND embedding_profile_hash IS NULL
|
||||
)
|
||||
),
|
||||
CONSTRAINT document_review_events_version_revision_key
|
||||
UNIQUE (document_version_id, resulting_revision)
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE INDEX document_review_events_document_timeline
|
||||
ON rag.document_review_events (document_id, created_at, id);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE FUNCTION rag.reject_document_review_event_mutation()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
SECURITY INVOKER
|
||||
SET search_path = pg_catalog, rag
|
||||
AS $function$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'document review events are append-only'
|
||||
USING ERRCODE = '23514';
|
||||
END
|
||||
$function$;
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE TRIGGER document_review_events_append_only
|
||||
BEFORE UPDATE OR DELETE
|
||||
ON rag.document_review_events
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION rag.reject_document_review_event_mutation();
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(
|
||||
"DROP TRIGGER IF EXISTS document_review_events_append_only ON rag.document_review_events;"
|
||||
)
|
||||
op.execute("DROP FUNCTION IF EXISTS rag.reject_document_review_event_mutation();")
|
||||
op.execute("DROP TABLE IF EXISTS rag.document_review_events;")
|
||||
op.execute(
|
||||
"""
|
||||
ALTER TABLE rag.document_versions
|
||||
DROP CONSTRAINT IF EXISTS document_versions_review_revision_valid,
|
||||
DROP COLUMN IF EXISTS review_revision;
|
||||
"""
|
||||
)
|
||||
Reference in New Issue
Block a user