YoVinchen
ecdb10c37a
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
295 lines
11 KiB
Python
295 lines
11 KiB
Python
"""Add governed document uploads and traceable parsed-page artifacts.
|
|
|
|
Revision ID: 0003_document_ingestion
|
|
Revises: 0002_model_profiles
|
|
Create Date: 2026-07-13
|
|
"""
|
|
|
|
from collections.abc import Sequence
|
|
|
|
from alembic import op
|
|
|
|
revision: str = "0003_document_ingestion"
|
|
down_revision: str | None = "0002_model_profiles"
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE rag.document_uploads (
|
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
actor_subject text NOT NULL,
|
|
knowledge_base_id uuid NOT NULL,
|
|
access_scope_id uuid NOT NULL,
|
|
idempotency_key_hash char(64) NOT NULL,
|
|
request_fingerprint char(64) NOT NULL,
|
|
original_filename text NOT NULL,
|
|
declared_mime_type text NOT NULL,
|
|
expected_size bigint NOT NULL,
|
|
expected_sha256 char(64) NOT NULL,
|
|
storage_key uuid NOT NULL DEFAULT gen_random_uuid(),
|
|
actual_size bigint,
|
|
actual_sha256 char(64),
|
|
status text NOT NULL DEFAULT 'CREATED',
|
|
document_id uuid,
|
|
parse_job_id uuid,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
completed_at timestamptz,
|
|
CONSTRAINT document_uploads_knowledge_base_fk
|
|
FOREIGN KEY (knowledge_base_id)
|
|
REFERENCES rag.knowledge_bases (id)
|
|
ON DELETE CASCADE,
|
|
CONSTRAINT document_uploads_access_scope_fk
|
|
FOREIGN KEY (knowledge_base_id, access_scope_id)
|
|
REFERENCES rag.access_scopes (knowledge_base_id, id)
|
|
ON DELETE RESTRICT,
|
|
CONSTRAINT document_uploads_document_fk
|
|
FOREIGN KEY (knowledge_base_id, document_id)
|
|
REFERENCES rag.documents (knowledge_base_id, id)
|
|
ON DELETE RESTRICT,
|
|
CONSTRAINT document_uploads_parse_job_fk
|
|
FOREIGN KEY (parse_job_id)
|
|
REFERENCES rag.background_jobs (id)
|
|
ON DELETE RESTRICT,
|
|
CONSTRAINT document_uploads_actor_nonempty
|
|
CHECK (btrim(actor_subject) <> '' AND length(actor_subject) <= 200),
|
|
CONSTRAINT document_uploads_hashes_valid
|
|
CHECK (
|
|
idempotency_key_hash ~ '^[0-9a-f]{64}$'
|
|
AND request_fingerprint ~ '^[0-9a-f]{64}$'
|
|
AND expected_sha256 ~ '^[0-9a-f]{64}$'
|
|
AND (
|
|
actual_sha256 IS NULL
|
|
OR actual_sha256 ~ '^[0-9a-f]{64}$'
|
|
)
|
|
),
|
|
CONSTRAINT document_uploads_filename_safe
|
|
CHECK (
|
|
btrim(original_filename) <> ''
|
|
AND length(original_filename) <= 240
|
|
AND original_filename !~ '[/\\\\]'
|
|
),
|
|
CONSTRAINT document_uploads_mime_valid
|
|
CHECK (declared_mime_type IN (
|
|
'text/plain',
|
|
'text/markdown',
|
|
'application/pdf',
|
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
|
)),
|
|
CONSTRAINT document_uploads_expected_size_valid
|
|
CHECK (expected_size BETWEEN 1 AND 2147483648),
|
|
CONSTRAINT document_uploads_actual_size_valid
|
|
CHECK (actual_size IS NULL OR actual_size BETWEEN 1 AND 2147483648),
|
|
CONSTRAINT document_uploads_status_valid
|
|
CHECK (status IN ('CREATED', 'STORED', 'COMPLETED')),
|
|
CONSTRAINT document_uploads_stored_content_exact
|
|
CHECK (
|
|
(
|
|
status = 'CREATED'
|
|
AND actual_size IS NULL
|
|
AND actual_sha256 IS NULL
|
|
)
|
|
OR (
|
|
status IN ('STORED', 'COMPLETED')
|
|
AND actual_size = expected_size
|
|
AND actual_sha256 = expected_sha256
|
|
)
|
|
),
|
|
CONSTRAINT document_uploads_completion_consistent
|
|
CHECK (
|
|
(
|
|
status = 'COMPLETED'
|
|
AND document_id IS NOT NULL
|
|
AND parse_job_id IS NOT NULL
|
|
AND completed_at IS NOT NULL
|
|
)
|
|
OR (
|
|
status <> 'COMPLETED'
|
|
AND document_id IS NULL
|
|
AND parse_job_id IS NULL
|
|
AND completed_at IS NULL
|
|
)
|
|
),
|
|
CONSTRAINT document_uploads_timestamps_valid
|
|
CHECK (
|
|
updated_at >= created_at
|
|
AND (completed_at IS NULL OR completed_at >= created_at)
|
|
),
|
|
CONSTRAINT document_uploads_actor_idempotency_key
|
|
UNIQUE (actor_subject, idempotency_key_hash),
|
|
CONSTRAINT document_uploads_storage_key_key
|
|
UNIQUE (storage_key)
|
|
);
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE INDEX document_uploads_actor_status_lookup
|
|
ON rag.document_uploads (
|
|
actor_subject,
|
|
knowledge_base_id,
|
|
access_scope_id,
|
|
status,
|
|
created_at DESC
|
|
);
|
|
"""
|
|
)
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE rag.document_upload_events (
|
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
upload_id uuid NOT NULL,
|
|
actor_subject text NOT NULL,
|
|
event_type text NOT NULL,
|
|
trace_id uuid NOT NULL,
|
|
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
CONSTRAINT document_upload_events_upload_fk
|
|
FOREIGN KEY (upload_id)
|
|
REFERENCES rag.document_uploads (id)
|
|
ON DELETE CASCADE,
|
|
CONSTRAINT document_upload_events_actor_nonempty
|
|
CHECK (btrim(actor_subject) <> '' AND length(actor_subject) <= 200),
|
|
CONSTRAINT document_upload_events_type_valid
|
|
CHECK (event_type IN ('CREATED', 'STORED', 'COMPLETED')),
|
|
CONSTRAINT document_upload_events_metadata_object
|
|
CHECK (jsonb_typeof(metadata) = 'object'),
|
|
CONSTRAINT document_upload_events_metadata_has_no_credentials
|
|
CHECK (
|
|
metadata::text !~*
|
|
'"[^"]*(api[_-]?key|secret|password|token|authorization|credential|storage[_-]?key|path)[^"]*"[[:space:]]*:'
|
|
)
|
|
);
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE INDEX document_upload_events_upload_timeline
|
|
ON rag.document_upload_events (upload_id, created_at, id);
|
|
"""
|
|
)
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE rag.document_pages (
|
|
id uuid PRIMARY KEY,
|
|
document_version_id uuid NOT NULL,
|
|
ordinal integer NOT NULL,
|
|
page_number integer,
|
|
display_text text NOT NULL,
|
|
text_sha256 char(64) NOT NULL,
|
|
line_start integer NOT NULL,
|
|
line_end integer NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
CONSTRAINT document_pages_version_fk
|
|
FOREIGN KEY (document_version_id)
|
|
REFERENCES rag.document_versions (id)
|
|
ON DELETE CASCADE,
|
|
CONSTRAINT document_pages_ordinal_valid
|
|
CHECK (ordinal >= 0),
|
|
CONSTRAINT document_pages_page_number_valid
|
|
CHECK (page_number IS NULL OR page_number > 0),
|
|
CONSTRAINT document_pages_text_nonempty
|
|
CHECK (btrim(display_text) <> ''),
|
|
CONSTRAINT document_pages_text_hash_valid
|
|
CHECK (text_sha256 ~ '^[0-9a-f]{64}$'),
|
|
CONSTRAINT document_pages_lines_valid
|
|
CHECK (line_start > 0 AND line_end >= line_start),
|
|
CONSTRAINT document_pages_version_ordinal_key
|
|
UNIQUE (document_version_id, ordinal),
|
|
CONSTRAINT document_pages_version_id_key
|
|
UNIQUE (document_version_id, id)
|
|
);
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE UNIQUE INDEX document_pages_version_number_key
|
|
ON rag.document_pages (document_version_id, page_number)
|
|
WHERE page_number IS NOT NULL;
|
|
"""
|
|
)
|
|
|
|
op.execute(
|
|
"""
|
|
CREATE TABLE rag.document_blocks (
|
|
id uuid PRIMARY KEY,
|
|
document_version_id uuid NOT NULL,
|
|
page_id uuid NOT NULL,
|
|
ordinal integer NOT NULL,
|
|
block_kind text NOT NULL,
|
|
display_text text NOT NULL,
|
|
text_sha256 char(64) NOT NULL,
|
|
section_path jsonb NOT NULL DEFAULT '[]'::jsonb,
|
|
anchor_id char(64) NOT NULL,
|
|
normalized_text_sha256 char(64) NOT NULL,
|
|
char_start integer NOT NULL,
|
|
char_end integer NOT NULL,
|
|
line_start integer NOT NULL,
|
|
line_end integer NOT NULL,
|
|
page_start integer,
|
|
page_end integer,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
CONSTRAINT document_blocks_version_fk
|
|
FOREIGN KEY (document_version_id)
|
|
REFERENCES rag.document_versions (id)
|
|
ON DELETE CASCADE,
|
|
CONSTRAINT document_blocks_page_fk
|
|
FOREIGN KEY (document_version_id, page_id)
|
|
REFERENCES rag.document_pages (document_version_id, id)
|
|
ON DELETE CASCADE,
|
|
CONSTRAINT document_blocks_ordinal_valid
|
|
CHECK (ordinal >= 0),
|
|
CONSTRAINT document_blocks_kind_valid
|
|
CHECK (block_kind IN ('HEADING', 'PARAGRAPH', 'TABLE_ROW')),
|
|
CONSTRAINT document_blocks_text_nonempty
|
|
CHECK (btrim(display_text) <> ''),
|
|
CONSTRAINT document_blocks_hashes_valid
|
|
CHECK (
|
|
text_sha256 ~ '^[0-9a-f]{64}$'
|
|
AND anchor_id ~ '^[0-9a-f]{64}$'
|
|
AND normalized_text_sha256 ~ '^[0-9a-f]{64}$'
|
|
),
|
|
CONSTRAINT document_blocks_section_path_array
|
|
CHECK (jsonb_typeof(section_path) = 'array'),
|
|
CONSTRAINT document_blocks_anchor_ranges_valid
|
|
CHECK (
|
|
char_start >= 0
|
|
AND char_end > char_start
|
|
AND line_start > 0
|
|
AND line_end >= line_start
|
|
AND (
|
|
(page_start IS NULL AND page_end IS NULL)
|
|
OR (
|
|
page_start IS NOT NULL
|
|
AND page_end IS NOT NULL
|
|
AND page_start > 0
|
|
AND page_end >= page_start
|
|
)
|
|
)
|
|
),
|
|
CONSTRAINT document_blocks_version_ordinal_key
|
|
UNIQUE (document_version_id, ordinal),
|
|
CONSTRAINT document_blocks_version_anchor_key
|
|
UNIQUE (document_version_id, anchor_id)
|
|
);
|
|
"""
|
|
)
|
|
op.execute(
|
|
"""
|
|
CREATE INDEX document_blocks_version_page_lookup
|
|
ON rag.document_blocks (document_version_id, page_id, ordinal);
|
|
"""
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.execute("DROP TABLE IF EXISTS rag.document_blocks;")
|
|
op.execute("DROP TABLE IF EXISTS rag.document_pages;")
|
|
op.execute("DROP TABLE IF EXISTS rag.document_upload_events;")
|
|
op.execute("DROP TABLE IF EXISTS rag.document_uploads;")
|