Make the governed RAG evidence path executable end to end

Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench.

Constraint: Live Bailian authentication currently fails for all three configured capabilities

Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding

Confidence: high

Scope-risk: broad

Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land

Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip

Not-tested: Successful live Bailian calls, OCR, real multi-user authorization

backend/tests/integration/test_document_ingestion_migration_contract.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+MIGRATION_PATH = ROOT / "backend/migrations/versions/0003_document_ingestion.py"
+MIGRATION = MIGRATION_PATH.read_text(encoding="utf-8")
+NORMALIZED = " ".join(MIGRATION.lower().split())
+
+
+def _table(name: str) -> str:
+    pattern = re.compile(rf"(?ms)create table rag\.{re.escape(name)} \((.*?)^        \);?")
+    match = pattern.search(MIGRATION.lower())
+    assert match is not None, f"missing table rag.{name}"
+    return " ".join(match.group(1).split())
+
+
+def test_revision_is_additive_after_model_profiles() -> None:
+    assert 'revision: str = "0003_document_ingestion"' in MIGRATION
+    assert 'down_revision: str | none = "0002_model_profiles"' in MIGRATION.lower()
+    assert "alter table rag.documents" not in NORMALIZED
+    assert "alter table rag.chunks" not in NORMALIZED
+    assert "drop table if exists rag.documents" not in NORMALIZED
+    assert "drop table if exists rag.background_jobs" not in NORMALIZED
+
+
+def test_uploads_bind_actor_scope_idempotency_content_and_completion() -> None:
+    table = _table("document_uploads")
+
+    for column in (
+        "actor_subject text not null",
+        "knowledge_base_id uuid not null",
+        "access_scope_id uuid not null",
+        "idempotency_key_hash char(64) not null",
+        "request_fingerprint char(64) not null",
+        "expected_size bigint not null",
+        "expected_sha256 char(64) not null",
+        "storage_key uuid not null",
+        "status text not null default 'created'",
+        "document_id uuid",
+        "parse_job_id uuid",
+    ):
+        assert column in table
+    assert "foreign key (knowledge_base_id, access_scope_id)" in table
+    assert "references rag.access_scopes (knowledge_base_id, id)" in table
+    assert "unique (actor_subject, idempotency_key_hash)" in table
+    assert "unique (storage_key)" in table
+    assert "status in ('created', 'stored', 'completed')" in table
+    assert "actual_size = expected_size" in table
+    assert "actual_sha256 = expected_sha256" in table
+    assert "status = 'completed'" in table
+    assert "document_id is not null" in table
+    assert "parse_job_id is not null" in table
+    assert "completed_at is not null" in table
+    assert "original_filename !~ '[/\\\\\\\\]'" in table
+
+
+def test_upload_audit_is_append_only_metadata_without_sensitive_fields() -> None:
+    table = _table("document_upload_events")
+
+    assert "upload_id uuid not null" in table
+    assert "actor_subject text not null" in table
+    assert "trace_id uuid not null" in table
+    assert "event_type in ('created', 'stored', 'completed')" in table
+    assert "jsonb_typeof(metadata) = 'object'" in table
+    assert "metadata_has_no_credentials" in table
+    for forbidden in (
+        "api[_-]?key",
+        "secret",
+        "password",
+        "token",
+        "authorization",
+        "credential",
+        "storage[_-]?key",
+        "path",
+    ):
+        assert forbidden in table
+
+
+def test_pages_and_blocks_preserve_version_source_anchors() -> None:
+    pages = _table("document_pages")
+    blocks = _table("document_blocks")
+
+    assert "foreign key (document_version_id)" in pages
+    assert "references rag.document_versions (id) on delete cascade" in pages
+    assert "unique (document_version_id, ordinal)" in pages
+    assert "page_number is null or page_number > 0" in pages
+    assert "text_sha256 ~ '^[0-9a-f]{64}$'" in pages
+    assert "line_start > 0 and line_end >= line_start" in pages
+
+    assert "foreign key (document_version_id, page_id)" in blocks
+    assert "references rag.document_pages (document_version_id, id)" in blocks
+    assert "block_kind in ('heading', 'paragraph', 'table_row')" in blocks
+    assert "jsonb_typeof(section_path) = 'array'" in blocks
+    assert "anchor_id ~ '^[0-9a-f]{64}$'" in blocks
+    assert "normalized_text_sha256 ~ '^[0-9a-f]{64}$'" in blocks
+    assert "char_end > char_start" in blocks
+    assert "line_end >= line_start" in blocks
+    assert "unique (document_version_id, anchor_id)" in blocks
+
+
+def test_downgrade_drops_only_new_dependents_in_safe_order() -> None:
+    block = NORMALIZED.index("drop table if exists rag.document_blocks")
+    page = NORMALIZED.index("drop table if exists rag.document_pages")
+    event = NORMALIZED.index("drop table if exists rag.document_upload_events")
+    upload = NORMALIZED.index("drop table if exists rag.document_uploads")
+
+    assert block < page < event < upload
+    assert "drop table if exists rag.documents" not in NORMALIZED
+    assert "drop table if exists rag.document_versions" not in NORMALIZED

backend/tests/integration/test_document_review_migration_contract.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+MIGRATION = (ROOT / "backend/migrations/versions/0004_document_review.py").read_text(
+    encoding="utf-8"
+)
+NORMALIZED = " ".join(MIGRATION.lower().split())
+
+
+def test_review_migration_is_additive_and_revisioned() -> None:
+    assert 'revision: str = "0004_document_review"' in MIGRATION
+    assert 'down_revision: str | none = "0003_document_ingestion"' in MIGRATION.lower()
+    assert "add column review_revision integer not null default 0" in NORMALIZED
+    assert "check (review_revision >= 0)" in NORMALIZED
+    assert "create table rag.document_review_events" in NORMALIZED
+    assert "unique (document_version_id, resulting_revision)" in NORMALIZED
+
+
+def test_review_audit_is_append_only_and_hash_bound() -> None:
+    assert "decision in ('approve', 'reject')" in NORMALIZED
+    assert "resulting_revision = previous_revision + 1" in NORMALIZED
+    assert "outbound_manifest_sha256 ~ '^[0-9a-f]{64}$'" in NORMALIZED
+    assert "embedding_profile_hash ~ '^[0-9a-f]{64}$'" in NORMALIZED
+    assert "document_review_events_append_only" in NORMALIZED
+    assert "reject_document_review_event_mutation" in NORMALIZED
+    assert "before update or delete" in NORMALIZED
+
+
+def test_review_downgrade_removes_only_review_additions() -> None:
+    assert "drop table if exists rag.document_review_events" in NORMALIZED
+    assert "drop column if exists review_revision" in NORMALIZED
+    assert "drop table if exists rag.documents" not in NORMALIZED
+    assert "drop table if exists rag.document_versions" not in NORMALIZED

backend/tests/integration/test_schema_contract.py

@@ -35,13 +35,17 @@ def _service_block(name: str) -> str:
 def test_compose_isolates_database_credentials_and_networks() -> None:
     db = _service_block("db")
     migrate = _service_block("migrate")
+    upload_init = _service_block("upload-init")
     api = _service_block("api")
     model_gateway = _service_block("model-gateway")
+    worker_local = _service_block("worker-local")
+    worker_model = _service_block("worker-model")
     gateway = _service_block("gateway")
     web = _service_block("web")
     provider_smoke = _service_block("provider-smoke")
     seed_demo = _service_block("seed-demo")
     seed_demo_offline = _service_block("seed-demo-offline")
+    document_smoke = _service_block("document-pipeline-smoke")
 
     assert "postgres_bootstrap_password" in db
     assert "postgres_migrator_password" in db
@@ -51,11 +55,19 @@ def test_compose_isolates_database_credentials_and_networks() -> None:
     assert "postgres_bootstrap_password" not in migrate
     assert "postgres_app_password" not in migrate
 
+    assert "network_mode: none" in upload_init
+    assert 'user: "0:0"' in upload_init
+    assert "uploads_data:/data/uploads" in upload_init
+    assert "secrets:" not in upload_init
+    assert "CHOWN" in upload_init
+    assert "DAC_OVERRIDE" in upload_init
+
     assert "postgres_app_password" in api
     assert "model_gateway_api_token" in api
     assert "postgres_bootstrap_password" not in api
     assert "postgres_migrator_password" not in api
     assert "bailian_api_key" not in api
+    assert "uploads_data:/data/uploads" in api
     assert '"127.0.0.1:8000:8000"' not in api
     assert "      - data" in api
     assert "      - model" in api
@@ -69,6 +81,7 @@ def test_compose_isolates_database_credentials_and_networks() -> None:
     assert "model_gateway_api_token" in model_gateway
     assert "model_gateway_worker_token" in model_gateway
     assert "postgres_" not in model_gateway
+    assert "uploads_data" not in model_gateway
     assert "      - model" in model_gateway
     assert "      - egress" in model_gateway
     assert "      - data" not in model_gateway
@@ -79,6 +92,32 @@ def test_compose_isolates_database_credentials_and_networks() -> None:
     assert "no-new-privileges:true" in model_gateway
     assert "cap_drop:" in model_gateway and "      - ALL" in model_gateway
 
+    assert "WORKER_CAPABILITIES: document_parse" in worker_local
+    assert "postgres_app_password" in worker_local
+    assert "model_gateway_" not in worker_local
+    assert "bailian_api_key" not in worker_local
+    assert "uploads_data:/data/uploads" in worker_local
+    assert "      - data" in worker_local
+    assert "      - model" not in worker_local
+    assert "      - egress" not in worker_local
+    assert "read_only: true" in worker_local
+    assert "no-new-privileges:true" in worker_local
+    assert "stop_grace_period: 150s" in worker_local
+
+    assert "WORKER_CAPABILITIES: embedding" in worker_model
+    assert "MODEL_GATEWAY_CALLER: worker" in worker_model
+    assert "model_gateway_worker_token" in worker_model
+    assert "model_gateway_api_token" not in worker_model
+    assert "postgres_app_password" in worker_model
+    assert "bailian_api_key" not in worker_model
+    assert "uploads_data" not in worker_model
+    assert "      - data" in worker_model
+    assert "      - model" in worker_model
+    assert "      - egress" not in worker_model
+    assert "read_only: true" in worker_model
+    assert "no-new-privileges:true" in worker_model
+    assert "stop_grace_period: 150s" in worker_model
+
     assert '"127.0.0.1:8000:8000"' not in gateway
     assert "      - ingress" in gateway
     assert "      - data" in gateway
@@ -125,6 +164,18 @@ def test_compose_isolates_database_credentials_and_networks() -> None:
     assert "DEMO_PROVIDER_MODE: fake" in seed_demo_offline
     assert "./data/samples/public:/demo:ro" in seed_demo_offline
 
+    assert "secrets:" not in document_smoke
+    assert "POSTGRES_" not in document_smoke
+    assert "BAILIAN_" not in document_smoke
+    assert "model_gateway_" not in document_smoke
+    assert "DOCUMENT_NAMESPACE_MODE:" in document_smoke
+    assert "./data/samples/public:/demo:ro" in document_smoke
+    assert "      - ingress" in document_smoke
+    assert "      - data" not in document_smoke
+    assert "      - model" not in document_smoke
+    assert "      - egress" not in document_smoke
+    assert "read_only: true" in document_smoke
+
     assert re.search(r"(?ms)^  data:\n.*?^    internal: true$", COMPOSE)
     assert re.search(r"(?ms)^  ingress:\n.*?^    internal: true$", COMPOSE)
     assert re.search(r"(?ms)^  model:\n.*?^    internal: true$", COMPOSE)