Make the first RAG slice executable without risking production data

The Stage 1 foundation now proves provider contracts with mocks and validates PostgreSQL/pgvector ingestion, approval binding, retrieval, reranking, and idempotency using only synthetic data. Live Bailian validation remains gated on rotating the exposed key.

Constraint: The key shown in chat is compromised and cannot be used or committed

Rejected: Mark Stage 1 complete from mock and offline results | real three-model smoke is still required

Confidence: high

Scope-risk: moderate

Reversibility: clean

Directive: Do not enable real-data ingestion until Stage 3 cloud approval and outbound manifest controls are enforced end to end

Tested: make verify; 41 pytest tests; strict mypy; Ruff; Compose config; pinned image build; empty-volume migration; role denial; two idempotent 20-vector seeds; database restart persistence

Not-tested: Live Bailian calls require a newly rotated key; React product UI is not implemented

backend/app/persistence/job_queue_sql.py

@@ -0,0 +1,129 @@
+"""Fenced PostgreSQL job-queue statements.
+
+Callers must execute each statement in a transaction and treat an empty
+``RETURNING`` result as loss of the lease. No external model call should hold
+the claim transaction open.
+"""
+
+CLAIM_JOB_SQL = """
+WITH candidate AS (
+    SELECT job.id
+    FROM rag.background_jobs AS job
+    WHERE job.attempt < job.max_attempts
+      AND job.required_capability = ANY(CAST(:worker_capabilities AS text[]))
+      AND (
+        (job.status = 'QUEUED' AND job.run_after <= now())
+        OR (job.status = 'RUNNING' AND job.lease_until < now())
+      )
+    ORDER BY job.priority DESC, job.run_after, job.created_at
+    FOR UPDATE SKIP LOCKED
+    LIMIT 1
+)
+UPDATE rag.background_jobs AS job
+SET status = 'RUNNING',
+    lease_owner = :worker_id,
+    lease_token = gen_random_uuid(),
+    lease_until = now() + make_interval(secs => CAST(:lease_seconds AS integer)),
+    attempt = job.attempt + 1,
+    updated_at = now(),
+    finished_at = NULL
+FROM candidate
+WHERE job.id = candidate.id
+RETURNING job.*
+"""
+
+HEARTBEAT_JOB_SQL = """
+UPDATE rag.background_jobs AS job
+SET lease_until = now() + make_interval(secs => CAST(:lease_seconds AS integer)),
+    updated_at = now()
+WHERE job.id = :job_id
+  AND job.status = 'RUNNING'
+  AND job.lease_owner = :worker_id
+  AND job.lease_token = :lease_token
+RETURNING job.id, job.lease_until
+"""
+
+COMPLETE_JOB_SQL = """
+UPDATE rag.background_jobs AS job
+SET status = 'SUCCEEDED',
+    progress = 100,
+    lease_owner = NULL,
+    lease_token = NULL,
+    lease_until = NULL,
+    finished_at = now(),
+    updated_at = now()
+WHERE job.id = :job_id
+  AND job.status = 'RUNNING'
+  AND job.lease_owner = :worker_id
+  AND job.lease_token = :lease_token
+RETURNING job.*
+"""
+
+FAIL_OR_RETRY_JOB_SQL = """
+UPDATE rag.background_jobs AS job
+SET status = CASE
+      WHEN job.attempt < job.max_attempts THEN 'QUEUED'
+      ELSE 'FAILED'
+    END,
+    run_after = CASE
+      WHEN job.attempt < job.max_attempts
+        THEN now() + make_interval(
+          secs => GREATEST(CAST(:retry_delay_seconds AS integer), 0)
+        )
+      ELSE job.run_after
+    END,
+    last_error_code = :error_code,
+    last_error_message = left(:error_message, 2000),
+    lease_owner = NULL,
+    lease_token = NULL,
+    lease_until = NULL,
+    finished_at = CASE
+      WHEN job.attempt < job.max_attempts THEN NULL
+      ELSE now()
+    END,
+    updated_at = now()
+WHERE job.id = :job_id
+  AND job.status = 'RUNNING'
+  AND job.lease_owner = :worker_id
+  AND job.lease_token = :lease_token
+RETURNING job.*
+"""
+
+REAP_EXPIRED_JOBS_SQL = """
+WITH maintenance_lock AS (
+    SELECT pg_try_advisory_xact_lock(CAST(:lock_key AS bigint)) AS acquired
+),
+expired AS (
+    SELECT job.id
+    FROM rag.background_jobs AS job
+    CROSS JOIN maintenance_lock
+    WHERE maintenance_lock.acquired
+      AND job.status = 'RUNNING'
+      AND job.lease_until < now()
+    ORDER BY job.lease_until, job.created_at
+    FOR UPDATE OF job SKIP LOCKED
+    LIMIT CAST(:batch_size AS integer)
+)
+UPDATE rag.background_jobs AS job
+SET status = CASE
+      WHEN job.attempt < job.max_attempts THEN 'QUEUED'
+      ELSE 'FAILED'
+    END,
+    run_after = CASE
+      WHEN job.attempt < job.max_attempts THEN now()
+      ELSE job.run_after
+    END,
+    last_error_code = 'WORKER_LEASE_EXPIRED',
+    last_error_message = 'Worker lease expired before a fenced terminal update.',
+    lease_owner = NULL,
+    lease_token = NULL,
+    lease_until = NULL,
+    finished_at = CASE
+      WHEN job.attempt < job.max_attempts THEN NULL
+      ELSE now()
+    END,
+    updated_at = now()
+FROM expired
+WHERE job.id = expired.id
+RETURNING job.*
+"""