Make the governed RAG evidence path executable end to end

Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench.

Constraint: Live Bailian authentication currently fails for all three configured capabilities

Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding

Confidence: high

Scope-risk: broad

Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land

Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip

Not-tested: Successful live Bailian calls, OCR, real multi-user authorization

backend/app/tools/document_pipeline_smoke.py

@@ -0,0 +1,336 @@
+"""Reproducible HTTP smoke for upload -> review -> vector -> retrieval."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import time
+import uuid
+from pathlib import Path
+from typing import Any, cast
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+
+from app.core.demo_identity import BAILIAN_KNOWLEDGE_BASE_ID, KNOWLEDGE_BASE_ID
+
+
+class DocumentPipelineSmokeError(RuntimeError):
+    """A safe smoke failure without source text, paths, or response bodies."""
+
+
+def _request(
+    base_url: str,
+    method: str,
+    path: str,
+    *,
+    body: dict[str, object] | None = None,
+    content: bytes | None = None,
+    headers: dict[str, str] | None = None,
+) -> dict[str, Any]:
+    request_headers = {"Accept": "application/json", **(headers or {})}
+    payload: bytes | None = None
+    if body is not None:
+        payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
+        request_headers["Content-Type"] = "application/json"
+    elif content is not None:
+        payload = content
+        request_headers["Content-Type"] = "application/octet-stream"
+    request = Request(  # noqa: S310 - base URL is operator-configured HTTP(S)
+        f"{base_url.rstrip('/')}{path}",
+        data=payload,
+        headers=request_headers,
+        method=method,
+    )
+    try:
+        with urlopen(request, timeout=15) as response:  # noqa: S310 - configured local endpoint
+            parsed = json.loads(response.read())
+    except HTTPError as exc:
+        code = "UNKNOWN"
+        try:
+            problem = json.loads(exc.read())
+            if isinstance(problem, dict) and isinstance(problem.get("code"), str):
+                code = problem["code"]
+        except (OSError, ValueError, TypeError):
+            pass
+        raise DocumentPipelineSmokeError(f"HTTP {exc.code} ({code}) for {method} {path}") from None
+    except (URLError, TimeoutError, OSError, ValueError, TypeError):
+        raise DocumentPipelineSmokeError(f"request failed for {method} {path}") from None
+    if not isinstance(parsed, dict):
+        raise DocumentPipelineSmokeError(f"invalid response for {method} {path}")
+    return cast(dict[str, Any], parsed)
+
+
+def _wait_job(base_url: str, job_id: str, *, timeout_seconds: float) -> dict[str, Any]:
+    deadline = time.monotonic() + timeout_seconds
+    while time.monotonic() < deadline:
+        job = _request(base_url, "GET", f"/api/v1/document-jobs/{job_id}")
+        status = job.get("status")
+        if status == "SUCCEEDED":
+            return job
+        if status in {"FAILED", "CANCELLED"}:
+            code = job.get("last_error_code")
+            safe_code = code if isinstance(code, str) else "UNKNOWN"
+            raise DocumentPipelineSmokeError(f"job terminated with {status} ({safe_code})")
+        time.sleep(0.25)
+    raise DocumentPipelineSmokeError("job polling timed out")
+
+
+def _wait_document_ready(
+    base_url: str,
+    document_id: str,
+    *,
+    timeout_seconds: float,
+) -> dict[str, Any]:
+    deadline = time.monotonic() + timeout_seconds
+    while time.monotonic() < deadline:
+        detail = _request(base_url, "GET", f"/api/v1/documents/{document_id}")
+        document = detail.get("document")
+        if isinstance(document, dict) and document.get("status") == "READY":
+            return detail
+        if isinstance(document, dict) and document.get("status") in {"FAILED", "REJECTED"}:
+            raise DocumentPipelineSmokeError("document reached a non-ready terminal state")
+        time.sleep(0.25)
+    raise DocumentPipelineSmokeError("document activation polling timed out")
+
+
+def run_smoke(
+    *,
+    base_url: str,
+    sample_path: Path,
+    timeout_seconds: float = 90.0,
+    run_id: uuid.UUID | None = None,
+    knowledge_base_id: uuid.UUID = KNOWLEDGE_BASE_ID,
+) -> dict[str, object]:
+    endpoint = urlsplit(base_url)
+    if (
+        endpoint.scheme not in {"http", "https"}
+        or not endpoint.hostname
+        or endpoint.username is not None
+        or endpoint.password is not None
+    ):
+        raise DocumentPipelineSmokeError("RAG base URL must be credential-free HTTP(S)")
+    try:
+        sample_content = sample_path.read_bytes()
+    except OSError:
+        raise DocumentPipelineSmokeError("synthetic upload sample is unavailable") from None
+    if not sample_content or len(sample_content) > 1024 * 1024:
+        raise DocumentPipelineSmokeError("synthetic upload sample has an invalid size")
+    smoke_run_id = run_id or uuid.uuid4()
+    content = sample_content + f"\n\nSynthetic smoke run: {smoke_run_id}\n".encode()
+    digest = hashlib.sha256(content).hexdigest()
+    key = uuid.uuid5(uuid.NAMESPACE_URL, f"geological-rag-document-smoke:{digest}")
+    filename = f"upload_demo-{smoke_run_id.hex[:12]}.md"
+    declaration_body: dict[str, object] = {
+        "filename": filename,
+        "declared_mime_type": "text/markdown",
+        "expected_size": len(content),
+        "expected_sha256": digest,
+    }
+    declared = _request(
+        base_url,
+        "POST",
+        "/api/v1/document-uploads",
+        headers={"Idempotency-Key": str(key)},
+        body=declaration_body,
+    )
+    if declared.get("replayed") is not False:
+        raise DocumentPipelineSmokeError("fresh upload declaration was unexpectedly replayed")
+    upload_id = _required_uuid_text(declared, "id")
+    _request(
+        base_url,
+        "PUT",
+        f"/api/v1/document-uploads/{upload_id}/content",
+        content=content,
+    )
+    completed = _request(
+        base_url,
+        "POST",
+        f"/api/v1/document-uploads/{upload_id}/complete",
+    )
+    document = _required_mapping(completed, "document")
+    document_id = _required_uuid_text(document, "id")
+    parse_job = _required_mapping(completed, "job")
+    parse_job_id = _required_uuid_text(parse_job, "id")
+    parsed = _wait_job(base_url, parse_job_id, timeout_seconds=timeout_seconds)
+    if parsed.get("stage") != "LOCAL_PARSED_PENDING_CLOUD_REVIEW":
+        raise DocumentPipelineSmokeError("parse job did not reach the review stage")
+
+    review = _request(
+        base_url,
+        "GET",
+        f"/api/v1/documents/{document_id}/review-bundle?after_ordinal=-1&limit=100",
+    )
+    version = _required_mapping(review, "version")
+    version_id = _required_uuid_text(version, "id")
+    review_state = version.get("review_state")
+    if review_state == "LOCAL_PARSED_PENDING_CLOUD_REVIEW":
+        manifest = _required_hash(version, "outbound_manifest_sha256")
+        revision = version.get("review_revision")
+        if not isinstance(revision, int) or isinstance(revision, bool) or revision < 0:
+            raise DocumentPipelineSmokeError("review revision is invalid")
+        decision = _request(
+            base_url,
+            "POST",
+            f"/api/v1/documents/{document_id}/review-decisions",
+            body={
+                "decision": "APPROVE",
+                "reason_code": "SYNTHETIC_REVIEW_APPROVED",
+                "expected_revision": revision,
+                "outbound_manifest_sha256": manifest,
+            },
+        )
+        embedding_job = _required_mapping(decision, "job")
+        embedding_job_id = _required_uuid_text(embedding_job, "id")
+        _wait_job(base_url, embedding_job_id, timeout_seconds=timeout_seconds)
+    elif review_state != "CLOUD_APPROVED":
+        raise DocumentPipelineSmokeError("document version is not eligible for indexing")
+
+    ready = _wait_document_ready(
+        base_url,
+        document_id,
+        timeout_seconds=timeout_seconds,
+    )
+    ready_document = _required_mapping(ready, "document")
+    if ready_document.get("active_version_id") != version_id:
+        raise DocumentPipelineSmokeError("ready document did not activate the reviewed version")
+
+    retrieval = _request(
+        base_url,
+        "POST",
+        "/api/v1/retrieval/search",
+        body={
+            "knowledge_base_id": str(knowledge_base_id),
+            "query": "海岳示范区萤石矿需要哪些综合找矿标志？",
+            "vector_top_k": 50,
+            "rerank_top_n": 10,
+        },
+    )
+    results = retrieval.get("results")
+    if not isinstance(results, list):
+        raise DocumentPipelineSmokeError("retrieval result is invalid")
+    match = next(
+        (
+            item
+            for item in results
+            if isinstance(item, dict) and item.get("document_id") == document_id
+        ),
+        None,
+    )
+    if match is None:
+        raise DocumentPipelineSmokeError("uploaded document was not retrieved")
+
+    replayed = _request(
+        base_url,
+        "POST",
+        "/api/v1/document-uploads",
+        headers={"Idempotency-Key": str(key)},
+        body=declaration_body,
+    )
+    if replayed.get("replayed") is not True or _required_uuid_text(replayed, "id") != upload_id:
+        raise DocumentPipelineSmokeError("upload declaration replay contract failed")
+    _request(
+        base_url,
+        "PUT",
+        f"/api/v1/document-uploads/{upload_id}/content",
+        content=content,
+    )
+    replayed_completion = _request(
+        base_url,
+        "POST",
+        f"/api/v1/document-uploads/{upload_id}/complete",
+    )
+    replayed_document = _required_mapping(replayed_completion, "document")
+    replayed_job = _required_mapping(replayed_completion, "job")
+    if _required_uuid_text(replayed_document, "id") != document_id:
+        raise DocumentPipelineSmokeError("document identity changed during replay")
+    if _required_uuid_text(replayed_job, "id") != parse_job_id:
+        raise DocumentPipelineSmokeError("parse job identity changed during replay")
+    replayed_ready = _wait_document_ready(
+        base_url,
+        document_id,
+        timeout_seconds=timeout_seconds,
+    )
+    if _required_mapping(replayed_ready, "document").get("active_version_id") != version_id:
+        raise DocumentPipelineSmokeError("active version changed during replay")
+    return {
+        "status": "ok",
+        "run_id": str(smoke_run_id),
+        "knowledge_base_id": str(knowledge_base_id),
+        "document_id": document_id,
+        "document_version_id": version_id,
+        "parse_job_id": parse_job_id,
+        "document_status": ready_document.get("status"),
+        "parse_stage": parsed.get("stage"),
+        "retrieval_rank": match.get("rank"),
+        "citation_id": match.get("citation_id"),
+        "embedding_model": retrieval.get("embedding_model"),
+        "rerank_status": retrieval.get("rerank_status"),
+        "replay_confirmed": True,
+    }
+
+
+def _required_mapping(value: dict[str, Any], key: str) -> dict[str, Any]:
+    item = value.get(key)
+    if not isinstance(item, dict):
+        raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
+    return cast(dict[str, Any], item)
+
+
+def _required_uuid_text(value: dict[str, Any], key: str) -> str:
+    item = value.get(key)
+    if not isinstance(item, str):
+        raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
+    try:
+        parsed = uuid.UUID(item)
+    except ValueError:
+        raise DocumentPipelineSmokeError(f"response field is invalid: {key}") from None
+    if str(parsed) != item:
+        raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
+    return item
+
+
+def _required_hash(value: dict[str, Any], key: str) -> str:
+    item = value.get(key)
+    if (
+        not isinstance(item, str)
+        or len(item) != 64
+        or any(character not in "0123456789abcdef" for character in item)
+    ):
+        raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
+    return item
+
+
+def main() -> None:
+    base_url = os.getenv("RAG_BASE_URL", "http://127.0.0.1:8000")
+    sample_path = Path(os.getenv("RAG_UPLOAD_SAMPLE", "data/samples/public/upload_demo.md"))
+    namespace_mode = os.getenv("DOCUMENT_NAMESPACE_MODE", "fake").strip().lower()
+    if namespace_mode == "fake":
+        knowledge_base_id = KNOWLEDGE_BASE_ID
+    elif namespace_mode == "bailian":
+        knowledge_base_id = BAILIAN_KNOWLEDGE_BASE_ID
+    else:
+        sys.stdout.write(
+            json.dumps(
+                {"status": "failed", "error": "document namespace mode is invalid"},
+                sort_keys=True,
+            )
+            + "\n"
+        )
+        raise SystemExit(1)
+    try:
+        result = run_smoke(
+            base_url=base_url,
+            sample_path=sample_path,
+            knowledge_base_id=knowledge_base_id,
+        )
+    except DocumentPipelineSmokeError as exc:
+        sys.stdout.write(json.dumps({"status": "failed", "error": str(exc)}, sort_keys=True) + "\n")
+        raise SystemExit(1) from None
+    sys.stdout.write(json.dumps(result, sort_keys=True) + "\n")
+
+
+if __name__ == "__main__":
+    main()

backend/app/tools/evaluate_demo.py

@@ -0,0 +1,213 @@
+"""Run a reproducible retrieval evaluation on the public synthetic corpus."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import sys
+import uuid
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Protocol
+
+from app.adapters.fake import FakeEmbeddingProvider, FakeReranker
+from app.core.config import Settings
+from app.core.demo_identity import ACCESS_SCOPE_ID, KNOWLEDGE_BASE_ID
+from app.persistence.retrieval import PostgresRetrievalRepository
+from app.services.evaluation import (
+    RankingMetrics,
+    bootstrap_mean_confidence_interval,
+    evaluate_ranking,
+    freeze_run_config,
+)
+from app.services.retrieval import RetrievalActor, RetrievalGrant, RetrievalResult, RetrievalService
+from app.tools.seed_demo import (
+    DEFAULT_SAMPLE_ROOT,
+    DemoDocument,
+    DemoQuery,
+    load_documents,
+    load_queries,
+)
+
+
+def _sha256_file(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _source_id(source_name: str) -> str:
+    return Path(source_name).stem
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values) if values else 0.0
+
+
+class DemoRetrievalService(Protocol):
+    async def search(
+        self,
+        *,
+        actor: RetrievalActor,
+        knowledge_base_id: uuid.UUID,
+        query: str,
+        vector_top_k: int,
+        rerank_top_n: int,
+    ) -> RetrievalResult: ...
+
+
+async def evaluate_demo_queries(
+    *,
+    service: DemoRetrievalService,
+    actor: RetrievalActor,
+    documents: list[DemoDocument],
+    queries: list[DemoQuery],
+    vector_top_k: int = 20,
+    rerank_top_n: int = 10,
+    metric_cutoff: int = 3,
+) -> dict[str, Any]:
+    """Evaluate answerable queries with a fully judged synthetic corpus pool."""
+
+    corpus_ids = frozenset(document.source_id for document in documents)
+    if len(corpus_ids) != len(documents):
+        raise ValueError("synthetic corpus document IDs must be unique")
+
+    cases: list[dict[str, Any]] = []
+    scored: list[RankingMetrics] = []
+    active_profile_hash: str | None = None
+    for query in queries:
+        result = await service.search(
+            actor=actor,
+            knowledge_base_id=KNOWLEDGE_BASE_ID,
+            query=query.query,
+            vector_top_k=vector_top_k,
+            rerank_top_n=rerank_top_n,
+        )
+        if active_profile_hash is None:
+            active_profile_hash = result.profile.profile_hash
+        elif active_profile_hash != result.profile.profile_hash:
+            raise ValueError("active profile changed during evaluation")
+
+        ranked_ids = [_source_id(hit.source_name) for hit in result.results]
+        case: dict[str, Any] = {
+            "qid": query.qid,
+            "answerable": query.answerable,
+            "ranked_document_ids": ranked_ids,
+            "retrieval_status": result.status,
+            "rerank_status": result.rerank_status,
+        }
+        if query.answerable:
+            relevance = {document_id: 0.0 for document_id in corpus_ids}
+            for expected_id in query.expected_doc_ids:
+                if expected_id not in corpus_ids:
+                    raise ValueError("expected document is outside the corpus manifest")
+                relevance[expected_id] = 1.0
+            groups = tuple(frozenset({expected_id}) for expected_id in query.expected_doc_ids)
+            metrics = evaluate_ranking(
+                ranked_ids,
+                relevance=relevance,
+                judged_document_ids=corpus_ids,
+                evidence_groups=groups,
+                k=metric_cutoff,
+            )
+            scored.append(metrics)
+            case["metrics"] = asdict(metrics)
+        else:
+            case["metrics"] = None
+        cases.append(case)
+
+    if active_profile_hash is None:
+        raise ValueError("evaluation query set is empty")
+    hit_values = [metric.hit_at_k for metric in scored]
+    hit_ci = bootstrap_mean_confidence_interval(
+        hit_values,
+        seed=20260713,
+        iterations=2_000,
+    )
+    return {
+        "status": "ok",
+        "dataset": "synthetic-demo",
+        "case_count": len(queries),
+        "answerable_case_count": len(scored),
+        "active_embedding_profile_hash": active_profile_hash,
+        "metrics": {
+            f"hit_at_{metric_cutoff}": _mean(hit_values),
+            "mrr": _mean([metric.reciprocal_rank for metric in scored]),
+            f"ndcg_at_{metric_cutoff}": _mean([metric.ndcg_at_k for metric in scored]),
+            f"complete_hit_at_{metric_cutoff}": _mean(
+                [metric.complete_hit_at_k for metric in scored]
+            ),
+            f"evidence_group_recall_at_{metric_cutoff}": _mean(
+                [metric.evidence_group_recall_at_k for metric in scored]
+            ),
+            f"hit_at_{metric_cutoff}_confidence_interval": asdict(hit_ci),
+        },
+        "cases": cases,
+    }
+
+
+async def async_main() -> int:
+    document_path = Path(
+        sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SAMPLE_ROOT / "demo_documents.jsonl"
+    )
+    query_path = Path(
+        sys.argv[2] if len(sys.argv) > 2 else DEFAULT_SAMPLE_ROOT / "demo_queries.jsonl"
+    )
+    try:
+        settings = Settings()
+        documents = load_documents(document_path)
+        queries = load_queries(query_path)
+        service = RetrievalService(
+            repository=PostgresRetrievalRepository(settings),
+            embedding_provider=FakeEmbeddingProvider(settings.embedding_dimension),
+            reranker=FakeReranker(),
+            synthetic_embedding_provider=FakeEmbeddingProvider(settings.embedding_dimension),
+            synthetic_reranker=FakeReranker(),
+        )
+        actor = RetrievalActor(
+            subject="synthetic-evaluation-runner",
+            grants=(
+                RetrievalGrant(
+                    knowledge_base_id=KNOWLEDGE_BASE_ID,
+                    access_scope_ids=(ACCESS_SCOPE_ID,),
+                ),
+            ),
+        )
+        artifact = await evaluate_demo_queries(
+            service=service,
+            actor=actor,
+            documents=documents,
+            queries=queries,
+        )
+        config, config_hash = freeze_run_config(
+            {
+                "corpus_sha256": _sha256_file(document_path),
+                "query_set_sha256": _sha256_file(query_path),
+                "embedding_profile_hash": artifact["active_embedding_profile_hash"],
+                "vector_top_k": 20,
+                "rerank_top_n": 10,
+                "metric_cutoff": 3,
+                "bootstrap_seed": 20260713,
+            }
+        )
+        artifact["frozen_config"] = json.loads(config)
+        artifact["frozen_config_sha256"] = config_hash
+        sys.stdout.write(json.dumps(artifact, ensure_ascii=False, sort_keys=True) + "\n")
+        return 0
+    except Exception:
+        # CLI output remains fixed and does not echo paths, document text, DSNs, or errors.
+        sys.stdout.write(
+            json.dumps(
+                {"status": "failed", "error_kind": "evaluation_failed"},
+                sort_keys=True,
+            )
+            + "\n"
+        )
+        return 1
+
+
+def main() -> None:
+    raise SystemExit(asyncio.run(async_main()))
+
+
+if __name__ == "__main__":
+    main()

backend/app/tools/export_openapi.py

@@ -0,0 +1,31 @@
+"""Export the FastAPI contract without opening a database or reading secrets."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+from app.main import create_app
+
+
+def export_schema() -> dict[str, Any]:
+    """Build the deterministic application schema from import-safe contracts."""
+
+    return create_app().openapi()
+
+
+def main() -> None:
+    sys.stdout.write(
+        json.dumps(
+            export_schema(),
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        + "\n"
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/app/tools/init_upload_storage.py

@@ -0,0 +1,66 @@
+"""Initialize the persistent upload volume without reading application secrets."""
+
+from __future__ import annotations
+
+import json
+import os
+import stat
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+APP_UID = 10_001
+APP_GID = 10_001
+
+
+class UploadStorageInitializationError(RuntimeError):
+    """A path-neutral initialization failure safe for container logs."""
+
+
+def initialize_upload_root(
+    root: Path,
+    *,
+    uid: int = APP_UID,
+    gid: int = APP_GID,
+    change_owner: Callable[[Path, int, int], None] | None = None,
+) -> None:
+    if not root.is_absolute() or uid < 1 or gid < 1:
+        raise UploadStorageInitializationError("invalid upload root contract")
+    try:
+        if root.exists() and root.is_symlink():
+            raise UploadStorageInitializationError("unsafe upload root")
+        root.mkdir(mode=0o750, parents=True, exist_ok=True)
+        if root.is_symlink() or not root.is_dir():
+            raise UploadStorageInitializationError("unsafe upload root")
+        owner = change_owner or (
+            lambda path, owner_uid, owner_gid: os.chown(path, owner_uid, owner_gid)
+        )
+        owner(root, uid, gid)
+        root.chmod(0o750, follow_symlinks=False)
+        metadata = root.stat(follow_symlinks=False)
+        if (
+            not stat.S_ISDIR(metadata.st_mode)
+            or metadata.st_uid != uid
+            or metadata.st_gid != gid
+            or stat.S_IMODE(metadata.st_mode) != 0o750
+        ):
+            raise UploadStorageInitializationError("upload root ownership verification failed")
+    except UploadStorageInitializationError:
+        raise
+    except OSError:
+        raise UploadStorageInitializationError("upload root initialization failed") from None
+
+
+def main() -> None:
+    try:
+        initialize_upload_root(Path(os.getenv("UPLOAD_ROOT", "/data/uploads")))
+    except UploadStorageInitializationError:
+        sys.stdout.write(
+            json.dumps({"status": "failed", "error_kind": "storage_init_failed"}) + "\n"
+        )
+        raise SystemExit(1) from None
+    sys.stdout.write(json.dumps({"status": "ok"}) + "\n")
+
+
+if __name__ == "__main__":
+    main()

backend/app/tools/seed_demo.py

@@ -25,6 +25,8 @@ from app.adapters.model_gateway import ModelGatewayAdapter
 from app.core.config import Settings
 from app.core.demo_identity import (
     ACCESS_SCOPE_ID,
+    BAILIAN_ACCESS_SCOPE_ID,
+    BAILIAN_KNOWLEDGE_BASE_ID,
     IDENTITY_NAMESPACE,
     KNOWLEDGE_BASE_ID,
     offline_embedding_profile_hash,
@@ -75,8 +77,8 @@ OFFLINE_NAMESPACE = DemoNamespace(
 )
 BAILIAN_NAMESPACE = DemoNamespace(
     mode="bailian",
-    knowledge_base_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-knowledge-base"),
-    access_scope_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-public-scope"),
+    knowledge_base_id=BAILIAN_KNOWLEDGE_BASE_ID,
+    access_scope_id=BAILIAN_ACCESS_SCOPE_ID,
     scope_name="synthetic-bailian-validation",
     knowledge_base_name="虚构地质 PoC 知识库（百炼验证）",
     storage_prefix="synthetic/bailian",

backend/app/tools/worker_smoke.py

@@ -0,0 +1,158 @@
+"""Run a destructive-free PostgreSQL smoke test for job lease fencing.
+
+The command creates one synthetic queue row, races two claimers, verifies that
+only one wins, proves that a stale token is rejected, completes the winning
+lease, and removes the synthetic row before exiting.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+
+import psycopg
+
+from app.core.config import Settings
+from app.persistence.job_queue import (
+    BackgroundJob,
+    JobLease,
+    LeaseLostError,
+    PsycopgJobQueue,
+)
+
+
+def _dsn(settings: Settings) -> str:
+    url = settings.database_url().set(drivername="postgresql")
+    return url.render_as_string(hide_password=False)
+
+
+def _insert_job(dsn: str, job_id: uuid.UUID, capability: str, idempotency_key: str) -> None:
+    with psycopg.connect(dsn, connect_timeout=5) as connection:
+        connection.execute(
+            """
+            INSERT INTO rag.background_jobs (
+              id, job_type, required_capability, resource_type, resource_id,
+              idempotency_key, payload, stage, max_attempts
+            ) VALUES (%s, 'WORKER_SMOKE', %s, 'synthetic_smoke', %s, %s, '{}'::jsonb,
+                      'VERIFYING_FENCE', 2)
+            """,
+            (job_id, capability, job_id, idempotency_key),
+        )
+
+
+def _delete_job(dsn: str, job_id: uuid.UUID) -> None:
+    with psycopg.connect(dsn, connect_timeout=5) as connection:
+        connection.execute("DELETE FROM rag.background_jobs WHERE id = %s", (job_id,))
+
+
+def _expire_lease(dsn: str, job_id: uuid.UUID) -> None:
+    """Move only the synthetic smoke lease into the past for recovery verification."""
+
+    with psycopg.connect(dsn, connect_timeout=5) as connection:
+        connection.execute(
+            """
+            UPDATE rag.background_jobs
+               SET lease_until = now() - interval '1 second'
+             WHERE id = %s AND status = 'RUNNING'
+            """,
+            (job_id,),
+        )
+
+
+def _race_claim(
+    queues: tuple[PsycopgJobQueue, PsycopgJobQueue],
+    capability: str,
+    *,
+    worker_prefix: str,
+) -> tuple[BackgroundJob | None, BackgroundJob | None]:
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        return tuple(
+            executor.map(
+                lambda item: item[0].claim(
+                    worker_id=item[1],
+                    worker_capabilities=(capability,),
+                    lease_seconds=30,
+                ),
+                (
+                    (queues[0], f"{worker_prefix}-a"),
+                    (queues[1], f"{worker_prefix}-b"),
+                ),
+            )
+        )
+
+
+def run_smoke(settings: Settings) -> dict[str, object]:
+    dsn = _dsn(settings)
+    job_id = uuid.uuid4()
+    nonce = uuid.uuid4().hex
+    capability = f"worker-smoke-{nonce}"
+    idempotency_key = f"worker-smoke:{nonce}"
+    _insert_job(dsn, job_id, capability, idempotency_key)
+    try:
+        queues = (PsycopgJobQueue(dsn), PsycopgJobQueue(dsn))
+        claims = _race_claim(queues, capability, worker_prefix="smoke-worker")
+        winners = tuple(claim for claim in claims if claim is not None)
+        if len(winners) != 1:
+            raise RuntimeError("concurrent claim contract failed")
+        winner = winners[0]
+        stale = JobLease(
+            job_id=winner.lease.job_id,
+            worker_id=winner.lease.worker_id,
+            lease_token=uuid.uuid4(),
+        )
+        try:
+            queues[0].heartbeat(stale, lease_seconds=30)
+        except LeaseLostError:
+            fence_rejected = True
+        else:
+            fence_rejected = False
+        if not fence_rejected:
+            raise RuntimeError("stale lease fence was accepted")
+        queues[0].heartbeat(winner.lease, lease_seconds=30)
+        _expire_lease(dsn, job_id)
+        try:
+            queues[0].heartbeat(winner.lease, lease_seconds=30)
+        except LeaseLostError:
+            expired_lease_rejected = True
+        else:
+            expired_lease_rejected = False
+        if not expired_lease_rejected:
+            raise RuntimeError("expired lease was renewed")
+        recovery_claims = _race_claim(queues, capability, worker_prefix="recovery-worker")
+        recovery_winners = tuple(claim for claim in recovery_claims if claim is not None)
+        if len(recovery_winners) != 1:
+            raise RuntimeError("expired lease recovery contract failed")
+        recovered = recovery_winners[0]
+        if recovered.lease.lease_token == winner.lease.lease_token:
+            raise RuntimeError("recovered lease did not rotate its fence token")
+        terminal = queues[0].complete(recovered.lease)
+        if terminal.status != "SUCCEEDED":
+            raise RuntimeError("winning lease did not complete")
+        return {
+            "status": "ok",
+            "claim_winners": 1,
+            "stale_fence_rejected": True,
+            "expired_lease_rejected": True,
+            "recovery_claim_winners": 1,
+            "recovery_token_rotated": True,
+            "terminal_status": terminal.status,
+        }
+    finally:
+        _delete_job(dsn, job_id)
+
+
+def main() -> None:
+    try:
+        result = run_smoke(Settings())
+        exit_code = 0
+    except Exception:
+        result = {"status": "failed", "error_kind": "worker_smoke_failed"}
+        exit_code = 1
+    sys.stdout.write(json.dumps(result, sort_keys=True) + "\n")
+    raise SystemExit(exit_code)
+
+
+if __name__ == "__main__":
+    main()