Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
This commit is contained in:
@@ -0,0 +1,336 @@
|
||||
"""Reproducible HTTP smoke for upload -> review -> vector -> retrieval."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from app.core.demo_identity import BAILIAN_KNOWLEDGE_BASE_ID, KNOWLEDGE_BASE_ID
|
||||
|
||||
|
||||
class DocumentPipelineSmokeError(RuntimeError):
|
||||
"""A safe smoke failure without source text, paths, or response bodies."""
|
||||
|
||||
|
||||
def _request(
|
||||
base_url: str,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, object] | None = None,
|
||||
content: bytes | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
request_headers = {"Accept": "application/json", **(headers or {})}
|
||||
payload: bytes | None = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode()
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
elif content is not None:
|
||||
payload = content
|
||||
request_headers["Content-Type"] = "application/octet-stream"
|
||||
request = Request( # noqa: S310 - base URL is operator-configured HTTP(S)
|
||||
f"{base_url.rstrip('/')}{path}",
|
||||
data=payload,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=15) as response: # noqa: S310 - configured local endpoint
|
||||
parsed = json.loads(response.read())
|
||||
except HTTPError as exc:
|
||||
code = "UNKNOWN"
|
||||
try:
|
||||
problem = json.loads(exc.read())
|
||||
if isinstance(problem, dict) and isinstance(problem.get("code"), str):
|
||||
code = problem["code"]
|
||||
except (OSError, ValueError, TypeError):
|
||||
pass
|
||||
raise DocumentPipelineSmokeError(f"HTTP {exc.code} ({code}) for {method} {path}") from None
|
||||
except (URLError, TimeoutError, OSError, ValueError, TypeError):
|
||||
raise DocumentPipelineSmokeError(f"request failed for {method} {path}") from None
|
||||
if not isinstance(parsed, dict):
|
||||
raise DocumentPipelineSmokeError(f"invalid response for {method} {path}")
|
||||
return cast(dict[str, Any], parsed)
|
||||
|
||||
|
||||
def _wait_job(base_url: str, job_id: str, *, timeout_seconds: float) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
job = _request(base_url, "GET", f"/api/v1/document-jobs/{job_id}")
|
||||
status = job.get("status")
|
||||
if status == "SUCCEEDED":
|
||||
return job
|
||||
if status in {"FAILED", "CANCELLED"}:
|
||||
code = job.get("last_error_code")
|
||||
safe_code = code if isinstance(code, str) else "UNKNOWN"
|
||||
raise DocumentPipelineSmokeError(f"job terminated with {status} ({safe_code})")
|
||||
time.sleep(0.25)
|
||||
raise DocumentPipelineSmokeError("job polling timed out")
|
||||
|
||||
|
||||
def _wait_document_ready(
|
||||
base_url: str,
|
||||
document_id: str,
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
detail = _request(base_url, "GET", f"/api/v1/documents/{document_id}")
|
||||
document = detail.get("document")
|
||||
if isinstance(document, dict) and document.get("status") == "READY":
|
||||
return detail
|
||||
if isinstance(document, dict) and document.get("status") in {"FAILED", "REJECTED"}:
|
||||
raise DocumentPipelineSmokeError("document reached a non-ready terminal state")
|
||||
time.sleep(0.25)
|
||||
raise DocumentPipelineSmokeError("document activation polling timed out")
|
||||
|
||||
|
||||
def run_smoke(
|
||||
*,
|
||||
base_url: str,
|
||||
sample_path: Path,
|
||||
timeout_seconds: float = 90.0,
|
||||
run_id: uuid.UUID | None = None,
|
||||
knowledge_base_id: uuid.UUID = KNOWLEDGE_BASE_ID,
|
||||
) -> dict[str, object]:
|
||||
endpoint = urlsplit(base_url)
|
||||
if (
|
||||
endpoint.scheme not in {"http", "https"}
|
||||
or not endpoint.hostname
|
||||
or endpoint.username is not None
|
||||
or endpoint.password is not None
|
||||
):
|
||||
raise DocumentPipelineSmokeError("RAG base URL must be credential-free HTTP(S)")
|
||||
try:
|
||||
sample_content = sample_path.read_bytes()
|
||||
except OSError:
|
||||
raise DocumentPipelineSmokeError("synthetic upload sample is unavailable") from None
|
||||
if not sample_content or len(sample_content) > 1024 * 1024:
|
||||
raise DocumentPipelineSmokeError("synthetic upload sample has an invalid size")
|
||||
smoke_run_id = run_id or uuid.uuid4()
|
||||
content = sample_content + f"\n\nSynthetic smoke run: {smoke_run_id}\n".encode()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
key = uuid.uuid5(uuid.NAMESPACE_URL, f"geological-rag-document-smoke:{digest}")
|
||||
filename = f"upload_demo-{smoke_run_id.hex[:12]}.md"
|
||||
declaration_body: dict[str, object] = {
|
||||
"filename": filename,
|
||||
"declared_mime_type": "text/markdown",
|
||||
"expected_size": len(content),
|
||||
"expected_sha256": digest,
|
||||
}
|
||||
declared = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
"/api/v1/document-uploads",
|
||||
headers={"Idempotency-Key": str(key)},
|
||||
body=declaration_body,
|
||||
)
|
||||
if declared.get("replayed") is not False:
|
||||
raise DocumentPipelineSmokeError("fresh upload declaration was unexpectedly replayed")
|
||||
upload_id = _required_uuid_text(declared, "id")
|
||||
_request(
|
||||
base_url,
|
||||
"PUT",
|
||||
f"/api/v1/document-uploads/{upload_id}/content",
|
||||
content=content,
|
||||
)
|
||||
completed = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
f"/api/v1/document-uploads/{upload_id}/complete",
|
||||
)
|
||||
document = _required_mapping(completed, "document")
|
||||
document_id = _required_uuid_text(document, "id")
|
||||
parse_job = _required_mapping(completed, "job")
|
||||
parse_job_id = _required_uuid_text(parse_job, "id")
|
||||
parsed = _wait_job(base_url, parse_job_id, timeout_seconds=timeout_seconds)
|
||||
if parsed.get("stage") != "LOCAL_PARSED_PENDING_CLOUD_REVIEW":
|
||||
raise DocumentPipelineSmokeError("parse job did not reach the review stage")
|
||||
|
||||
review = _request(
|
||||
base_url,
|
||||
"GET",
|
||||
f"/api/v1/documents/{document_id}/review-bundle?after_ordinal=-1&limit=100",
|
||||
)
|
||||
version = _required_mapping(review, "version")
|
||||
version_id = _required_uuid_text(version, "id")
|
||||
review_state = version.get("review_state")
|
||||
if review_state == "LOCAL_PARSED_PENDING_CLOUD_REVIEW":
|
||||
manifest = _required_hash(version, "outbound_manifest_sha256")
|
||||
revision = version.get("review_revision")
|
||||
if not isinstance(revision, int) or isinstance(revision, bool) or revision < 0:
|
||||
raise DocumentPipelineSmokeError("review revision is invalid")
|
||||
decision = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
f"/api/v1/documents/{document_id}/review-decisions",
|
||||
body={
|
||||
"decision": "APPROVE",
|
||||
"reason_code": "SYNTHETIC_REVIEW_APPROVED",
|
||||
"expected_revision": revision,
|
||||
"outbound_manifest_sha256": manifest,
|
||||
},
|
||||
)
|
||||
embedding_job = _required_mapping(decision, "job")
|
||||
embedding_job_id = _required_uuid_text(embedding_job, "id")
|
||||
_wait_job(base_url, embedding_job_id, timeout_seconds=timeout_seconds)
|
||||
elif review_state != "CLOUD_APPROVED":
|
||||
raise DocumentPipelineSmokeError("document version is not eligible for indexing")
|
||||
|
||||
ready = _wait_document_ready(
|
||||
base_url,
|
||||
document_id,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
ready_document = _required_mapping(ready, "document")
|
||||
if ready_document.get("active_version_id") != version_id:
|
||||
raise DocumentPipelineSmokeError("ready document did not activate the reviewed version")
|
||||
|
||||
retrieval = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
"/api/v1/retrieval/search",
|
||||
body={
|
||||
"knowledge_base_id": str(knowledge_base_id),
|
||||
"query": "海岳示范区萤石矿需要哪些综合找矿标志？",
|
||||
"vector_top_k": 50,
|
||||
"rerank_top_n": 10,
|
||||
},
|
||||
)
|
||||
results = retrieval.get("results")
|
||||
if not isinstance(results, list):
|
||||
raise DocumentPipelineSmokeError("retrieval result is invalid")
|
||||
match = next(
|
||||
(
|
||||
item
|
||||
for item in results
|
||||
if isinstance(item, dict) and item.get("document_id") == document_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
raise DocumentPipelineSmokeError("uploaded document was not retrieved")
|
||||
|
||||
replayed = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
"/api/v1/document-uploads",
|
||||
headers={"Idempotency-Key": str(key)},
|
||||
body=declaration_body,
|
||||
)
|
||||
if replayed.get("replayed") is not True or _required_uuid_text(replayed, "id") != upload_id:
|
||||
raise DocumentPipelineSmokeError("upload declaration replay contract failed")
|
||||
_request(
|
||||
base_url,
|
||||
"PUT",
|
||||
f"/api/v1/document-uploads/{upload_id}/content",
|
||||
content=content,
|
||||
)
|
||||
replayed_completion = _request(
|
||||
base_url,
|
||||
"POST",
|
||||
f"/api/v1/document-uploads/{upload_id}/complete",
|
||||
)
|
||||
replayed_document = _required_mapping(replayed_completion, "document")
|
||||
replayed_job = _required_mapping(replayed_completion, "job")
|
||||
if _required_uuid_text(replayed_document, "id") != document_id:
|
||||
raise DocumentPipelineSmokeError("document identity changed during replay")
|
||||
if _required_uuid_text(replayed_job, "id") != parse_job_id:
|
||||
raise DocumentPipelineSmokeError("parse job identity changed during replay")
|
||||
replayed_ready = _wait_document_ready(
|
||||
base_url,
|
||||
document_id,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if _required_mapping(replayed_ready, "document").get("active_version_id") != version_id:
|
||||
raise DocumentPipelineSmokeError("active version changed during replay")
|
||||
return {
|
||||
"status": "ok",
|
||||
"run_id": str(smoke_run_id),
|
||||
"knowledge_base_id": str(knowledge_base_id),
|
||||
"document_id": document_id,
|
||||
"document_version_id": version_id,
|
||||
"parse_job_id": parse_job_id,
|
||||
"document_status": ready_document.get("status"),
|
||||
"parse_stage": parsed.get("stage"),
|
||||
"retrieval_rank": match.get("rank"),
|
||||
"citation_id": match.get("citation_id"),
|
||||
"embedding_model": retrieval.get("embedding_model"),
|
||||
"rerank_status": retrieval.get("rerank_status"),
|
||||
"replay_confirmed": True,
|
||||
}
|
||||
|
||||
|
||||
def _required_mapping(value: dict[str, Any], key: str) -> dict[str, Any]:
|
||||
item = value.get(key)
|
||||
if not isinstance(item, dict):
|
||||
raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
|
||||
return cast(dict[str, Any], item)
|
||||
|
||||
|
||||
def _required_uuid_text(value: dict[str, Any], key: str) -> str:
|
||||
item = value.get(key)
|
||||
if not isinstance(item, str):
|
||||
raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
|
||||
try:
|
||||
parsed = uuid.UUID(item)
|
||||
except ValueError:
|
||||
raise DocumentPipelineSmokeError(f"response field is invalid: {key}") from None
|
||||
if str(parsed) != item:
|
||||
raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
|
||||
return item
|
||||
|
||||
|
||||
def _required_hash(value: dict[str, Any], key: str) -> str:
|
||||
item = value.get(key)
|
||||
if (
|
||||
not isinstance(item, str)
|
||||
or len(item) != 64
|
||||
or any(character not in "0123456789abcdef" for character in item)
|
||||
):
|
||||
raise DocumentPipelineSmokeError(f"response field is invalid: {key}")
|
||||
return item
|
||||
|
||||
|
||||
def main() -> None:
|
||||
base_url = os.getenv("RAG_BASE_URL", "http://127.0.0.1:8000")
|
||||
sample_path = Path(os.getenv("RAG_UPLOAD_SAMPLE", "data/samples/public/upload_demo.md"))
|
||||
namespace_mode = os.getenv("DOCUMENT_NAMESPACE_MODE", "fake").strip().lower()
|
||||
if namespace_mode == "fake":
|
||||
knowledge_base_id = KNOWLEDGE_BASE_ID
|
||||
elif namespace_mode == "bailian":
|
||||
knowledge_base_id = BAILIAN_KNOWLEDGE_BASE_ID
|
||||
else:
|
||||
sys.stdout.write(
|
||||
json.dumps(
|
||||
{"status": "failed", "error": "document namespace mode is invalid"},
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
raise SystemExit(1)
|
||||
try:
|
||||
result = run_smoke(
|
||||
base_url=base_url,
|
||||
sample_path=sample_path,
|
||||
knowledge_base_id=knowledge_base_id,
|
||||
)
|
||||
except DocumentPipelineSmokeError as exc:
|
||||
sys.stdout.write(json.dumps({"status": "failed", "error": str(exc)}, sort_keys=True) + "\n")
|
||||
raise SystemExit(1) from None
|
||||
sys.stdout.write(json.dumps(result, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Run a reproducible retrieval evaluation on the public synthetic corpus."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.adapters.fake import FakeEmbeddingProvider, FakeReranker
|
||||
from app.core.config import Settings
|
||||
from app.core.demo_identity import ACCESS_SCOPE_ID, KNOWLEDGE_BASE_ID
|
||||
from app.persistence.retrieval import PostgresRetrievalRepository
|
||||
from app.services.evaluation import (
|
||||
RankingMetrics,
|
||||
bootstrap_mean_confidence_interval,
|
||||
evaluate_ranking,
|
||||
freeze_run_config,
|
||||
)
|
||||
from app.services.retrieval import RetrievalActor, RetrievalGrant, RetrievalResult, RetrievalService
|
||||
from app.tools.seed_demo import (
|
||||
DEFAULT_SAMPLE_ROOT,
|
||||
DemoDocument,
|
||||
DemoQuery,
|
||||
load_documents,
|
||||
load_queries,
|
||||
)
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _source_id(source_name: str) -> str:
|
||||
return Path(source_name).stem
|
||||
|
||||
|
||||
def _mean(values: list[float]) -> float:
|
||||
return sum(values) / len(values) if values else 0.0
|
||||
|
||||
|
||||
class DemoRetrievalService(Protocol):
|
||||
async def search(
|
||||
self,
|
||||
*,
|
||||
actor: RetrievalActor,
|
||||
knowledge_base_id: uuid.UUID,
|
||||
query: str,
|
||||
vector_top_k: int,
|
||||
rerank_top_n: int,
|
||||
) -> RetrievalResult: ...
|
||||
|
||||
|
||||
async def evaluate_demo_queries(
|
||||
*,
|
||||
service: DemoRetrievalService,
|
||||
actor: RetrievalActor,
|
||||
documents: list[DemoDocument],
|
||||
queries: list[DemoQuery],
|
||||
vector_top_k: int = 20,
|
||||
rerank_top_n: int = 10,
|
||||
metric_cutoff: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
"""Evaluate answerable queries with a fully judged synthetic corpus pool."""
|
||||
|
||||
corpus_ids = frozenset(document.source_id for document in documents)
|
||||
if len(corpus_ids) != len(documents):
|
||||
raise ValueError("synthetic corpus document IDs must be unique")
|
||||
|
||||
cases: list[dict[str, Any]] = []
|
||||
scored: list[RankingMetrics] = []
|
||||
active_profile_hash: str | None = None
|
||||
for query in queries:
|
||||
result = await service.search(
|
||||
actor=actor,
|
||||
knowledge_base_id=KNOWLEDGE_BASE_ID,
|
||||
query=query.query,
|
||||
vector_top_k=vector_top_k,
|
||||
rerank_top_n=rerank_top_n,
|
||||
)
|
||||
if active_profile_hash is None:
|
||||
active_profile_hash = result.profile.profile_hash
|
||||
elif active_profile_hash != result.profile.profile_hash:
|
||||
raise ValueError("active profile changed during evaluation")
|
||||
|
||||
ranked_ids = [_source_id(hit.source_name) for hit in result.results]
|
||||
case: dict[str, Any] = {
|
||||
"qid": query.qid,
|
||||
"answerable": query.answerable,
|
||||
"ranked_document_ids": ranked_ids,
|
||||
"retrieval_status": result.status,
|
||||
"rerank_status": result.rerank_status,
|
||||
}
|
||||
if query.answerable:
|
||||
relevance = {document_id: 0.0 for document_id in corpus_ids}
|
||||
for expected_id in query.expected_doc_ids:
|
||||
if expected_id not in corpus_ids:
|
||||
raise ValueError("expected document is outside the corpus manifest")
|
||||
relevance[expected_id] = 1.0
|
||||
groups = tuple(frozenset({expected_id}) for expected_id in query.expected_doc_ids)
|
||||
metrics = evaluate_ranking(
|
||||
ranked_ids,
|
||||
relevance=relevance,
|
||||
judged_document_ids=corpus_ids,
|
||||
evidence_groups=groups,
|
||||
k=metric_cutoff,
|
||||
)
|
||||
scored.append(metrics)
|
||||
case["metrics"] = asdict(metrics)
|
||||
else:
|
||||
case["metrics"] = None
|
||||
cases.append(case)
|
||||
|
||||
if active_profile_hash is None:
|
||||
raise ValueError("evaluation query set is empty")
|
||||
hit_values = [metric.hit_at_k for metric in scored]
|
||||
hit_ci = bootstrap_mean_confidence_interval(
|
||||
hit_values,
|
||||
seed=20260713,
|
||||
iterations=2_000,
|
||||
)
|
||||
return {
|
||||
"status": "ok",
|
||||
"dataset": "synthetic-demo",
|
||||
"case_count": len(queries),
|
||||
"answerable_case_count": len(scored),
|
||||
"active_embedding_profile_hash": active_profile_hash,
|
||||
"metrics": {
|
||||
f"hit_at_{metric_cutoff}": _mean(hit_values),
|
||||
"mrr": _mean([metric.reciprocal_rank for metric in scored]),
|
||||
f"ndcg_at_{metric_cutoff}": _mean([metric.ndcg_at_k for metric in scored]),
|
||||
f"complete_hit_at_{metric_cutoff}": _mean(
|
||||
[metric.complete_hit_at_k for metric in scored]
|
||||
),
|
||||
f"evidence_group_recall_at_{metric_cutoff}": _mean(
|
||||
[metric.evidence_group_recall_at_k for metric in scored]
|
||||
),
|
||||
f"hit_at_{metric_cutoff}_confidence_interval": asdict(hit_ci),
|
||||
},
|
||||
"cases": cases,
|
||||
}
|
||||
|
||||
|
||||
async def async_main() -> int:
|
||||
document_path = Path(
|
||||
sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SAMPLE_ROOT / "demo_documents.jsonl"
|
||||
)
|
||||
query_path = Path(
|
||||
sys.argv[2] if len(sys.argv) > 2 else DEFAULT_SAMPLE_ROOT / "demo_queries.jsonl"
|
||||
)
|
||||
try:
|
||||
settings = Settings()
|
||||
documents = load_documents(document_path)
|
||||
queries = load_queries(query_path)
|
||||
service = RetrievalService(
|
||||
repository=PostgresRetrievalRepository(settings),
|
||||
embedding_provider=FakeEmbeddingProvider(settings.embedding_dimension),
|
||||
reranker=FakeReranker(),
|
||||
synthetic_embedding_provider=FakeEmbeddingProvider(settings.embedding_dimension),
|
||||
synthetic_reranker=FakeReranker(),
|
||||
)
|
||||
actor = RetrievalActor(
|
||||
subject="synthetic-evaluation-runner",
|
||||
grants=(
|
||||
RetrievalGrant(
|
||||
knowledge_base_id=KNOWLEDGE_BASE_ID,
|
||||
access_scope_ids=(ACCESS_SCOPE_ID,),
|
||||
),
|
||||
),
|
||||
)
|
||||
artifact = await evaluate_demo_queries(
|
||||
service=service,
|
||||
actor=actor,
|
||||
documents=documents,
|
||||
queries=queries,
|
||||
)
|
||||
config, config_hash = freeze_run_config(
|
||||
{
|
||||
"corpus_sha256": _sha256_file(document_path),
|
||||
"query_set_sha256": _sha256_file(query_path),
|
||||
"embedding_profile_hash": artifact["active_embedding_profile_hash"],
|
||||
"vector_top_k": 20,
|
||||
"rerank_top_n": 10,
|
||||
"metric_cutoff": 3,
|
||||
"bootstrap_seed": 20260713,
|
||||
}
|
||||
)
|
||||
artifact["frozen_config"] = json.loads(config)
|
||||
artifact["frozen_config_sha256"] = config_hash
|
||||
sys.stdout.write(json.dumps(artifact, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
return 0
|
||||
except Exception:
|
||||
# CLI output remains fixed and does not echo paths, document text, DSNs, or errors.
|
||||
sys.stdout.write(
|
||||
json.dumps(
|
||||
{"status": "failed", "error_kind": "evaluation_failed"},
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> None:
|
||||
raise SystemExit(asyncio.run(async_main()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Export the FastAPI contract without opening a database or reading secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from app.main import create_app
|
||||
|
||||
|
||||
def export_schema() -> dict[str, Any]:
|
||||
"""Build the deterministic application schema from import-safe contracts."""
|
||||
|
||||
return create_app().openapi()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sys.stdout.write(
|
||||
json.dumps(
|
||||
export_schema(),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Initialize the persistent upload volume without reading application secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
APP_UID = 10_001
|
||||
APP_GID = 10_001
|
||||
|
||||
|
||||
class UploadStorageInitializationError(RuntimeError):
|
||||
"""A path-neutral initialization failure safe for container logs."""
|
||||
|
||||
|
||||
def initialize_upload_root(
|
||||
root: Path,
|
||||
*,
|
||||
uid: int = APP_UID,
|
||||
gid: int = APP_GID,
|
||||
change_owner: Callable[[Path, int, int], None] | None = None,
|
||||
) -> None:
|
||||
if not root.is_absolute() or uid < 1 or gid < 1:
|
||||
raise UploadStorageInitializationError("invalid upload root contract")
|
||||
try:
|
||||
if root.exists() and root.is_symlink():
|
||||
raise UploadStorageInitializationError("unsafe upload root")
|
||||
root.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise UploadStorageInitializationError("unsafe upload root")
|
||||
owner = change_owner or (
|
||||
lambda path, owner_uid, owner_gid: os.chown(path, owner_uid, owner_gid)
|
||||
)
|
||||
owner(root, uid, gid)
|
||||
root.chmod(0o750, follow_symlinks=False)
|
||||
metadata = root.stat(follow_symlinks=False)
|
||||
if (
|
||||
not stat.S_ISDIR(metadata.st_mode)
|
||||
or metadata.st_uid != uid
|
||||
or metadata.st_gid != gid
|
||||
or stat.S_IMODE(metadata.st_mode) != 0o750
|
||||
):
|
||||
raise UploadStorageInitializationError("upload root ownership verification failed")
|
||||
except UploadStorageInitializationError:
|
||||
raise
|
||||
except OSError:
|
||||
raise UploadStorageInitializationError("upload root initialization failed") from None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
initialize_upload_root(Path(os.getenv("UPLOAD_ROOT", "/data/uploads")))
|
||||
except UploadStorageInitializationError:
|
||||
sys.stdout.write(
|
||||
json.dumps({"status": "failed", "error_kind": "storage_init_failed"}) + "\n"
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
sys.stdout.write(json.dumps({"status": "ok"}) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -25,6 +25,8 @@ from app.adapters.model_gateway import ModelGatewayAdapter
|
||||
from app.core.config import Settings
|
||||
from app.core.demo_identity import (
|
||||
ACCESS_SCOPE_ID,
|
||||
BAILIAN_ACCESS_SCOPE_ID,
|
||||
BAILIAN_KNOWLEDGE_BASE_ID,
|
||||
IDENTITY_NAMESPACE,
|
||||
KNOWLEDGE_BASE_ID,
|
||||
offline_embedding_profile_hash,
|
||||
@@ -75,8 +77,8 @@ OFFLINE_NAMESPACE = DemoNamespace(
|
||||
)
|
||||
BAILIAN_NAMESPACE = DemoNamespace(
|
||||
mode="bailian",
|
||||
knowledge_base_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-knowledge-base"),
|
||||
access_scope_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-public-scope"),
|
||||
knowledge_base_id=BAILIAN_KNOWLEDGE_BASE_ID,
|
||||
access_scope_id=BAILIAN_ACCESS_SCOPE_ID,
|
||||
scope_name="synthetic-bailian-validation",
|
||||
knowledge_base_name="虚构地质 PoC 知识库（百炼验证）",
|
||||
storage_prefix="synthetic/bailian",
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Run a destructive-free PostgreSQL smoke test for job lease fencing.
|
||||
|
||||
The command creates one synthetic queue row, races two claimers, verifies that
|
||||
only one wins, proves that a stale token is rejected, completes the winning
|
||||
lease, and removes the synthetic row before exiting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.persistence.job_queue import (
|
||||
BackgroundJob,
|
||||
JobLease,
|
||||
LeaseLostError,
|
||||
PsycopgJobQueue,
|
||||
)
|
||||
|
||||
|
||||
def _dsn(settings: Settings) -> str:
|
||||
url = settings.database_url().set(drivername="postgresql")
|
||||
return url.render_as_string(hide_password=False)
|
||||
|
||||
|
||||
def _insert_job(dsn: str, job_id: uuid.UUID, capability: str, idempotency_key: str) -> None:
|
||||
with psycopg.connect(dsn, connect_timeout=5) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rag.background_jobs (
|
||||
id, job_type, required_capability, resource_type, resource_id,
|
||||
idempotency_key, payload, stage, max_attempts
|
||||
) VALUES (%s, 'WORKER_SMOKE', %s, 'synthetic_smoke', %s, %s, '{}'::jsonb,
|
||||
'VERIFYING_FENCE', 2)
|
||||
""",
|
||||
(job_id, capability, job_id, idempotency_key),
|
||||
)
|
||||
|
||||
|
||||
def _delete_job(dsn: str, job_id: uuid.UUID) -> None:
|
||||
with psycopg.connect(dsn, connect_timeout=5) as connection:
|
||||
connection.execute("DELETE FROM rag.background_jobs WHERE id = %s", (job_id,))
|
||||
|
||||
|
||||
def _expire_lease(dsn: str, job_id: uuid.UUID) -> None:
|
||||
"""Move only the synthetic smoke lease into the past for recovery verification."""
|
||||
|
||||
with psycopg.connect(dsn, connect_timeout=5) as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE rag.background_jobs
|
||||
SET lease_until = now() - interval '1 second'
|
||||
WHERE id = %s AND status = 'RUNNING'
|
||||
""",
|
||||
(job_id,),
|
||||
)
|
||||
|
||||
|
||||
def _race_claim(
|
||||
queues: tuple[PsycopgJobQueue, PsycopgJobQueue],
|
||||
capability: str,
|
||||
*,
|
||||
worker_prefix: str,
|
||||
) -> tuple[BackgroundJob | None, BackgroundJob | None]:
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
return tuple(
|
||||
executor.map(
|
||||
lambda item: item[0].claim(
|
||||
worker_id=item[1],
|
||||
worker_capabilities=(capability,),
|
||||
lease_seconds=30,
|
||||
),
|
||||
(
|
||||
(queues[0], f"{worker_prefix}-a"),
|
||||
(queues[1], f"{worker_prefix}-b"),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_smoke(settings: Settings) -> dict[str, object]:
|
||||
dsn = _dsn(settings)
|
||||
job_id = uuid.uuid4()
|
||||
nonce = uuid.uuid4().hex
|
||||
capability = f"worker-smoke-{nonce}"
|
||||
idempotency_key = f"worker-smoke:{nonce}"
|
||||
_insert_job(dsn, job_id, capability, idempotency_key)
|
||||
try:
|
||||
queues = (PsycopgJobQueue(dsn), PsycopgJobQueue(dsn))
|
||||
claims = _race_claim(queues, capability, worker_prefix="smoke-worker")
|
||||
winners = tuple(claim for claim in claims if claim is not None)
|
||||
if len(winners) != 1:
|
||||
raise RuntimeError("concurrent claim contract failed")
|
||||
winner = winners[0]
|
||||
stale = JobLease(
|
||||
job_id=winner.lease.job_id,
|
||||
worker_id=winner.lease.worker_id,
|
||||
lease_token=uuid.uuid4(),
|
||||
)
|
||||
try:
|
||||
queues[0].heartbeat(stale, lease_seconds=30)
|
||||
except LeaseLostError:
|
||||
fence_rejected = True
|
||||
else:
|
||||
fence_rejected = False
|
||||
if not fence_rejected:
|
||||
raise RuntimeError("stale lease fence was accepted")
|
||||
queues[0].heartbeat(winner.lease, lease_seconds=30)
|
||||
_expire_lease(dsn, job_id)
|
||||
try:
|
||||
queues[0].heartbeat(winner.lease, lease_seconds=30)
|
||||
except LeaseLostError:
|
||||
expired_lease_rejected = True
|
||||
else:
|
||||
expired_lease_rejected = False
|
||||
if not expired_lease_rejected:
|
||||
raise RuntimeError("expired lease was renewed")
|
||||
recovery_claims = _race_claim(queues, capability, worker_prefix="recovery-worker")
|
||||
recovery_winners = tuple(claim for claim in recovery_claims if claim is not None)
|
||||
if len(recovery_winners) != 1:
|
||||
raise RuntimeError("expired lease recovery contract failed")
|
||||
recovered = recovery_winners[0]
|
||||
if recovered.lease.lease_token == winner.lease.lease_token:
|
||||
raise RuntimeError("recovered lease did not rotate its fence token")
|
||||
terminal = queues[0].complete(recovered.lease)
|
||||
if terminal.status != "SUCCEEDED":
|
||||
raise RuntimeError("winning lease did not complete")
|
||||
return {
|
||||
"status": "ok",
|
||||
"claim_winners": 1,
|
||||
"stale_fence_rejected": True,
|
||||
"expired_lease_rejected": True,
|
||||
"recovery_claim_winners": 1,
|
||||
"recovery_token_rotated": True,
|
||||
"terminal_status": terminal.status,
|
||||
}
|
||||
finally:
|
||||
_delete_job(dsn, job_id)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
result = run_smoke(Settings())
|
||||
exit_code = 0
|
||||
except Exception:
|
||||
result = {"status": "failed", "error_kind": "worker_smoke_failed"}
|
||||
exit_code = 1
|
||||
sys.stdout.write(json.dumps(result, sort_keys=True) + "\n")
|
||||
raise SystemExit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user