Make the first RAG slice executable without risking production data
Some checks failed
verify / verify (push) Has been cancelled
Some checks failed
verify / verify (push) Has been cancelled
The Stage 1 foundation now proves provider contracts with mocks and validates PostgreSQL/pgvector ingestion, approval binding, retrieval, reranking, and idempotency using only synthetic data. Live Bailian validation remains gated on rotating the exposed key. Constraint: The key shown in chat is compromised and cannot be used or committed Rejected: Mark Stage 1 complete from mock and offline results | real three-model smoke is still required Confidence: high Scope-risk: moderate Reversibility: clean Directive: Do not enable real-data ingestion until Stage 3 cloud approval and outbound manifest controls are enforced end to end Tested: make verify; 41 pytest tests; strict mypy; Ruff; Compose config; pinned image build; empty-volume migration; role denial; two idempotent 20-vector seeds; database restart persistence Not-tested: Live Bailian calls require a newly rotated key; React product UI is not implemented
This commit is contained in:
@@ -1 +0,0 @@
|
||||
|
||||
175
backend/tests/contract/test_bailian_chat.py
Normal file
175
backend/tests/contract/test_bailian_chat.py
Normal file
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.adapters.bailian.chat import BailianChatAdapter
|
||||
from app.ports.model_providers import ChatMessage, ModelProviderError
|
||||
|
||||
BASE_URL = "https://workspace.cn-beijing.maas.aliyuncs.com/compatible-mode/v1"
|
||||
API_KEY = "sk-test-chat-secret"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_disables_thinking_and_search() -> None:
|
||||
messages = [ChatMessage(role="user", content="Give a grounded answer")]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/compatible-mode/v1/chat/completions"
|
||||
assert request.headers["Authorization"] == f"Bearer {API_KEY}"
|
||||
payload = json.loads(request.content)
|
||||
assert payload["model"] == "deepseek-v4-flash"
|
||||
assert payload["messages"] == [{"role": "user", "content": "Give a grounded answer"}]
|
||||
assert payload["enable_thinking"] is False
|
||||
assert payload["enable_search"] is False
|
||||
assert payload["stream"] is False
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "req_chat_1",
|
||||
"model": "deepseek-v4-flash",
|
||||
"choices": [
|
||||
{
|
||||
"message": {"role": "assistant", "content": "grounded"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 6,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianChatAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
http_client=client,
|
||||
)
|
||||
result = await adapter.complete(messages, max_tokens=32)
|
||||
|
||||
assert result.content == "grounded"
|
||||
assert result.finish_reason == "stop"
|
||||
assert result.usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_disables_thinking_and_search_and_parses_sse() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
payload = json.loads(request.content)
|
||||
assert payload["enable_thinking"] is False
|
||||
assert payload["enable_search"] is False
|
||||
assert payload["stream"] is True
|
||||
assert payload["stream_options"] == {"include_usage": True}
|
||||
chunks = (
|
||||
'data: {"id":"req_stream","model":"deepseek-v4-flash",'
|
||||
'"choices":[{"delta":{"content":"矿"},"finish_reason":null}]}\n\n'
|
||||
'data: {"id":"req_stream","model":"deepseek-v4-flash",'
|
||||
'"choices":[{"delta":{"content":"体"},"finish_reason":"stop"}]}\n\n'
|
||||
'data: {"id":"req_stream","model":"deepseek-v4-flash",'
|
||||
'"choices":[],"usage":{"prompt_tokens":4,"completion_tokens":2,'
|
||||
'"total_tokens":6}}\n\n'
|
||||
"data: [DONE]\n\n"
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content=chunks.encode(),
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianChatAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
http_client=client,
|
||||
)
|
||||
events = [
|
||||
event
|
||||
async for event in adapter.stream(
|
||||
[ChatMessage(role="user", content="question")],
|
||||
max_tokens=32,
|
||||
)
|
||||
]
|
||||
|
||||
assert "".join(event.delta for event in events) == "矿体"
|
||||
assert events[1].finish_reason == "stop"
|
||||
assert events[2].usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_error_does_not_expose_key_or_message_body() -> None:
|
||||
sensitive_text = "CONFIDENTIAL_PROMPT_BODY"
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
401,
|
||||
json={
|
||||
"error": {
|
||||
"code": API_KEY,
|
||||
"message": f"{sensitive_text}: {API_KEY}",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianChatAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
http_client=client,
|
||||
)
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.complete(
|
||||
[ChatMessage(role="user", content=sensitive_text)],
|
||||
max_tokens=8,
|
||||
)
|
||||
|
||||
rendered = f"{exc_info.value!s} {exc_info.value!r}"
|
||||
assert API_KEY not in rendered
|
||||
assert sensitive_text not in rendered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_honors_retry_after_before_any_token() -> None:
|
||||
calls = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
return httpx.Response(
|
||||
429,
|
||||
headers={"Retry-After": "0"},
|
||||
json={"error": {"code": "rate_limited"}},
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content=(
|
||||
'data: {"model":"deepseek-v4-flash",'
|
||||
'"choices":[{"delta":{"content":"正常"},"finish_reason":"stop"}]}\n\n'
|
||||
"data: [DONE]\n\n"
|
||||
).encode(),
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianChatAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
http_client=client,
|
||||
max_retries=1,
|
||||
retry_base_seconds=0,
|
||||
)
|
||||
events = [
|
||||
event
|
||||
async for event in adapter.stream(
|
||||
[ChatMessage(role="user", content="question")],
|
||||
max_tokens=8,
|
||||
)
|
||||
]
|
||||
|
||||
assert "".join(event.delta for event in events) == "正常"
|
||||
assert calls == 2
|
||||
208
backend/tests/contract/test_bailian_embedding.py
Normal file
208
backend/tests/contract/test_bailian_embedding.py
Normal file
@@ -0,0 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.adapters.bailian._base import response_model
|
||||
from app.adapters.bailian.embedding import BailianEmbeddingAdapter
|
||||
from app.ports.model_providers import ModelProviderError, ProviderErrorKind
|
||||
|
||||
BASE_URL = "https://workspace.cn-beijing.maas.aliyuncs.com/compatible-mode/v1"
|
||||
API_KEY = "sk-test-embedding-secret"
|
||||
|
||||
|
||||
def _vector(marker: float = 1.0) -> list[float]:
|
||||
return [marker, *([0.0] * 1_023)]
|
||||
|
||||
|
||||
def test_embedding_rejects_non_bailian_endpoint_before_request() -> None:
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url="https://attacker.invalid/compatible-mode/v1",
|
||||
)
|
||||
|
||||
assert exc_info.value.kind is ProviderErrorKind.INVALID_REQUEST
|
||||
assert exc_info.value.provider_code == "invalid_base_url"
|
||||
|
||||
|
||||
def test_response_model_falls_back_when_provider_echoes_sensitive_value() -> None:
|
||||
assert (
|
||||
response_model(
|
||||
{"model": API_KEY},
|
||||
"text-embedding-v4",
|
||||
sensitive_values=(API_KEY,),
|
||||
)
|
||||
== "text-embedding-v4"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_uses_compatible_endpoint_and_restores_index_order() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/compatible-mode/v1/embeddings"
|
||||
assert request.headers["Authorization"] == f"Bearer {API_KEY}"
|
||||
payload = json.loads(request.content)
|
||||
assert payload == {
|
||||
"model": "text-embedding-v4",
|
||||
"input": ["first", "second"],
|
||||
"dimensions": 1_024,
|
||||
"encoding_format": "float",
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "req_embedding_1",
|
||||
"model": "text-embedding-v4",
|
||||
"data": [
|
||||
{"index": 1, "embedding": _vector(2.0)},
|
||||
{"index": 0, "embedding": _vector(1.0)},
|
||||
],
|
||||
"usage": {"prompt_tokens": 2, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
result = await adapter.embed_documents(["first", "second"])
|
||||
|
||||
assert len(result.vectors) == 2
|
||||
assert all(len(vector) == 1_024 for vector in result.vectors)
|
||||
assert result.vectors[0][0] == 1.0
|
||||
assert result.vectors[1][0] == 2.0
|
||||
assert result.request_id == "req_embedding_1"
|
||||
assert result.usage.input_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_enforces_10_8192_and_33000_token_boundaries() -> None:
|
||||
calls = 0
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
payload = json.loads(request.content)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [
|
||||
{"index": index, "embedding": _vector()}
|
||||
for index, _ in enumerate(payload["input"])
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
|
||||
await adapter.embed_documents(["x"] * 10)
|
||||
await adapter.embed_documents(["x" * 8_192])
|
||||
await adapter.embed_documents([*(["x" * 8_192] * 4), "x" * 232])
|
||||
assert calls == 3
|
||||
|
||||
invalid_batches = (
|
||||
["x"] * 11,
|
||||
["x" * 8_193],
|
||||
[*(["x" * 8_192] * 4), "x" * 233],
|
||||
)
|
||||
for texts in invalid_batches:
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.embed_documents(texts)
|
||||
assert exc_info.value.kind is ProviderErrorKind.INVALID_REQUEST
|
||||
assert calls == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_rejects_non_1024_dimensional_response() -> None:
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": [{"index": 0, "embedding": [1.0] * 1_023}]},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.embed_query("sensitive geological paragraph")
|
||||
|
||||
assert exc_info.value.kind is ProviderErrorKind.INVALID_RESPONSE
|
||||
assert exc_info.value.provider_code == "invalid_embedding_dimensions"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_provider_error_does_not_expose_key_or_body() -> None:
|
||||
sensitive_text = "CONFIDENTIAL_GEOLOGICAL_BODY"
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
400,
|
||||
json={
|
||||
"id": sensitive_text,
|
||||
"error": {
|
||||
"code": API_KEY,
|
||||
"message": f"failed for {sensitive_text} using {API_KEY}",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.embed_query(sensitive_text)
|
||||
|
||||
rendered = f"{exc_info.value!s} {exc_info.value!r}"
|
||||
assert API_KEY not in rendered
|
||||
assert sensitive_text not in rendered
|
||||
assert exc_info.value.provider_code is None
|
||||
assert exc_info.value.request_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_retries_timeout_with_a_fixed_bound() -> None:
|
||||
calls = 0
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise httpx.ReadTimeout("simulated timeout", request=request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"data": [{"index": 0, "embedding": _vector()}]},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
max_retries=1,
|
||||
retry_base_seconds=0,
|
||||
)
|
||||
result = await adapter.embed_query("query")
|
||||
|
||||
assert len(result.vectors[0]) == 1_024
|
||||
assert calls == 2
|
||||
149
backend/tests/contract/test_bailian_rerank.py
Normal file
149
backend/tests/contract/test_bailian_rerank.py
Normal file
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.adapters.bailian.rerank import BailianRerankerAdapter
|
||||
from app.ports.model_providers import ModelProviderError, ProviderErrorKind
|
||||
|
||||
BASE_URL = "https://workspace.cn-beijing.maas.aliyuncs.com/compatible-api/v1"
|
||||
API_KEY = "sk-test-rerank-secret"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_uses_separate_endpoint_and_maps_result_indices() -> None:
|
||||
documents = ["copper evidence", "gold evidence", "irrelevant"]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/compatible-api/v1/reranks"
|
||||
assert request.headers["Authorization"] == f"Bearer {API_KEY}"
|
||||
payload = json.loads(request.content)
|
||||
assert payload == {
|
||||
"model": "qwen3-rerank",
|
||||
"query": "where is gold",
|
||||
"documents": documents,
|
||||
"top_n": 2,
|
||||
"instruct": "rank geological evidence",
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "req_rerank_1",
|
||||
"model": "qwen3-rerank",
|
||||
"results": [
|
||||
{"index": 1, "relevance_score": 0.91},
|
||||
{"index": 0, "relevance_score": 0.72},
|
||||
],
|
||||
"usage": {"input_tokens": 42, "total_tokens": 42},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianRerankerAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
result = await adapter.rerank(
|
||||
"where is gold",
|
||||
documents,
|
||||
top_n=2,
|
||||
instruct="rank geological evidence",
|
||||
)
|
||||
|
||||
assert [item.index for item in result.items] == [1, 0]
|
||||
assert [item.document for item in result.items] == [documents[1], documents[0]]
|
||||
assert [item.relevance_score for item in result.items] == [0.91, 0.72]
|
||||
assert result.request_id == "req_rerank_1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_enforces_500_4000_and_repeated_query_120000_formula() -> None:
|
||||
calls = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"results": [{"index": 0, "relevance_score": 1.0}]},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianRerankerAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
|
||||
await adapter.rerank("q" * 4_000, ["d" * 4_000], top_n=1)
|
||||
exactly_120000 = ["d" * 40] * 500
|
||||
await adapter.rerank("q" * 200, exactly_120000, top_n=1)
|
||||
assert calls == 2
|
||||
|
||||
invalid_requests = (
|
||||
("q" * 4_001, ["d"]),
|
||||
("q", ["d" * 4_001]),
|
||||
("q", ["d"] * 501),
|
||||
("q" * 200, [*(["d" * 40] * 499), "d" * 41]),
|
||||
)
|
||||
for query, documents in invalid_requests:
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.rerank(query, documents, top_n=1)
|
||||
assert exc_info.value.kind is ProviderErrorKind.INVALID_REQUEST
|
||||
assert calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_rejects_invalid_provider_index() -> None:
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"results": [{"index": 2, "relevance_score": 0.9}]},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianRerankerAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
)
|
||||
with pytest.raises(ModelProviderError) as exc_info:
|
||||
await adapter.rerank("query", ["only document"], top_n=1)
|
||||
|
||||
assert exc_info.value.kind is ProviderErrorKind.INVALID_RESPONSE
|
||||
assert exc_info.value.provider_code == "invalid_rerank_index"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerank_retries_5xx_with_a_fixed_bound() -> None:
|
||||
calls = 0
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
return httpx.Response(503, json={"error": {"code": "temporarily_unavailable"}})
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"results": [{"index": 0, "relevance_score": 0.9}]},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
|
||||
adapter = BailianRerankerAdapter(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
token_counter=len,
|
||||
http_client=client,
|
||||
max_retries=1,
|
||||
retry_base_seconds=0,
|
||||
)
|
||||
result = await adapter.rerank("query", ["document"], top_n=1)
|
||||
|
||||
assert result.items[0].index == 0
|
||||
assert calls == 2
|
||||
Reference in New Issue
Block a user