Isolate cloud model access before enabling product RAG workflows
Some checks failed
verify / verify (push) Has been cancelled
Some checks failed
verify / verify (push) Has been cancelled
The API and ingestion tools now use a fixed internal model gateway while governed profiles, embedding cache assignments, traceable citations, and stable API errors establish the boundaries required by later workflows. Constraint: The current Alibaba Cloud workspace rejects all three live model calls with authentication failures Rejected: Give the API or seed tools the Bailian key and direct egress | combines database access, cloud credentials, and public network access Rejected: Mix offline and Bailian vectors in one demo namespace | makes profile activation and retrieval ambiguous Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep Bailian credentials and egress exclusive to model-gateway and create a new immutable profile hash for any embedding identity change Tested: make verify; 121 backend tests; 14 frontend tests; fresh and populated Alembic upgrade-downgrade-upgrade; two idempotent offline seeds; Docker health and HTTP retrieval; isolated provider smoke Not-tested: Successful live Bailian responses because the supplied workspace credential currently fails authentication
This commit is contained in:
@@ -9,11 +9,7 @@ from collections.abc import Awaitable, Callable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.adapters.bailian import (
|
||||
BailianChatAdapter,
|
||||
BailianEmbeddingAdapter,
|
||||
BailianRerankerAdapter,
|
||||
)
|
||||
from app.adapters.model_gateway import ModelGatewayAdapter
|
||||
from app.core.config import Settings
|
||||
from app.core.secrets import SecretFileError
|
||||
from app.ports.model_providers import ChatMessage, ModelProviderError
|
||||
@@ -30,89 +26,59 @@ class ProbeResult:
|
||||
status_code: int | None = None
|
||||
|
||||
|
||||
async def probe_embedding(settings: Settings, api_key: str) -> ProbeResult:
|
||||
adapter = BailianEmbeddingAdapter(
|
||||
api_key=api_key,
|
||||
base_url=settings.bailian_openai_base_url,
|
||||
model=settings.embedding_model,
|
||||
dimensions=settings.embedding_dimension,
|
||||
timeout_seconds=settings.model_timeout_seconds,
|
||||
max_retries=settings.model_max_retries,
|
||||
async def probe_embedding(settings: Settings, adapter: ModelGatewayAdapter) -> ProbeResult:
|
||||
# API identity probes query embedding. Document embedding remains worker-only.
|
||||
result = await adapter.embed_query("用于能力探测的虚构地质问题。")
|
||||
if len(result.vectors) != 1 or len(result.vectors[0]) != settings.embedding_dimension:
|
||||
raise RuntimeError("embedding contract mismatch")
|
||||
return ProbeResult(
|
||||
capability="embedding",
|
||||
status="ok",
|
||||
model=result.model,
|
||||
elapsed_ms=round(result.elapsed_ms, 2),
|
||||
request_id=result.request_id,
|
||||
)
|
||||
try:
|
||||
result = await adapter.embed_documents(["用于能力探测的虚构地质文本。"])
|
||||
if len(result.vectors) != 1 or len(result.vectors[0]) != settings.embedding_dimension:
|
||||
raise RuntimeError("embedding contract mismatch")
|
||||
return ProbeResult(
|
||||
capability="embedding",
|
||||
status="ok",
|
||||
model=result.model,
|
||||
elapsed_ms=round(result.elapsed_ms, 2),
|
||||
request_id=result.request_id,
|
||||
)
|
||||
finally:
|
||||
await adapter.aclose()
|
||||
|
||||
|
||||
async def probe_rerank(settings: Settings, api_key: str) -> ProbeResult:
|
||||
adapter = BailianRerankerAdapter(
|
||||
api_key=api_key,
|
||||
base_url=settings.bailian_rerank_base_url,
|
||||
model=settings.rerank_model,
|
||||
timeout_seconds=settings.model_timeout_seconds,
|
||||
max_retries=settings.model_max_retries,
|
||||
async def probe_rerank(_: Settings, adapter: ModelGatewayAdapter) -> ProbeResult:
|
||||
result = await adapter.rerank(
|
||||
"哪段文本提到了斑岩铜矿?",
|
||||
["虚构斑岩铜矿具有钾化带。", "虚构煤层采用测井曲线对比。"],
|
||||
top_n=1,
|
||||
)
|
||||
try:
|
||||
result = await adapter.rerank(
|
||||
"哪段文本提到了斑岩铜矿?",
|
||||
["虚构斑岩铜矿具有钾化带。", "虚构煤层采用测井曲线对比。"],
|
||||
top_n=1,
|
||||
)
|
||||
if len(result.items) != 1 or result.items[0].index not in (0, 1):
|
||||
raise RuntimeError("rerank contract mismatch")
|
||||
return ProbeResult(
|
||||
capability="rerank",
|
||||
status="ok",
|
||||
model=result.model,
|
||||
elapsed_ms=round(result.elapsed_ms, 2),
|
||||
request_id=result.request_id,
|
||||
)
|
||||
finally:
|
||||
await adapter.aclose()
|
||||
|
||||
|
||||
async def probe_chat(settings: Settings, api_key: str) -> ProbeResult:
|
||||
adapter = BailianChatAdapter(
|
||||
api_key=api_key,
|
||||
base_url=settings.bailian_openai_base_url,
|
||||
model=settings.llm_model,
|
||||
timeout_seconds=settings.model_timeout_seconds,
|
||||
max_retries=settings.model_max_retries,
|
||||
if len(result.items) != 1 or result.items[0].index not in (0, 1):
|
||||
raise RuntimeError("rerank contract mismatch")
|
||||
return ProbeResult(
|
||||
capability="rerank",
|
||||
status="ok",
|
||||
model=result.model,
|
||||
elapsed_ms=round(result.elapsed_ms, 2),
|
||||
request_id=result.request_id,
|
||||
)
|
||||
|
||||
|
||||
async def probe_chat(_: Settings, adapter: ModelGatewayAdapter) -> ProbeResult:
|
||||
model: str | None = None
|
||||
request_id: str | None = None
|
||||
elapsed_ms = 0.0
|
||||
content_seen = False
|
||||
try:
|
||||
async for event in adapter.stream(
|
||||
[ChatMessage(role="user", content="只回复:能力正常")],
|
||||
max_tokens=16,
|
||||
):
|
||||
model = event.model
|
||||
request_id = event.request_id or request_id
|
||||
elapsed_ms = max(elapsed_ms, event.elapsed_ms)
|
||||
content_seen = content_seen or bool(event.delta)
|
||||
if not content_seen:
|
||||
raise RuntimeError("chat stream contained no text")
|
||||
return ProbeResult(
|
||||
capability="chat",
|
||||
status="ok",
|
||||
model=model,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
request_id=request_id,
|
||||
)
|
||||
finally:
|
||||
await adapter.aclose()
|
||||
async for event in adapter.stream(
|
||||
[ChatMessage(role="user", content="只回复:能力正常")],
|
||||
max_tokens=16,
|
||||
):
|
||||
model = event.model
|
||||
request_id = event.request_id or request_id
|
||||
elapsed_ms = max(elapsed_ms, event.elapsed_ms)
|
||||
content_seen = content_seen or bool(event.delta)
|
||||
if not content_seen:
|
||||
raise RuntimeError("chat stream contained no text")
|
||||
return ProbeResult(
|
||||
capability="chat",
|
||||
status="ok",
|
||||
model=model,
|
||||
elapsed_ms=round(elapsed_ms, 2),
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
|
||||
def failed_probe(capability: str, error: BaseException) -> ProbeResult:
|
||||
@@ -133,12 +99,12 @@ def failed_probe(capability: str, error: BaseException) -> ProbeResult:
|
||||
|
||||
async def run_probe(
|
||||
capability: str,
|
||||
operation: Callable[[Settings, str], Awaitable[ProbeResult]],
|
||||
operation: Callable[[Settings, ModelGatewayAdapter], Awaitable[ProbeResult]],
|
||||
settings: Settings,
|
||||
api_key: str,
|
||||
adapter: ModelGatewayAdapter,
|
||||
) -> ProbeResult:
|
||||
try:
|
||||
return await operation(settings, api_key)
|
||||
return await operation(settings, adapter)
|
||||
except Exception as exc: # The output is deliberately reduced to a safe category.
|
||||
return failed_probe(capability, exc)
|
||||
|
||||
@@ -148,17 +114,10 @@ def write_json_line(payload: dict[str, Any]) -> None:
|
||||
|
||||
|
||||
async def async_main() -> int:
|
||||
adapter: ModelGatewayAdapter | None = None
|
||||
try:
|
||||
settings = Settings()
|
||||
if any(
|
||||
"<workspace-id>" in url
|
||||
for url in (
|
||||
settings.bailian_openai_base_url,
|
||||
settings.bailian_rerank_base_url,
|
||||
)
|
||||
):
|
||||
raise ValueError("workspace endpoint placeholders are not runnable")
|
||||
api_key = settings.bailian_api_key()
|
||||
adapter = ModelGatewayAdapter.from_settings(settings)
|
||||
except (SecretFileError, ValueError):
|
||||
write_json_line(
|
||||
{
|
||||
@@ -174,12 +133,15 @@ async def async_main() -> int:
|
||||
("rerank", probe_rerank),
|
||||
("chat", probe_chat),
|
||||
)
|
||||
results = []
|
||||
for capability, operation in probes:
|
||||
result = await run_probe(capability, operation, settings, api_key)
|
||||
results.append(result)
|
||||
write_json_line(asdict(result))
|
||||
return 0 if all(result.status == "ok" for result in results) else 1
|
||||
try:
|
||||
results = []
|
||||
for capability, operation in probes:
|
||||
result = await run_probe(capability, operation, settings, adapter)
|
||||
results.append(result)
|
||||
write_json_line(asdict(result))
|
||||
return 0 if all(result.status == "ok" for result in results) else 1
|
||||
finally:
|
||||
await adapter.aclose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -20,8 +20,8 @@ from pgvector.psycopg import register_vector
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from app.adapters.bailian import BailianEmbeddingAdapter, BailianRerankerAdapter
|
||||
from app.adapters.fake import FakeEmbeddingProvider, FakeReranker, lexical_features
|
||||
from app.adapters.model_gateway import ModelGatewayAdapter
|
||||
from app.core.config import Settings
|
||||
from app.core.demo_identity import (
|
||||
ACCESS_SCOPE_ID,
|
||||
@@ -55,6 +55,42 @@ class DemoQuery:
|
||||
answerable: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DemoNamespace:
|
||||
mode: str
|
||||
knowledge_base_id: uuid.UUID
|
||||
access_scope_id: uuid.UUID
|
||||
scope_name: str
|
||||
knowledge_base_name: str
|
||||
storage_prefix: str
|
||||
|
||||
|
||||
OFFLINE_NAMESPACE = DemoNamespace(
|
||||
mode="fake",
|
||||
knowledge_base_id=KNOWLEDGE_BASE_ID,
|
||||
access_scope_id=ACCESS_SCOPE_ID,
|
||||
scope_name="synthetic-demo",
|
||||
knowledge_base_name="虚构地质 PoC 知识库(离线)",
|
||||
storage_prefix="synthetic/offline",
|
||||
)
|
||||
BAILIAN_NAMESPACE = DemoNamespace(
|
||||
mode="bailian",
|
||||
knowledge_base_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-knowledge-base"),
|
||||
access_scope_id=uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-bailian-public-scope"),
|
||||
scope_name="synthetic-bailian-validation",
|
||||
knowledge_base_name="虚构地质 PoC 知识库(百炼验证)",
|
||||
storage_prefix="synthetic/bailian",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EmbeddedVector:
|
||||
vector: tuple[float, ...]
|
||||
request_id: str | None
|
||||
usage: dict[str, int | None]
|
||||
elapsed_ms: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PreparedChunk:
|
||||
source_id: str
|
||||
@@ -71,6 +107,9 @@ class PreparedChunk:
|
||||
embedding_profile_hash: str
|
||||
vector: tuple[float, ...]
|
||||
embedding_model: str
|
||||
provider_request_id: str | None
|
||||
embedding_usage: dict[str, int | None]
|
||||
embedding_elapsed_ms: int
|
||||
title: str
|
||||
region: str
|
||||
mineral: str
|
||||
@@ -88,6 +127,14 @@ def sha256_text(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def namespace_for_mode(mode: str) -> DemoNamespace:
|
||||
if mode == "fake":
|
||||
return OFFLINE_NAMESPACE
|
||||
if mode == "bailian":
|
||||
return BAILIAN_NAMESPACE
|
||||
raise SeedContractError("invalid_provider_mode")
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.is_file():
|
||||
raise SeedContractError("fixture_missing")
|
||||
@@ -142,8 +189,10 @@ def load_queries(path: Path) -> list[DemoQuery]:
|
||||
|
||||
|
||||
def embedding_profile_hash(settings: Settings, mode: str) -> str:
|
||||
if mode != "bailian":
|
||||
if mode == "fake":
|
||||
return offline_embedding_profile_hash(settings.embedding_dimension)
|
||||
if mode != "bailian":
|
||||
raise SeedContractError("invalid_provider_mode")
|
||||
|
||||
endpoint_identity = sha256_text(urlsplit(settings.bailian_openai_base_url).hostname or "")
|
||||
model = settings.embedding_model
|
||||
@@ -164,15 +213,30 @@ def embedding_profile_hash(settings: Settings, mode: str) -> str:
|
||||
async def embed_in_batches(
|
||||
provider: EmbeddingProvider,
|
||||
texts: Sequence[str],
|
||||
) -> tuple[tuple[tuple[float, ...], ...], str]:
|
||||
vectors: list[tuple[float, ...]] = []
|
||||
) -> tuple[tuple[EmbeddedVector, ...], str]:
|
||||
vectors: list[EmbeddedVector] = []
|
||||
resolved_model: str | None = None
|
||||
for offset in range(0, len(texts), 10):
|
||||
result = await provider.embed_documents(texts[offset : offset + 10])
|
||||
if resolved_model is not None and result.model != resolved_model:
|
||||
raise SeedContractError("embedding_model_changed_between_batches")
|
||||
resolved_model = result.model
|
||||
vectors.extend(result.vectors)
|
||||
if len(result.vectors) != len(texts[offset : offset + 10]):
|
||||
raise SeedContractError("embedding_batch_count_mismatch")
|
||||
usage = {
|
||||
"input_tokens": result.usage.input_tokens,
|
||||
"output_tokens": result.usage.output_tokens,
|
||||
"total_tokens": result.usage.total_tokens,
|
||||
}
|
||||
vectors.extend(
|
||||
EmbeddedVector(
|
||||
vector=vector,
|
||||
request_id=result.request_id,
|
||||
usage=usage,
|
||||
elapsed_ms=max(0, round(result.elapsed_ms)),
|
||||
)
|
||||
for vector in result.vectors
|
||||
)
|
||||
if len(vectors) != len(texts) or resolved_model is None:
|
||||
raise SeedContractError("embedding_result_count_mismatch")
|
||||
return tuple(vectors), resolved_model
|
||||
@@ -180,10 +244,11 @@ async def embed_in_batches(
|
||||
|
||||
def prepare_chunks(
|
||||
documents: Sequence[DemoDocument],
|
||||
vectors: Sequence[tuple[float, ...]],
|
||||
vectors: Sequence[EmbeddedVector],
|
||||
*,
|
||||
profile_hash: str,
|
||||
embedding_model: str,
|
||||
namespace: DemoNamespace = OFFLINE_NAMESPACE,
|
||||
) -> list[PreparedChunk]:
|
||||
prepared = []
|
||||
for document, vector in zip(documents, vectors, strict=True):
|
||||
@@ -200,7 +265,12 @@ def prepare_chunks(
|
||||
separators=(",", ":"),
|
||||
)
|
||||
raw_hash = sha256_text(raw_payload)
|
||||
document_id = uuid.uuid5(IDENTITY_NAMESPACE, f"document:{document.source_id}")
|
||||
document_identity = (
|
||||
f"document:{document.source_id}"
|
||||
if namespace.mode == "fake"
|
||||
else f"document:{namespace.mode}:{document.source_id}"
|
||||
)
|
||||
document_id = uuid.uuid5(IDENTITY_NAMESPACE, document_identity)
|
||||
version_id = uuid.uuid5(
|
||||
IDENTITY_NAMESPACE,
|
||||
f"version:{document.source_id}:{raw_hash}:{profile_hash}",
|
||||
@@ -237,8 +307,11 @@ def prepare_chunks(
|
||||
embedding_text_sha256=embedding_hash,
|
||||
outbound_manifest_sha256=sha256_text(manifest_payload),
|
||||
embedding_profile_hash=profile_hash,
|
||||
vector=vector,
|
||||
vector=vector.vector,
|
||||
embedding_model=embedding_model,
|
||||
provider_request_id=vector.request_id,
|
||||
embedding_usage=vector.usage,
|
||||
embedding_elapsed_ms=vector.elapsed_ms,
|
||||
title=document.title,
|
||||
region=document.region,
|
||||
mineral=document.mineral,
|
||||
@@ -255,20 +328,96 @@ def database_dsn(settings: Settings) -> str:
|
||||
)
|
||||
|
||||
|
||||
def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[str, int]:
|
||||
def write_chunks(
|
||||
settings: Settings,
|
||||
chunks: Sequence[PreparedChunk],
|
||||
*,
|
||||
namespace: DemoNamespace,
|
||||
) -> dict[str, int]:
|
||||
if not chunks:
|
||||
raise SeedContractError("chunks_empty")
|
||||
profile_hashes = {item.embedding_profile_hash for item in chunks}
|
||||
resolved_models = {item.embedding_model for item in chunks}
|
||||
if len(profile_hashes) != 1 or len(resolved_models) != 1:
|
||||
raise SeedContractError("mixed_embedding_profiles")
|
||||
profile_hash = next(iter(profile_hashes))
|
||||
resolved_model = next(iter(resolved_models))
|
||||
if namespace.mode == "fake":
|
||||
provider = "local-synthetic"
|
||||
api_mode = "deterministic-offline"
|
||||
endpoint_identity_hash = sha256_text("local-fake")
|
||||
else:
|
||||
provider = "aliyun-bailian"
|
||||
api_mode = "model-gateway/openai-compatible"
|
||||
endpoint_identity_hash = sha256_text(
|
||||
urlsplit(settings.bailian_openai_base_url).hostname or ""
|
||||
)
|
||||
|
||||
with psycopg.connect(database_dsn(settings), row_factory=dict_row) as connection:
|
||||
register_vector(connection)
|
||||
connection.execute("SELECT pg_advisory_xact_lock(724202607120001)")
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rag.knowledge_bases (id, name, description)
|
||||
VALUES (%s, %s, %s)
|
||||
INSERT INTO rag.model_profiles (
|
||||
profile_hash, alias, kind, provider, model, api_mode, dimension,
|
||||
endpoint_identity_hash, config_snapshot, synthetic, enabled
|
||||
) VALUES (
|
||||
%s, %s, 'embedding', %s, %s, %s, 1024, %s, %s, %s, true
|
||||
)
|
||||
ON CONFLICT (profile_hash) DO NOTHING
|
||||
""",
|
||||
(
|
||||
profile_hash,
|
||||
f"{namespace.mode}-embedding-{profile_hash[:12]}",
|
||||
provider,
|
||||
resolved_model,
|
||||
api_mode,
|
||||
endpoint_identity_hash,
|
||||
Jsonb(
|
||||
{
|
||||
"dimension": settings.embedding_dimension,
|
||||
"requested_model": settings.embedding_model,
|
||||
"source": "synthetic-seed-v1",
|
||||
}
|
||||
),
|
||||
namespace.mode == "fake",
|
||||
),
|
||||
)
|
||||
registered_profile = connection.execute(
|
||||
"""
|
||||
SELECT kind, provider, model, api_mode, dimension, endpoint_identity_hash
|
||||
FROM rag.model_profiles
|
||||
WHERE profile_hash = %s
|
||||
""",
|
||||
(profile_hash,),
|
||||
).fetchone()
|
||||
if registered_profile is None or (
|
||||
registered_profile["kind"] != "embedding"
|
||||
or registered_profile["provider"] != provider
|
||||
or registered_profile["model"] != resolved_model
|
||||
or registered_profile["api_mode"] != api_mode
|
||||
or registered_profile["dimension"] != settings.embedding_dimension
|
||||
or registered_profile["endpoint_identity_hash"] != endpoint_identity_hash
|
||||
):
|
||||
raise SeedContractError("embedding_profile_collision")
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rag.knowledge_bases (
|
||||
id, name, description, active_embedding_profile_hash
|
||||
)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
SET name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
active_embedding_profile_hash = EXCLUDED.active_embedding_profile_hash,
|
||||
updated_at = now()
|
||||
""",
|
||||
(KNOWLEDGE_BASE_ID, "虚构地质 PoC 知识库", "仅含公开的合成验证文本"),
|
||||
(
|
||||
namespace.knowledge_base_id,
|
||||
namespace.knowledge_base_name,
|
||||
"仅含公开的合成验证文本",
|
||||
profile_hash,
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
@@ -276,7 +425,11 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
VALUES (%s, %s, %s)
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
""",
|
||||
(ACCESS_SCOPE_ID, KNOWLEDGE_BASE_ID, "synthetic-demo"),
|
||||
(
|
||||
namespace.access_scope_id,
|
||||
namespace.knowledge_base_id,
|
||||
namespace.scope_name,
|
||||
),
|
||||
)
|
||||
|
||||
for item in chunks:
|
||||
@@ -295,11 +448,11 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
""",
|
||||
(
|
||||
item.document_id,
|
||||
KNOWLEDGE_BASE_ID,
|
||||
ACCESS_SCOPE_ID,
|
||||
namespace.knowledge_base_id,
|
||||
namespace.access_scope_id,
|
||||
item.raw_sha256,
|
||||
f"{item.source_id}.json",
|
||||
f"synthetic/{item.source_id}",
|
||||
f"{namespace.storage_prefix}/{item.source_id}",
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
@@ -384,10 +537,10 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
""",
|
||||
(
|
||||
item.chunk_id,
|
||||
KNOWLEDGE_BASE_ID,
|
||||
namespace.knowledge_base_id,
|
||||
item.document_id,
|
||||
item.version_id,
|
||||
ACCESS_SCOPE_ID,
|
||||
namespace.access_scope_id,
|
||||
item.cloud_text,
|
||||
item.cloud_text,
|
||||
item.cloud_text_sha256,
|
||||
@@ -425,6 +578,41 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
""",
|
||||
(item.version_id,),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rag.embedding_cache (
|
||||
profile_hash, embedding_text_sha256, embedding, resolved_model,
|
||||
provider_request_id, usage, elapsed_ms
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (profile_hash, embedding_text_sha256) DO NOTHING
|
||||
""",
|
||||
(
|
||||
item.embedding_profile_hash,
|
||||
item.embedding_text_sha256,
|
||||
Vector(list(item.vector)),
|
||||
item.embedding_model,
|
||||
item.provider_request_id,
|
||||
Jsonb(item.embedding_usage),
|
||||
item.embedding_elapsed_ms,
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO rag.chunk_embedding_assignments (
|
||||
chunk_id, profile_hash, embedding_text_sha256,
|
||||
cache_profile_hash, cache_embedding_text_sha256,
|
||||
status, completed_at
|
||||
) VALUES (%s, %s, %s, %s, %s, 'READY', now())
|
||||
ON CONFLICT (chunk_id, profile_hash) DO NOTHING
|
||||
""",
|
||||
(
|
||||
item.chunk_id,
|
||||
item.embedding_profile_hash,
|
||||
item.embedding_text_sha256,
|
||||
item.embedding_profile_hash,
|
||||
item.embedding_text_sha256,
|
||||
),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE rag.chunks
|
||||
@@ -459,8 +647,9 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
count(*) FILTER (WHERE searchable)::integer AS searchable
|
||||
FROM rag.chunks
|
||||
WHERE knowledge_base_id = %s
|
||||
AND embedding_profile_hash = %s
|
||||
""",
|
||||
(KNOWLEDGE_BASE_ID,),
|
||||
(namespace.knowledge_base_id, profile_hash),
|
||||
).fetchone()
|
||||
if counts is None:
|
||||
raise SeedContractError("database_count_missing")
|
||||
@@ -470,6 +659,9 @@ def write_chunks(settings: Settings, chunks: Sequence[PreparedChunk]) -> dict[st
|
||||
def retrieve(
|
||||
settings: Settings,
|
||||
query_vector: tuple[float, ...],
|
||||
*,
|
||||
namespace: DemoNamespace,
|
||||
profile_hash: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
with psycopg.connect(database_dsn(settings), row_factory=dict_row) as connection:
|
||||
register_vector(connection)
|
||||
@@ -477,19 +669,25 @@ def retrieve(
|
||||
connection.execute("SET LOCAL hnsw.ef_search = 100")
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT id, metadata, embedding_text,
|
||||
1 - (embedding <=> %s) AS vector_score
|
||||
FROM rag.chunks
|
||||
WHERE searchable
|
||||
AND knowledge_base_id = %s
|
||||
AND access_scope_id = %s
|
||||
ORDER BY embedding <=> %s
|
||||
SELECT chunk.id, chunk.metadata, chunk.embedding_text,
|
||||
1 - (chunk.embedding <=> %s) AS vector_score
|
||||
FROM rag.chunks AS chunk
|
||||
JOIN rag.knowledge_bases AS knowledge_base
|
||||
ON knowledge_base.id = chunk.knowledge_base_id
|
||||
AND knowledge_base.active_embedding_profile_hash = %s
|
||||
WHERE chunk.searchable
|
||||
AND chunk.knowledge_base_id = %s
|
||||
AND chunk.access_scope_id = %s
|
||||
AND chunk.embedding_profile_hash = %s
|
||||
ORDER BY chunk.embedding <=> %s
|
||||
LIMIT %s
|
||||
""",
|
||||
(
|
||||
Vector(list(query_vector)),
|
||||
KNOWLEDGE_BASE_ID,
|
||||
ACCESS_SCOPE_ID,
|
||||
profile_hash,
|
||||
namespace.knowledge_base_id,
|
||||
namespace.access_scope_id,
|
||||
profile_hash,
|
||||
Vector(list(query_vector)),
|
||||
settings.vector_top_k,
|
||||
),
|
||||
@@ -502,12 +700,20 @@ async def evaluate_queries(
|
||||
queries: Sequence[DemoQuery],
|
||||
embedder: EmbeddingProvider,
|
||||
reranker: Reranker,
|
||||
*,
|
||||
namespace: DemoNamespace,
|
||||
profile_hash: str,
|
||||
) -> dict[str, float | int]:
|
||||
hits = 0
|
||||
answerable = 0
|
||||
for query in queries:
|
||||
query_result = await embedder.embed_query(query.query)
|
||||
candidates = retrieve(settings, query_result.vectors[0])
|
||||
candidates = retrieve(
|
||||
settings,
|
||||
query_result.vectors[0],
|
||||
namespace=namespace,
|
||||
profile_hash=profile_hash,
|
||||
)
|
||||
if not candidates:
|
||||
continue
|
||||
reranked = await reranker.rerank(
|
||||
@@ -552,14 +758,14 @@ async def async_main() -> int:
|
||||
return 2
|
||||
|
||||
settings = Settings()
|
||||
namespace = namespace_for_mode(mode)
|
||||
documents_path = Path(
|
||||
os.getenv("DEMO_DOCUMENTS_PATH", str(DEFAULT_SAMPLE_ROOT / "demo_documents.jsonl"))
|
||||
)
|
||||
queries_path = Path(
|
||||
os.getenv("DEMO_QUERIES_PATH", str(DEFAULT_SAMPLE_ROOT / "demo_queries.jsonl"))
|
||||
)
|
||||
cloud_embedder: BailianEmbeddingAdapter | None = None
|
||||
cloud_reranker: BailianRerankerAdapter | None = None
|
||||
cloud_gateway: ModelGatewayAdapter | None = None
|
||||
try:
|
||||
documents = load_documents(documents_path)
|
||||
queries = load_queries(queries_path)
|
||||
@@ -567,24 +773,9 @@ async def async_main() -> int:
|
||||
embedder: EmbeddingProvider
|
||||
reranker: Reranker
|
||||
if mode == "bailian":
|
||||
api_key = settings.bailian_api_key()
|
||||
cloud_embedder = BailianEmbeddingAdapter(
|
||||
api_key=api_key,
|
||||
base_url=settings.bailian_openai_base_url,
|
||||
model=settings.embedding_model,
|
||||
dimensions=settings.embedding_dimension,
|
||||
timeout_seconds=settings.model_timeout_seconds,
|
||||
max_retries=settings.model_max_retries,
|
||||
)
|
||||
cloud_reranker = BailianRerankerAdapter(
|
||||
api_key=api_key,
|
||||
base_url=settings.bailian_rerank_base_url,
|
||||
model=settings.rerank_model,
|
||||
timeout_seconds=settings.model_timeout_seconds,
|
||||
max_retries=settings.model_max_retries,
|
||||
)
|
||||
embedder = cloud_embedder
|
||||
reranker = cloud_reranker
|
||||
cloud_gateway = ModelGatewayAdapter.from_settings(settings)
|
||||
embedder = cloud_gateway
|
||||
reranker = cloud_gateway
|
||||
else:
|
||||
embedder = FakeEmbeddingProvider(settings.embedding_dimension)
|
||||
reranker = FakeReranker()
|
||||
@@ -599,9 +790,17 @@ async def async_main() -> int:
|
||||
vectors,
|
||||
profile_hash=profile_hash,
|
||||
embedding_model=resolved_model,
|
||||
namespace=namespace,
|
||||
)
|
||||
counts = write_chunks(settings, prepared, namespace=namespace)
|
||||
metrics = await evaluate_queries(
|
||||
settings,
|
||||
queries,
|
||||
embedder,
|
||||
reranker,
|
||||
namespace=namespace,
|
||||
profile_hash=profile_hash,
|
||||
)
|
||||
counts = write_chunks(settings, prepared)
|
||||
metrics = await evaluate_queries(settings, queries, embedder, reranker)
|
||||
output_summary(
|
||||
{
|
||||
"counts": counts,
|
||||
@@ -654,10 +853,8 @@ async def async_main() -> int:
|
||||
)
|
||||
return 1
|
||||
finally:
|
||||
if cloud_embedder is not None:
|
||||
await cloud_embedder.aclose()
|
||||
if cloud_reranker is not None:
|
||||
await cloud_reranker.aclose()
|
||||
if cloud_gateway is not None:
|
||||
await cloud_gateway.aclose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
Reference in New Issue
Block a user