YoVinchen
ecdb10c37a
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
98 lines
3.1 KiB
Python
98 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
|
|
import pytest
|
|
|
|
from app.core.demo_identity import KNOWLEDGE_BASE_ID, offline_embedding_profile_hash
|
|
from app.persistence.retrieval import ActiveEmbeddingProfile
|
|
from app.services.retrieval import (
|
|
EffectiveRetrievalParameters,
|
|
RetrievalActor,
|
|
RetrievalHit,
|
|
RetrievalResult,
|
|
RetrievalTimings,
|
|
)
|
|
from app.tools.evaluate_demo import evaluate_demo_queries
|
|
from app.tools.seed_demo import DemoDocument, DemoQuery
|
|
|
|
|
|
@dataclass
|
|
class StubService:
|
|
async def search(
|
|
self,
|
|
*,
|
|
actor: RetrievalActor,
|
|
knowledge_base_id: uuid.UUID,
|
|
query: str,
|
|
vector_top_k: int,
|
|
rerank_top_n: int,
|
|
) -> RetrievalResult:
|
|
del actor, query, vector_top_k, rerank_top_n
|
|
assert knowledge_base_id == KNOWLEDGE_BASE_ID
|
|
return RetrievalResult(
|
|
status="ok",
|
|
knowledge_base_id=KNOWLEDGE_BASE_ID,
|
|
access_scope_count=1,
|
|
profile=ActiveEmbeddingProfile(
|
|
profile_hash=offline_embedding_profile_hash(1024),
|
|
model="fake-feature-hash-v1",
|
|
dimension=1024,
|
|
synthetic=True,
|
|
),
|
|
parameters=EffectiveRetrievalParameters(vector_top_k=2, rerank_top_n=2),
|
|
rerank_status="applied",
|
|
degradation_reason=None,
|
|
embedding_request_id=None,
|
|
rerank_request_id=None,
|
|
embedding_model="fake-feature-hash-v1",
|
|
rerank_model="fake-lexical-rerank-v1",
|
|
timings=RetrievalTimings(1, 1, 1, 3),
|
|
results=(
|
|
RetrievalHit(
|
|
rank=1,
|
|
vector_rank=1,
|
|
citation_id=uuid.uuid4(),
|
|
document_id=uuid.uuid4(),
|
|
source_name="doc-relevant.json",
|
|
snippet="synthetic evidence",
|
|
section_path=("Synthetic",),
|
|
page_start=1,
|
|
page_end=1,
|
|
page_label="第 1 页",
|
|
vector_score=0.9,
|
|
rerank_score=0.9,
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_demo_runner_builds_scored_and_unanswerable_cases() -> None:
|
|
documents = [
|
|
DemoDocument("doc-relevant", "t", "c", "r", "m", 1, "synthetic"),
|
|
DemoDocument("doc-negative", "t", "c", "r", "m", 2, "synthetic"),
|
|
]
|
|
queries = [
|
|
DemoQuery("q1", "answerable", ("doc-relevant",), True),
|
|
DemoQuery("q2", "unanswerable", (), False),
|
|
]
|
|
|
|
artifact = await evaluate_demo_queries(
|
|
service=StubService(),
|
|
actor=RetrievalActor(subject="test", grants=()),
|
|
documents=documents,
|
|
queries=queries,
|
|
vector_top_k=2,
|
|
rerank_top_n=2,
|
|
metric_cutoff=1,
|
|
)
|
|
|
|
assert artifact["case_count"] == 2
|
|
assert artifact["answerable_case_count"] == 1
|
|
assert artifact["metrics"]["hit_at_1"] == 1.0
|
|
assert artifact["metrics"]["mrr"] == 1.0
|
|
assert artifact["cases"][0]["metrics"]["complete_hit_at_k"] == 1.0
|
|
assert artifact["cases"][1]["metrics"] is None
|