YoVinchen
ecdb10c37a
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
113 lines
3.8 KiB
Python
113 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import pytest
|
|
|
|
from app.services.evaluation import (
|
|
EvaluationContractError,
|
|
UnjudgedCandidateError,
|
|
bootstrap_mean_confidence_interval,
|
|
evaluate_citations,
|
|
evaluate_ranking,
|
|
evaluate_refusals,
|
|
freeze_run_config,
|
|
)
|
|
|
|
|
|
def test_ranking_metrics_match_hand_calculated_case() -> None:
|
|
metrics = evaluate_ranking(
|
|
["negative", "relevant-b", "relevant-a"],
|
|
relevance={"relevant-a": 2.0, "relevant-b": 1.0, "negative": 0.0},
|
|
judged_document_ids=frozenset({"negative", "relevant-a", "relevant-b"}),
|
|
evidence_groups=(frozenset({"relevant-a"}), frozenset({"relevant-b"})),
|
|
k=3,
|
|
)
|
|
|
|
expected_dcg = 1 / math.log2(3) + 3 / math.log2(4)
|
|
ideal_dcg = 3 + 1 / math.log2(3)
|
|
assert metrics.hit_at_k == 1.0
|
|
assert metrics.recall_at_k == 1.0
|
|
assert metrics.reciprocal_rank == 0.5
|
|
assert metrics.ndcg_at_k == pytest.approx(expected_dcg / ideal_dcg)
|
|
assert metrics.complete_hit_at_k == 1.0
|
|
assert metrics.evidence_group_recall_at_k == 1.0
|
|
|
|
|
|
def test_unjudged_candidate_is_never_silently_scored_as_zero() -> None:
|
|
with pytest.raises(UnjudgedCandidateError, match="1 unjudged"):
|
|
evaluate_ranking(
|
|
["pooled-but-unjudged", "relevant"],
|
|
relevance={"relevant": 1.0},
|
|
judged_document_ids=frozenset({"relevant"}),
|
|
evidence_groups=(frozenset({"relevant"}),),
|
|
k=2,
|
|
)
|
|
|
|
|
|
def test_partial_evidence_groups_are_not_complete_hits() -> None:
|
|
metrics = evaluate_ranking(
|
|
["evidence-a", "negative"],
|
|
relevance={"evidence-a": 1.0, "evidence-b": 1.0, "negative": 0.0},
|
|
judged_document_ids=frozenset({"evidence-a", "evidence-b", "negative"}),
|
|
evidence_groups=(frozenset({"evidence-a"}), frozenset({"evidence-b"})),
|
|
k=2,
|
|
)
|
|
|
|
assert metrics.hit_at_k == 1.0
|
|
assert metrics.complete_hit_at_k == 0.0
|
|
assert metrics.evidence_group_recall_at_k == 0.5
|
|
|
|
|
|
def test_citation_precision_recall_and_empty_success_contract() -> None:
|
|
partial = evaluate_citations(
|
|
["supported", "unsupported"],
|
|
supported_source_ids=frozenset({"supported", "missed"}),
|
|
)
|
|
empty = evaluate_citations([], supported_source_ids=frozenset())
|
|
|
|
assert partial.precision == 0.5
|
|
assert partial.recall == 0.5
|
|
assert partial.f1 == 0.5
|
|
assert empty.precision == empty.recall == empty.f1 == 1.0
|
|
|
|
|
|
def test_refusal_metrics_use_unanswerable_as_positive_class() -> None:
|
|
metrics = evaluate_refusals(
|
|
[True, False, True, False],
|
|
answerable_labels=[False, False, True, True],
|
|
)
|
|
|
|
assert metrics.true_positive == 1
|
|
assert metrics.false_positive == 1
|
|
assert metrics.false_negative == 1
|
|
assert metrics.true_negative == 1
|
|
assert metrics.precision == 0.5
|
|
assert metrics.recall == 0.5
|
|
assert metrics.f1 == 0.5
|
|
assert metrics.accuracy == 0.5
|
|
|
|
|
|
def test_bootstrap_confidence_interval_is_seeded_and_bounded() -> None:
|
|
first = bootstrap_mean_confidence_interval([0.0, 0.5, 1.0], seed=20260713, iterations=500)
|
|
second = bootstrap_mean_confidence_interval([0.0, 0.5, 1.0], seed=20260713, iterations=500)
|
|
|
|
assert first == second
|
|
assert first.mean == 0.5
|
|
assert 0.0 <= first.lower <= first.mean <= first.upper <= 1.0
|
|
|
|
|
|
def test_run_config_freeze_is_canonical_and_rejects_secrets() -> None:
|
|
first_json, first_hash = freeze_run_config(
|
|
{"models": {"embedding": "text-embedding-v4"}, "seed": 7}
|
|
)
|
|
second_json, second_hash = freeze_run_config(
|
|
{"seed": 7, "models": {"embedding": "text-embedding-v4"}}
|
|
)
|
|
|
|
assert first_json == second_json
|
|
assert first_hash == second_hash
|
|
assert len(first_hash) == 64
|
|
with pytest.raises(EvaluationContractError, match="secret-shaped"):
|
|
freeze_run_config({"api_key": "must-not-be-frozen"})
|