Expose a runnable backend without giving the ingress layer secrets
Some checks failed
verify / verify (push) Has been cancelled

The backend can now be inspected through a loopback-only gateway while the database-aware API remains on the internal data network. A governed synthetic demo proves readiness, pgvector retrieval, reranking, and citation output through real HTTP without invoking cloud models.

Constraint: The previously exposed Bailian key is compromised and cannot be used for live validation

Constraint: The API must be locally reachable while retaining no internet egress

Rejected: Attach the API directly to the ingress network | a real socket test proved that configuration still had egress

Rejected: Publish a port from the internal-only network | Docker Desktop did not expose the host port

Confidence: high

Scope-risk: moderate

Reversibility: clean

Directive: Keep model and database credentials out of the gateway; do not relax the fixed demo identity/profile filters

Tested: make verify; 63 pytest tests; strict mypy; Ruff; Secret scan; Compose config; three backend image builds; API/DB/gateway healthy; migration exit 0; Swagger browser check; live/ready/meta/status/search HTTP; 20/20/20 index; API egress ENETUNREACH; empty gateway mounts and business environment

Not-tested: Live Bailian calls require a newly rotated key; full generated-answer flow and React UI are not implemented
This commit is contained in:
2026-07-12 16:37:02 +08:00
parent e89cca2b55
commit cfd6d4cbad
16 changed files with 1207 additions and 37 deletions

View File

@@ -0,0 +1 @@
"""HTTP API routers for the geological RAG backend."""

View File

@@ -0,0 +1,5 @@
"""Version 1 HTTP API routers."""
from app.api.v1.demo import router as demo_router
__all__ = ["demo_router"]

356
backend/app/api/v1/demo.py Normal file
View File

@@ -0,0 +1,356 @@
"""Read-only offline RAG demo endpoints backed by the synthetic dataset."""
from __future__ import annotations
import asyncio
import hashlib
import re
from dataclasses import dataclass
from typing import Annotated, Any, Literal, Protocol, cast
import psycopg
from fastapi import APIRouter, Depends, HTTPException, status
from pgvector.psycopg import register_vector
from pgvector.vector import Vector
from psycopg.rows import dict_row
from pydantic import BaseModel, Field, field_validator
from app.adapters.fake import FakeEmbeddingProvider, FakeReranker
from app.core.config import Settings, get_settings
from app.core.demo_identity import (
ACCESS_SCOPE_ID,
DEMO_EXPECTED_CHUNKS,
DEMO_FAKE_EMBEDDING_MODEL,
DEMO_SCOPE_NAME,
KNOWLEDGE_BASE_ID,
offline_embedding_profile_hash,
)
from app.core.secrets import SecretFileError
QUERY_MAX_LENGTH = 500
SNIPPET_MAX_LENGTH = 600
TITLE_MAX_LENGTH = 120
_SPACE_PATTERN = re.compile(r"\s+")
class DemoCounts(BaseModel):
"""Public aggregate counts for the synthetic dataset."""
chunks: int = Field(ge=0)
vectors: int = Field(ge=0)
searchable: int = Field(ge=0)
class DemoStatusResponse(BaseModel):
"""Safe readiness summary without database or approval internals."""
status: Literal["ready", "empty_dataset", "incomplete_dataset"]
dataset: Literal["synthetic-demo"] = "synthetic-demo"
counts: DemoCounts
class DemoSearchRequest(BaseModel):
"""Bounded request accepted by the offline demo search."""
query: str = Field(min_length=1, max_length=QUERY_MAX_LENGTH)
top_k: int = Field(default=5, ge=1, le=10)
@field_validator("query")
@classmethod
def normalize_query(cls, value: str) -> str:
normalized = _SPACE_PATTERN.sub(" ", value).strip()
if not normalized:
raise ValueError("query must contain non-whitespace text")
return normalized
class DemoSearchItem(BaseModel):
"""Public synthetic result; internal identifiers and hashes are excluded."""
title: str
snippet: str
page_label: str
score: float = Field(ge=0.0, le=1.0)
citation_id: str
class DemoSearchResponse(BaseModel):
"""Offline retrieval response with an explicit empty-dataset state."""
status: Literal["ok", "empty_dataset"]
dataset: Literal["synthetic-demo"] = "synthetic-demo"
results: list[DemoSearchItem]
@dataclass(frozen=True, slots=True)
class DemoCandidate:
"""Private retrieval projection used only while constructing safe results."""
source_key: str
title: str
text: str
page_start: int | None
page_end: int | None
@property
def rerank_text(self) -> str:
return f"{self.title}\n{self.text}"
class DemoRepository(Protocol):
"""Read-only persistence boundary used by the demo router."""
def counts(self) -> DemoCounts: ...
def search(self, query_vector: tuple[float, ...], *, limit: int) -> list[DemoCandidate]: ...
class PostgresDemoRepository:
"""Read-only PostgreSQL/pgvector projection for approved synthetic chunks."""
def __init__(self, settings: Settings) -> None:
self._settings = settings
def _dsn(self) -> str:
return (
self._settings.database_url()
.set(drivername="postgresql")
.render_as_string(hide_password=False)
)
def counts(self) -> DemoCounts:
profile_hash = offline_embedding_profile_hash(self._settings.embedding_dimension)
with psycopg.connect(
self._dsn(),
connect_timeout=2,
row_factory=dict_row,
) as connection:
row = connection.execute(
"""
SELECT
count(*)::integer AS chunks,
count(*) FILTER (WHERE chunk.embedding IS NOT NULL)::integer AS vectors,
count(*) FILTER (WHERE chunk.searchable)::integer AS searchable
FROM rag.chunks AS chunk
JOIN rag.access_scopes AS scope
ON scope.id = chunk.access_scope_id
AND scope.knowledge_base_id = chunk.knowledge_base_id
JOIN rag.documents AS document
ON document.id = chunk.document_id
AND document.knowledge_base_id = chunk.knowledge_base_id
AND document.access_scope_id = chunk.access_scope_id
JOIN rag.document_versions AS version
ON version.id = chunk.document_version_id
AND version.document_id = chunk.document_id
WHERE chunk.knowledge_base_id = %s
AND chunk.access_scope_id = %s
AND scope.name = %s
AND chunk.metadata ->> 'source_type' = 'synthetic'
AND chunk.index_status = 'READY'
AND chunk.approval_status = 'CLOUD_APPROVED'
AND chunk.deleted_at IS NULL
AND chunk.embedding_model = %s
AND chunk.embedding_profile_hash = %s
AND document.status = 'READY'
AND document.deleted_at IS NULL
AND document.active_version_id = chunk.document_version_id
AND version.status = 'READY'
AND version.review_state = 'CLOUD_APPROVED'
AND version.outbound_manifest_sha256 = chunk.outbound_manifest_sha256
AND version.embedding_profile_hash = chunk.embedding_profile_hash
""",
(
KNOWLEDGE_BASE_ID,
ACCESS_SCOPE_ID,
DEMO_SCOPE_NAME,
DEMO_FAKE_EMBEDDING_MODEL,
profile_hash,
),
).fetchone()
if row is None:
return DemoCounts(chunks=0, vectors=0, searchable=0)
return DemoCounts(
chunks=int(row["chunks"]),
vectors=int(row["vectors"]),
searchable=int(row["searchable"]),
)
def search(self, query_vector: tuple[float, ...], *, limit: int) -> list[DemoCandidate]:
if len(query_vector) != self._settings.embedding_dimension:
raise ValueError("query vector dimension does not match the demo index")
vector = Vector(list(query_vector))
profile_hash = offline_embedding_profile_hash(self._settings.embedding_dimension)
with psycopg.connect(
self._dsn(),
connect_timeout=2,
row_factory=dict_row,
) as connection:
register_vector(connection)
connection.execute("SET LOCAL statement_timeout = '3000ms'")
connection.execute("SET LOCAL hnsw.iterative_scan = strict_order")
connection.execute("SET LOCAL hnsw.ef_search = 100")
rows = connection.execute(
"""
SELECT
chunk.id::text AS source_key,
COALESCE(NULLIF(chunk.section_path ->> 0, ''), '合成地质资料') AS title,
chunk.cloud_text AS text,
chunk.page_start,
chunk.page_end
FROM rag.chunks AS chunk
JOIN rag.access_scopes AS scope
ON scope.id = chunk.access_scope_id
AND scope.knowledge_base_id = chunk.knowledge_base_id
JOIN rag.documents AS document
ON document.id = chunk.document_id
AND document.knowledge_base_id = chunk.knowledge_base_id
AND document.access_scope_id = chunk.access_scope_id
JOIN rag.document_versions AS version
ON version.id = chunk.document_version_id
AND version.document_id = chunk.document_id
WHERE chunk.knowledge_base_id = %s
AND chunk.access_scope_id = %s
AND scope.name = %s
AND chunk.metadata ->> 'source_type' = 'synthetic'
AND chunk.searchable IS TRUE
AND chunk.embedding IS NOT NULL
AND chunk.index_status = 'READY'
AND chunk.approval_status = 'CLOUD_APPROVED'
AND chunk.deleted_at IS NULL
AND chunk.embedding_model = %s
AND chunk.embedding_profile_hash = %s
AND document.status = 'READY'
AND document.deleted_at IS NULL
AND document.active_version_id = chunk.document_version_id
AND version.status = 'READY'
AND version.review_state = 'CLOUD_APPROVED'
AND version.outbound_manifest_sha256 = chunk.outbound_manifest_sha256
AND version.embedding_profile_hash = chunk.embedding_profile_hash
ORDER BY chunk.embedding <=> %s
LIMIT %s
""",
(
KNOWLEDGE_BASE_ID,
ACCESS_SCOPE_ID,
DEMO_SCOPE_NAME,
DEMO_FAKE_EMBEDDING_MODEL,
profile_hash,
vector,
limit,
),
).fetchall()
return [
DemoCandidate(
source_key=cast(str, row["source_key"]),
title=cast(str, row["title"]),
text=cast(str, row["text"]),
page_start=cast(int | None, row["page_start"]),
page_end=cast(int | None, row["page_end"]),
)
for row in rows
]
def get_demo_repository(
settings: Annotated[Settings, Depends(get_settings)],
) -> DemoRepository:
"""Build the default repository without loading any model credential."""
return PostgresDemoRepository(settings)
def _bounded_text(value: str, max_length: int) -> str:
normalized = _SPACE_PATTERN.sub(" ", value).strip()
if len(normalized) <= max_length:
return normalized
return f"{normalized[: max_length - 1]}"
def make_citation_id(source_key: str) -> str:
"""Derive a stable opaque citation without returning an internal UUID."""
digest = hashlib.sha256(f"{DEMO_SCOPE_NAME}:{source_key}".encode()).hexdigest()
return f"demo-{digest[:16]}"
def make_page_label(page_start: int | None, page_end: int | None) -> str:
if page_start is None or page_end is None:
return "页码未知"
if page_start == page_end:
return f"{page_start}"
return f"{page_start}-{page_end}"
def _safe_result(candidate: DemoCandidate, score: float) -> DemoSearchItem:
return DemoSearchItem(
title=f"合成资料|{_bounded_text(candidate.title, TITLE_MAX_LENGTH)}",
snippet=_bounded_text(candidate.text, SNIPPET_MAX_LENGTH),
page_label=make_page_label(candidate.page_start, candidate.page_end),
score=round(max(0.0, min(1.0, score)), 6),
citation_id=make_citation_id(candidate.source_key),
)
def _database_unavailable(exc: BaseException) -> HTTPException:
return HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="database unavailable",
)
router = APIRouter(prefix="/api/v1/demo", tags=["offline-demo"])
@router.get("/status", response_model=DemoStatusResponse)
def demo_status(repository: Annotated[DemoRepository, Depends(get_demo_repository)]) -> Any:
try:
counts = repository.counts()
except (OSError, SecretFileError, psycopg.Error) as exc:
raise _database_unavailable(exc) from exc
state: Literal["ready", "empty_dataset", "incomplete_dataset"]
if counts.chunks == 0:
state = "empty_dataset"
elif (
counts.chunks == DEMO_EXPECTED_CHUNKS
and counts.vectors == DEMO_EXPECTED_CHUNKS
and counts.searchable == DEMO_EXPECTED_CHUNKS
):
state = "ready"
else:
state = "incomplete_dataset"
return DemoStatusResponse(status=state, counts=counts)
@router.post("/search", response_model=DemoSearchResponse)
async def demo_search(
request: DemoSearchRequest,
repository: Annotated[DemoRepository, Depends(get_demo_repository)],
settings: Annotated[Settings, Depends(get_settings)],
) -> Any:
embedder = FakeEmbeddingProvider(settings.embedding_dimension)
reranker = FakeReranker()
query_result = await embedder.embed_query(request.query)
candidate_limit = min(max(request.top_k * 3, 10), 20)
try:
candidates = await asyncio.to_thread(
repository.search,
query_result.vectors[0],
limit=candidate_limit,
)
except (OSError, SecretFileError, psycopg.Error) as exc:
raise _database_unavailable(exc) from exc
if not candidates:
return DemoSearchResponse(status="empty_dataset", results=[])
reranked = await reranker.rerank(
request.query,
[candidate.rerank_text for candidate in candidates],
top_n=min(request.top_k, len(candidates)),
)
results = [
_safe_result(candidates[item.index], item.relevance_score) for item in reranked.items
]
return DemoSearchResponse(status="ok", results=results)

View File

@@ -0,0 +1,35 @@
"""Stable identity and embedding profile for the public synthetic demo corpus."""
from __future__ import annotations
import hashlib
import json
import uuid
DEMO_SCOPE_NAME = "synthetic-demo"
DEMO_EXPECTED_CHUNKS = 20
DEMO_FAKE_EMBEDDING_MODEL = "fake-feature-hash-v1"
IDENTITY_NAMESPACE = uuid.UUID("eef85571-1f64-4a09-86d7-53fd329c3eb2")
KNOWLEDGE_BASE_ID = uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-demo-knowledge-base")
ACCESS_SCOPE_ID = uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-demo-public-scope")
def offline_embedding_profile_hash(dimension: int) -> str:
"""Return the exact profile bound to vectors produced by the offline embedder."""
profile = {
"api_mode": "deterministic-offline",
"dimension": dimension,
"endpoint_identity_hash": "local-fake",
"model": DEMO_FAKE_EMBEDDING_MODEL,
"normalization": "provider-default",
"profile_version": 1,
}
serialized = json.dumps(
profile,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()

161
backend/app/gateway.py Normal file
View File

@@ -0,0 +1,161 @@
"""Small fixed-origin ingress gateway with explicit proxy boundaries."""
from collections.abc import AsyncIterator, Mapping
from contextlib import asynccontextmanager
import httpx
from fastapi import FastAPI, Request, Response, status
from fastapi.responses import JSONResponse
UPSTREAM_ORIGIN = httpx.URL("http://api:8000")
MAX_REQUEST_BODY_BYTES = 1024 * 1024
SUPPORTED_METHODS = ["GET", "POST", "HEAD", "OPTIONS"]
REQUEST_HEADER_ALLOWLIST = frozenset(
{
"accept",
"accept-language",
"access-control-request-headers",
"access-control-request-method",
"authorization",
"content-type",
"if-match",
"if-none-match",
"origin",
"range",
"traceparent",
"tracestate",
"x-request-id",
}
)
RESPONSE_HEADER_ALLOWLIST = frozenset(
{
"accept-ranges",
"access-control-allow-credentials",
"access-control-allow-headers",
"access-control-allow-methods",
"access-control-allow-origin",
"access-control-max-age",
"allow",
"cache-control",
"content-disposition",
"content-language",
"content-range",
"content-type",
"etag",
"last-modified",
"retry-after",
"vary",
"www-authenticate",
"x-request-id",
}
)
def _allowlisted_headers(headers: Mapping[str, str], allowlist: frozenset[str]) -> dict[str, str]:
return {name: value for name, value in headers.items() if name.lower() in allowlist}
def _upstream_url(request: Request) -> httpx.URL:
raw_path_value: object = request.scope.get("raw_path")
if isinstance(raw_path_value, bytes):
raw_path = raw_path_value
else:
raw_path = request.url.path.encode("utf-8")
if not raw_path.startswith(b"/"):
raw_path = b"/" + raw_path
query_value: object = request.scope.get("query_string")
query = query_value if isinstance(query_value, bytes) else b""
raw_target = raw_path + (b"?" + query if query else b"")
# copy_with preserves the fixed scheme, host, and port even for //host-like paths.
return UPSTREAM_ORIGIN.copy_with(raw_path=raw_target)
async def _bounded_body(request: Request) -> bytes | None:
declared_length = request.headers.get("content-length")
if declared_length is not None:
try:
parsed_length = int(declared_length)
except ValueError:
return None
if parsed_length < 0 or parsed_length > MAX_REQUEST_BODY_BYTES:
return None
body = bytearray()
async for chunk in request.stream():
if len(body) + len(chunk) > MAX_REQUEST_BODY_BYTES:
return None
body.extend(chunk)
return bytes(body)
def create_gateway_app(transport: httpx.AsyncBaseTransport | None = None) -> FastAPI:
"""Create a no-secret gateway; an injected transport enables hermetic tests."""
upstream_client = httpx.AsyncClient(
timeout=httpx.Timeout(10.0, connect=2.0),
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
follow_redirects=False,
transport=transport,
trust_env=False,
)
# Drop httpx convenience defaults; Host and body framing are added by the protocol layer.
upstream_client.headers.clear()
@asynccontextmanager
async def lifespan(_: FastAPI) -> AsyncIterator[None]:
try:
yield
finally:
await upstream_client.aclose()
gateway = FastAPI(
title="Geological RAG Gateway",
docs_url=None,
redoc_url=None,
openapi_url=None,
lifespan=lifespan,
)
@gateway.get("/gateway/live", include_in_schema=False)
async def gateway_live() -> dict[str, str]:
return {"status": "ok"}
@gateway.api_route("/{path:path}", methods=SUPPORTED_METHODS, include_in_schema=False)
async def proxy(request: Request, path: str) -> Response: # noqa: ARG001
body = await _bounded_body(request)
if body is None:
return JSONResponse(
status_code=status.HTTP_413_CONTENT_TOO_LARGE,
content={"detail": "request body too large"},
)
upstream_request = upstream_client.build_request(
request.method,
_upstream_url(request),
headers=_allowlisted_headers(request.headers, REQUEST_HEADER_ALLOWLIST),
content=body,
)
try:
upstream_response = await upstream_client.send(upstream_request)
except httpx.RequestError:
return JSONResponse(
status_code=status.HTTP_502_BAD_GATEWAY,
content={"detail": "upstream unavailable"},
)
return Response(
content=upstream_response.content,
status_code=upstream_response.status_code,
headers=_allowlisted_headers(
upstream_response.headers,
RESPONSE_HEADER_ALLOWLIST,
),
)
return gateway
app = create_gateway_app()

View File

@@ -1,39 +1,51 @@
"""Minimal FastAPI entrypoint; product endpoints are added in stage 2."""
"""FastAPI entrypoint with dependency-free liveness and database readiness probes."""
from typing import Any
import psycopg
import uvicorn
from fastapi import FastAPI, HTTPException, status
from fastapi import FastAPI, Response, status
from app import __version__
from app.api.v1 import demo_router
from app.core.config import get_settings
from app.core.secrets import SecretFileError
app = FastAPI(title="Geological RAG API", version=__version__)
app.include_router(demo_router)
type HealthPayload = dict[str, str | dict[str, str]]
@app.get("/health/live", tags=["health"])
@app.get("/api/v1/health/live", tags=["health"])
def live() -> dict[str, str]:
return {"status": "ok", "version": __version__}
@app.get(
"/health/ready",
tags=["health"],
responses={status.HTTP_503_SERVICE_UNAVAILABLE: {"description": "Database unavailable"}},
)
@app.get("/api/v1/health/ready", tags=["health"])
def ready() -> dict[str, str]:
def ready(response: Response) -> HealthPayload:
settings = get_settings()
try:
dsn = settings.database_url().set(drivername="postgresql")
with psycopg.connect(
dsn.render_as_string(hide_password=False),
connect_timeout=2,
autocommit=True,
) as connection:
connection.execute("SELECT 1")
except (OSError, SecretFileError, psycopg.Error) as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="database unavailable",
) from exc
return {"status": "ready"}
result = connection.execute("SELECT 1").fetchone()
if result != (1,):
raise psycopg.DatabaseError("readiness query returned an unexpected result")
except (OSError, SecretFileError, psycopg.Error):
# Do not expose connection strings, secret paths, hostnames, or driver errors.
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
return {"status": "not_ready", "checks": {"database": "unavailable"}}
return {"status": "ready", "checks": {"database": "ok"}}
@app.get("/api/v1/meta", tags=["meta"])
@@ -52,5 +64,5 @@ def meta() -> dict[str, Any]:
if __name__ == "__main__":
# Container ingress is controlled by Compose; only the edge proxy publishes a port.
# Compose publishes this listener only on the host loopback interface.
uvicorn.run("app.main:app", host="0.0.0.0", port=8000) # noqa: S104

View File

@@ -23,14 +23,17 @@ from psycopg.types.json import Jsonb
from app.adapters.bailian import BailianEmbeddingAdapter, BailianRerankerAdapter
from app.adapters.fake import FakeEmbeddingProvider, FakeReranker, lexical_features
from app.core.config import Settings
from app.core.demo_identity import (
ACCESS_SCOPE_ID,
IDENTITY_NAMESPACE,
KNOWLEDGE_BASE_ID,
offline_embedding_profile_hash,
)
from app.core.secrets import SecretFileError
from app.ports.model_providers import EmbeddingProvider, ModelProviderError, Reranker
PROJECT_ROOT = Path(__file__).resolve().parents[3]
DEFAULT_SAMPLE_ROOT = PROJECT_ROOT / "data" / "samples" / "public"
IDENTITY_NAMESPACE = uuid.UUID("eef85571-1f64-4a09-86d7-53fd329c3eb2")
KNOWLEDGE_BASE_ID = uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-demo-knowledge-base")
ACCESS_SCOPE_ID = uuid.uuid5(IDENTITY_NAMESPACE, "synthetic-demo-public-scope")
@dataclass(frozen=True, slots=True)
@@ -139,13 +142,12 @@ def load_queries(path: Path) -> list[DemoQuery]:
def embedding_profile_hash(settings: Settings, mode: str) -> str:
endpoint_identity = "local-fake"
model = "fake-feature-hash-v1"
api_mode = "deterministic-offline"
if mode == "bailian":
endpoint_identity = sha256_text(urlsplit(settings.bailian_openai_base_url).hostname or "")
model = settings.embedding_model
api_mode = "openai-compatible"
if mode != "bailian":
return offline_embedding_profile_hash(settings.embedding_dimension)
endpoint_identity = sha256_text(urlsplit(settings.bailian_openai_base_url).hostname or "")
model = settings.embedding_model
api_mode = "openai-compatible"
profile = {
"api_mode": api_mode,
"dimension": settings.embedding_dimension,