Make the governed RAG evidence path executable end to end
Some checks failed
verify / verify (push) Has been cancelled
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
This commit is contained in:
345
backend/app/persistence/job_queue.py
Normal file
345
backend/app/persistence/job_queue.py
Normal file
@@ -0,0 +1,345 @@
|
||||
"""Transactional PostgreSQL runtime for the fenced background-job queue.
|
||||
|
||||
Every repository call owns a short database transaction. A claimed job is
|
||||
returned only after that transaction commits, so handlers never run while the
|
||||
claim row lock is held. Terminal mutations require the immutable worker/token
|
||||
lease pair and fail closed when the lease has moved to another worker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import psycopg
|
||||
from psycopg import Connection
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
from app.persistence.job_queue_sql import (
|
||||
CLAIM_JOB_SQL,
|
||||
COMPLETE_JOB_SQL,
|
||||
FAIL_OR_RETRY_JOB_SQL,
|
||||
HEARTBEAT_JOB_SQL,
|
||||
REAP_EXPIRED_JOBS_SQL,
|
||||
)
|
||||
|
||||
type JobRow = dict[str, Any]
|
||||
type ConnectionFactory = Callable[[str, int], Connection[JobRow]]
|
||||
|
||||
_NAMED_BIND = re.compile(r"(?<!:):([a-zA-Z_][a-zA-Z0-9_]*)")
|
||||
_ERROR_CODE = re.compile(r"^[A-Z][A-Z0-9_]{0,127}$")
|
||||
|
||||
|
||||
def _psycopg_statement(statement: str) -> str:
|
||||
"""Translate the canonical named binds to psycopg's mapping syntax."""
|
||||
|
||||
return _NAMED_BIND.sub(r"%(\1)s", statement)
|
||||
|
||||
|
||||
_CLAIM = _psycopg_statement(CLAIM_JOB_SQL)
|
||||
_HEARTBEAT = _psycopg_statement(HEARTBEAT_JOB_SQL)
|
||||
_COMPLETE = _psycopg_statement(COMPLETE_JOB_SQL)
|
||||
_FAIL_OR_RETRY = _psycopg_statement(FAIL_OR_RETRY_JOB_SQL)
|
||||
_REAP_EXPIRED = _psycopg_statement(REAP_EXPIRED_JOBS_SQL)
|
||||
|
||||
|
||||
class JobQueueError(RuntimeError):
|
||||
"""Base class for safe, non-secret queue errors."""
|
||||
|
||||
|
||||
class LeaseLostError(JobQueueError):
|
||||
"""Raised when a fenced mutation no longer owns the job lease."""
|
||||
|
||||
|
||||
class InvalidJobRowError(JobQueueError):
|
||||
"""Raised when a claimed row violates the runtime shape contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JobLease:
|
||||
"""The complete fencing identity required for mutations after claim."""
|
||||
|
||||
job_id: uuid.UUID
|
||||
worker_id: str
|
||||
lease_token: uuid.UUID
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackgroundJob:
|
||||
"""A claimed job and its immutable lease identity."""
|
||||
|
||||
id: uuid.UUID
|
||||
job_type: str
|
||||
required_capability: str
|
||||
resource_type: str
|
||||
resource_id: uuid.UUID
|
||||
idempotency_key: str
|
||||
payload: Mapping[str, object]
|
||||
stage: str
|
||||
progress: int
|
||||
priority: int
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
run_after: datetime
|
||||
lease_until: datetime
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
lease: JobLease
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LeaseHeartbeat:
|
||||
job_id: uuid.UUID
|
||||
lease_until: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JobState:
|
||||
"""Metadata returned by a fenced terminal or reaper mutation."""
|
||||
|
||||
job_id: uuid.UUID
|
||||
status: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
finished_at: datetime | None
|
||||
|
||||
|
||||
def _default_connection_factory(dsn: str, connect_timeout: int) -> Connection[JobRow]:
|
||||
return psycopg.connect(
|
||||
dsn,
|
||||
connect_timeout=connect_timeout,
|
||||
row_factory=dict_row,
|
||||
application_name="geological-rag-worker",
|
||||
)
|
||||
|
||||
|
||||
def _require_uuid(row: Mapping[str, object], name: str) -> uuid.UUID:
|
||||
value = row.get(name)
|
||||
if not isinstance(value, uuid.UUID):
|
||||
raise InvalidJobRowError(f"job row has invalid {name}")
|
||||
return value
|
||||
|
||||
|
||||
def _require_text(row: Mapping[str, object], name: str) -> str:
|
||||
value = row.get(name)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise InvalidJobRowError(f"job row has invalid {name}")
|
||||
return value
|
||||
|
||||
|
||||
def _require_integer(row: Mapping[str, object], name: str) -> int:
|
||||
value = row.get(name)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise InvalidJobRowError(f"job row has invalid {name}")
|
||||
return value
|
||||
|
||||
|
||||
def _require_datetime(row: Mapping[str, object], name: str) -> datetime:
|
||||
value = row.get(name)
|
||||
if not isinstance(value, datetime):
|
||||
raise InvalidJobRowError(f"job row has invalid {name}")
|
||||
return value
|
||||
|
||||
|
||||
def _job_from_row(row: JobRow, expected_worker_id: str) -> BackgroundJob:
|
||||
job_id = _require_uuid(row, "id")
|
||||
lease_owner = _require_text(row, "lease_owner")
|
||||
if lease_owner != expected_worker_id:
|
||||
raise InvalidJobRowError("claimed job has an unexpected lease owner")
|
||||
lease_token = _require_uuid(row, "lease_token")
|
||||
if row.get("status") != "RUNNING":
|
||||
raise InvalidJobRowError("claimed job is not running")
|
||||
|
||||
payload_value = row.get("payload")
|
||||
if not isinstance(payload_value, dict) or not all(
|
||||
isinstance(key, str) for key in payload_value
|
||||
):
|
||||
raise InvalidJobRowError("job row has invalid payload")
|
||||
|
||||
return BackgroundJob(
|
||||
id=job_id,
|
||||
job_type=_require_text(row, "job_type"),
|
||||
required_capability=_require_text(row, "required_capability"),
|
||||
resource_type=_require_text(row, "resource_type"),
|
||||
resource_id=_require_uuid(row, "resource_id"),
|
||||
idempotency_key=_require_text(row, "idempotency_key"),
|
||||
payload=dict(payload_value),
|
||||
stage=_require_text(row, "stage"),
|
||||
progress=_require_integer(row, "progress"),
|
||||
priority=_require_integer(row, "priority"),
|
||||
attempt=_require_integer(row, "attempt"),
|
||||
max_attempts=_require_integer(row, "max_attempts"),
|
||||
run_after=_require_datetime(row, "run_after"),
|
||||
lease_until=_require_datetime(row, "lease_until"),
|
||||
created_at=_require_datetime(row, "created_at"),
|
||||
updated_at=_require_datetime(row, "updated_at"),
|
||||
lease=JobLease(
|
||||
job_id=job_id,
|
||||
worker_id=lease_owner,
|
||||
lease_token=lease_token,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _state_from_row(row: JobRow) -> JobState:
|
||||
finished_at = row.get("finished_at")
|
||||
if finished_at is not None and not isinstance(finished_at, datetime):
|
||||
raise InvalidJobRowError("job row has invalid finished_at")
|
||||
return JobState(
|
||||
job_id=_require_uuid(row, "id"),
|
||||
status=_require_text(row, "status"),
|
||||
attempt=_require_integer(row, "attempt"),
|
||||
max_attempts=_require_integer(row, "max_attempts"),
|
||||
finished_at=finished_at,
|
||||
)
|
||||
|
||||
|
||||
def _validate_worker_id(worker_id: str) -> str:
|
||||
normalized = worker_id.strip()
|
||||
if not normalized or len(normalized) > 200:
|
||||
raise ValueError("worker_id must contain 1 to 200 characters")
|
||||
return normalized
|
||||
|
||||
|
||||
def _validate_lease_seconds(lease_seconds: int) -> int:
|
||||
if isinstance(lease_seconds, bool) or not 1 <= lease_seconds <= 86_400:
|
||||
raise ValueError("lease_seconds must be between 1 and 86400")
|
||||
return lease_seconds
|
||||
|
||||
|
||||
class PsycopgJobQueue:
|
||||
"""Short-transaction repository around the canonical queue statements."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dsn: str,
|
||||
*,
|
||||
connect_timeout: int = 5,
|
||||
connection_factory: ConnectionFactory = _default_connection_factory,
|
||||
) -> None:
|
||||
if not dsn.strip():
|
||||
raise ValueError("dsn must not be empty")
|
||||
if isinstance(connect_timeout, bool) or not 1 <= connect_timeout <= 60:
|
||||
raise ValueError("connect_timeout must be between 1 and 60")
|
||||
self._dsn = dsn
|
||||
self._connect_timeout = connect_timeout
|
||||
self._connection_factory = connection_factory
|
||||
|
||||
def _fetch_one(self, statement: str, parameters: Mapping[str, object]) -> JobRow | None:
|
||||
with self._connection_factory(self._dsn, self._connect_timeout) as connection:
|
||||
with connection.transaction():
|
||||
cursor = connection.execute(statement, parameters)
|
||||
return cursor.fetchone()
|
||||
|
||||
def _fetch_all(self, statement: str, parameters: Mapping[str, object]) -> list[JobRow]:
|
||||
with self._connection_factory(self._dsn, self._connect_timeout) as connection:
|
||||
with connection.transaction():
|
||||
cursor = connection.execute(statement, parameters)
|
||||
return list(cursor.fetchall())
|
||||
|
||||
def claim(
|
||||
self,
|
||||
*,
|
||||
worker_id: str,
|
||||
worker_capabilities: Sequence[str],
|
||||
lease_seconds: int,
|
||||
) -> BackgroundJob | None:
|
||||
owner = _validate_worker_id(worker_id)
|
||||
capabilities = tuple(
|
||||
capability.strip() for capability in worker_capabilities if capability.strip()
|
||||
)
|
||||
if not capabilities:
|
||||
raise ValueError("worker_capabilities must not be empty")
|
||||
if len(capabilities) != len(set(capabilities)):
|
||||
raise ValueError("worker_capabilities must not contain duplicates")
|
||||
|
||||
row = self._fetch_one(
|
||||
_CLAIM,
|
||||
{
|
||||
"worker_id": owner,
|
||||
"worker_capabilities": list(capabilities),
|
||||
"lease_seconds": _validate_lease_seconds(lease_seconds),
|
||||
},
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return _job_from_row(row, owner)
|
||||
|
||||
def heartbeat(self, lease: JobLease, *, lease_seconds: int) -> LeaseHeartbeat:
|
||||
row = self._fetch_one(
|
||||
_HEARTBEAT,
|
||||
{
|
||||
"job_id": lease.job_id,
|
||||
"worker_id": _validate_worker_id(lease.worker_id),
|
||||
"lease_token": lease.lease_token,
|
||||
"lease_seconds": _validate_lease_seconds(lease_seconds),
|
||||
},
|
||||
)
|
||||
if row is None:
|
||||
raise LeaseLostError("job lease is no longer owned")
|
||||
return LeaseHeartbeat(
|
||||
job_id=_require_uuid(row, "id"),
|
||||
lease_until=_require_datetime(row, "lease_until"),
|
||||
)
|
||||
|
||||
def complete(self, lease: JobLease) -> JobState:
|
||||
row = self._fetch_one(_COMPLETE, self._lease_parameters(lease))
|
||||
if row is None:
|
||||
raise LeaseLostError("job lease is no longer owned")
|
||||
return _state_from_row(row)
|
||||
|
||||
def fail_or_retry(
|
||||
self,
|
||||
lease: JobLease,
|
||||
*,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
retry_delay_seconds: int,
|
||||
) -> JobState:
|
||||
if not _ERROR_CODE.fullmatch(error_code):
|
||||
raise ValueError("error_code must be a stable uppercase identifier")
|
||||
if not error_message.strip():
|
||||
raise ValueError("error_message must not be empty")
|
||||
if isinstance(retry_delay_seconds, bool) or not 0 <= retry_delay_seconds <= 86_400:
|
||||
raise ValueError("retry_delay_seconds must be between 0 and 86400")
|
||||
|
||||
parameters = self._lease_parameters(lease)
|
||||
parameters.update(
|
||||
{
|
||||
"error_code": error_code,
|
||||
"error_message": error_message[:2000],
|
||||
"retry_delay_seconds": retry_delay_seconds,
|
||||
}
|
||||
)
|
||||
row = self._fetch_one(_FAIL_OR_RETRY, parameters)
|
||||
if row is None:
|
||||
raise LeaseLostError("job lease is no longer owned")
|
||||
return _state_from_row(row)
|
||||
|
||||
def reap_expired(
|
||||
self,
|
||||
*,
|
||||
lock_key: int,
|
||||
batch_size: int = 100,
|
||||
) -> tuple[JobState, ...]:
|
||||
if isinstance(lock_key, bool) or not -(2**63) <= lock_key < 2**63:
|
||||
raise ValueError("lock_key must be a signed 64-bit integer")
|
||||
if isinstance(batch_size, bool) or not 1 <= batch_size <= 1000:
|
||||
raise ValueError("batch_size must be between 1 and 1000")
|
||||
rows = self._fetch_all(
|
||||
_REAP_EXPIRED,
|
||||
{"lock_key": lock_key, "batch_size": batch_size},
|
||||
)
|
||||
return tuple(_state_from_row(row) for row in rows)
|
||||
|
||||
@staticmethod
|
||||
def _lease_parameters(lease: JobLease) -> dict[str, object]:
|
||||
return {
|
||||
"job_id": lease.job_id,
|
||||
"worker_id": _validate_worker_id(lease.worker_id),
|
||||
"lease_token": lease.lease_token,
|
||||
}
|
||||
Reference in New Issue
Block a user