"""Fenced PostgreSQL job-queue statements. Callers must execute each statement in a transaction and treat an empty ``RETURNING`` result as loss of the lease. No external model call should hold the claim transaction open. """ CLAIM_JOB_SQL = """ WITH candidate AS ( SELECT job.id FROM rag.background_jobs AS job WHERE job.attempt < job.max_attempts AND job.required_capability = ANY(CAST(:worker_capabilities AS text[])) AND ( (job.status = 'QUEUED' AND job.run_after <= now()) OR (job.status = 'RUNNING' AND job.lease_until < now()) ) ORDER BY job.priority DESC, job.run_after, job.created_at FOR UPDATE SKIP LOCKED LIMIT 1 ) UPDATE rag.background_jobs AS job SET status = 'RUNNING', lease_owner = :worker_id, lease_token = gen_random_uuid(), lease_until = now() + make_interval(secs => CAST(:lease_seconds AS integer)), attempt = job.attempt + 1, updated_at = now(), finished_at = NULL FROM candidate WHERE job.id = candidate.id RETURNING job.* """ HEARTBEAT_JOB_SQL = """ UPDATE rag.background_jobs AS job SET lease_until = now() + make_interval(secs => CAST(:lease_seconds AS integer)), updated_at = now() WHERE job.id = :job_id AND job.status = 'RUNNING' AND job.lease_owner = :worker_id AND job.lease_token = :lease_token AND job.lease_until >= now() RETURNING job.id, job.lease_until """ COMPLETE_JOB_SQL = """ UPDATE rag.background_jobs AS job SET status = 'SUCCEEDED', progress = 100, lease_owner = NULL, lease_token = NULL, lease_until = NULL, finished_at = now(), updated_at = now() WHERE job.id = :job_id AND job.status = 'RUNNING' AND job.lease_owner = :worker_id AND job.lease_token = :lease_token AND job.lease_until >= now() RETURNING job.* """ FAIL_OR_RETRY_JOB_SQL = """ UPDATE rag.background_jobs AS job SET status = CASE WHEN job.attempt < job.max_attempts THEN 'QUEUED' ELSE 'FAILED' END, run_after = CASE WHEN job.attempt < job.max_attempts THEN now() + make_interval( secs => GREATEST(CAST(:retry_delay_seconds AS integer), 0) ) ELSE job.run_after END, last_error_code = :error_code, last_error_message = left(:error_message, 2000), lease_owner = NULL, lease_token = NULL, lease_until = NULL, finished_at = CASE WHEN job.attempt < job.max_attempts THEN NULL ELSE now() END, updated_at = now() WHERE job.id = :job_id AND job.status = 'RUNNING' AND job.lease_owner = :worker_id AND job.lease_token = :lease_token AND job.lease_until >= now() RETURNING job.* """ REAP_EXPIRED_JOBS_SQL = """ WITH maintenance_lock AS ( SELECT pg_try_advisory_xact_lock(CAST(:lock_key AS bigint)) AS acquired ), expired AS ( SELECT job.id FROM rag.background_jobs AS job CROSS JOIN maintenance_lock WHERE maintenance_lock.acquired AND job.status = 'RUNNING' AND job.lease_until < now() ORDER BY job.lease_until, job.created_at FOR UPDATE OF job SKIP LOCKED LIMIT CAST(:batch_size AS integer) ) UPDATE rag.background_jobs AS job SET status = CASE WHEN job.attempt < job.max_attempts THEN 'QUEUED' ELSE 'FAILED' END, run_after = CASE WHEN job.attempt < job.max_attempts THEN now() ELSE job.run_after END, last_error_code = 'WORKER_LEASE_EXPIRED', last_error_message = 'Worker lease expired before a fenced terminal update.', lease_owner = NULL, lease_token = NULL, lease_until = NULL, finished_at = CASE WHEN job.attempt < job.max_attempts THEN NULL ELSE now() END, updated_at = now() FROM expired WHERE job.id = expired.id RETURNING job.* """