YoVinchen
ecdb10c37a
Some checks failed
verify / verify (push) Has been cancelled
Separate local parsing from model indexing, bind review decisions to immutable manifests, persist vectors behind active profiles, and expose retrieval, chat, evaluation, and document workflows through the React workbench. Constraint: Live Bailian authentication currently fails for all three configured capabilities Rejected: Direct upload-to-embedding flow | bypasses local review and manifest binding Confidence: high Scope-risk: broad Directive: Keep private-data deployment blocked until authentication, RBAC, and separate database roles land Tested: make verify; fresh and replay Docker document smoke; worker recovery smoke; frozen synthetic evaluation; migration 0003-0004 roundtrip Not-tested: Successful live Bailian calls, OCR, real multi-user authorization
1112 lines
40 KiB
Python
1112 lines
40 KiB
Python
"""Deterministic, fail-closed local document-ingestion primitives.
|
|
|
|
This module deliberately has no persistence, HTTP, provider, or third-party parser
|
|
dependency. It turns a bounded in-memory upload into traceable local-review
|
|
artifacts. PDF contents are not interpreted without a reliable parser; valid PDF
|
|
uploads are routed to an explicit OCR/parser-required state instead.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import bisect
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import re
|
|
import stat
|
|
import uuid
|
|
import zipfile
|
|
import zlib
|
|
from dataclasses import dataclass
|
|
from enum import StrEnum
|
|
from pathlib import PurePosixPath
|
|
from typing import Final
|
|
from xml.etree import ElementTree
|
|
|
|
DEFAULT_MAX_UPLOAD_BYTES: Final = 100 * 1024 * 1024
|
|
DOCX_MAX_ENTRIES: Final = 512
|
|
DOCX_MAX_ENTRY_BYTES: Final = 16 * 1024 * 1024
|
|
DOCX_MAX_TOTAL_UNCOMPRESSED_BYTES: Final = 48 * 1024 * 1024
|
|
DOCX_MAX_COMPRESSION_RATIO: Final = 200
|
|
DOCX_MAX_XML_BYTES: Final = 12 * 1024 * 1024
|
|
|
|
_ARTIFACT_NAMESPACE: Final = uuid.UUID("1cf190a2-bf5e-52b0-af34-b632ac7d31d5")
|
|
_WORD_NAMESPACE: Final = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
|
|
_CONTENT_TYPES_NAMESPACE: Final = "http://schemas.openxmlformats.org/package/2006/content-types"
|
|
_WORD_DOCUMENT_CONTENT_TYPE: Final = (
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"
|
|
)
|
|
_DOCX_MIME: Final = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
|
|
_TOKEN_PATTERN = re.compile(
|
|
r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]"
|
|
r"|[A-Za-z][A-Za-z0-9]*(?:[-_][A-Za-z0-9]+)*"
|
|
r"|\d+(?:\.\d+)?"
|
|
r"|[^\s]"
|
|
)
|
|
_ATX_HEADING = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$")
|
|
_SETEXT_HEADING = re.compile(r"^[ \t]*(=+|-+)[ \t]*$")
|
|
_FENCE = re.compile(r"^[ \t]*(`{3,}|~{3,})")
|
|
_HEADING_STYLE = re.compile(r"^heading[ _-]?([1-6])$", re.IGNORECASE)
|
|
_SAFE_POLICY_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,79}$")
|
|
_SECRET_PATTERNS: Final = (
|
|
re.compile(r"(?i)\bsk-[A-Za-z0-9_-]{16,}\b"),
|
|
re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]{16,}=*"),
|
|
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
|
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
|
re.compile(
|
|
r"(?i)\b(?:api[_-]?key|access[_-]?token|secret[_-]?key)\s*[:=]\s*"
|
|
r"[A-Za-z0-9._~+/-]{16,}=*"
|
|
),
|
|
)
|
|
|
|
|
|
class DocumentFormat(StrEnum):
|
|
TEXT = "TEXT"
|
|
MARKDOWN = "MARKDOWN"
|
|
DOCX = "DOCX"
|
|
PDF = "PDF"
|
|
|
|
|
|
class IngestionStatus(StrEnum):
|
|
READY_FOR_LOCAL_REVIEW = "READY_FOR_LOCAL_REVIEW"
|
|
OCR_REQUIRED = "OCR_REQUIRED"
|
|
|
|
|
|
class BlockKind(StrEnum):
|
|
HEADING = "HEADING"
|
|
PARAGRAPH = "PARAGRAPH"
|
|
TABLE_ROW = "TABLE_ROW"
|
|
|
|
|
|
class IngestionErrorCode(StrEnum):
|
|
EMPTY_FILE = "EMPTY_FILE"
|
|
FILE_TOO_LARGE = "FILE_TOO_LARGE"
|
|
INVALID_FILENAME = "INVALID_FILENAME"
|
|
UNSUPPORTED_MEDIA_TYPE = "UNSUPPORTED_MEDIA_TYPE"
|
|
MIME_EXTENSION_MISMATCH = "MIME_EXTENSION_MISMATCH"
|
|
MIME_CONTENT_MISMATCH = "MIME_CONTENT_MISMATCH"
|
|
INVALID_TEXT_ENCODING = "INVALID_TEXT_ENCODING"
|
|
EMPTY_DOCUMENT_TEXT = "EMPTY_DOCUMENT_TEXT"
|
|
INVALID_DOCX_PACKAGE = "INVALID_DOCX_PACKAGE"
|
|
DOCX_PATH_TRAVERSAL = "DOCX_PATH_TRAVERSAL"
|
|
DOCX_ENCRYPTED = "DOCX_ENCRYPTED"
|
|
DOCX_PACKAGE_LIMIT = "DOCX_PACKAGE_LIMIT"
|
|
DOCX_ACTIVE_CONTENT = "DOCX_ACTIVE_CONTENT"
|
|
DOCX_UNSAFE_XML = "DOCX_UNSAFE_XML"
|
|
SENSITIVE_CONTENT_DETECTED = "SENSITIVE_CONTENT_DETECTED"
|
|
INVALID_INGESTION_CONFIGURATION = "INVALID_INGESTION_CONFIGURATION"
|
|
|
|
|
|
_SAFE_ERROR_MESSAGES: Final[dict[IngestionErrorCode, str]] = {
|
|
IngestionErrorCode.EMPTY_FILE: "The uploaded file is empty.",
|
|
IngestionErrorCode.FILE_TOO_LARGE: "The uploaded file exceeds the configured size limit.",
|
|
IngestionErrorCode.INVALID_FILENAME: "The uploaded filename is not safe.",
|
|
IngestionErrorCode.UNSUPPORTED_MEDIA_TYPE: "The uploaded document type is not supported.",
|
|
IngestionErrorCode.MIME_EXTENSION_MISMATCH: (
|
|
"The declared media type does not match the filename extension."
|
|
),
|
|
IngestionErrorCode.MIME_CONTENT_MISMATCH: (
|
|
"The uploaded bytes do not match the declared document type."
|
|
),
|
|
IngestionErrorCode.INVALID_TEXT_ENCODING: "The text encoding is not supported or is invalid.",
|
|
IngestionErrorCode.EMPTY_DOCUMENT_TEXT: "The document does not contain reviewable text.",
|
|
IngestionErrorCode.INVALID_DOCX_PACKAGE: "The DOCX package is malformed or incomplete.",
|
|
IngestionErrorCode.DOCX_PATH_TRAVERSAL: "The DOCX package contains an unsafe entry path.",
|
|
IngestionErrorCode.DOCX_ENCRYPTED: "Encrypted DOCX packages are not accepted.",
|
|
IngestionErrorCode.DOCX_PACKAGE_LIMIT: "The DOCX package exceeds a safety limit.",
|
|
IngestionErrorCode.DOCX_ACTIVE_CONTENT: "Active or embedded DOCX content is not accepted.",
|
|
IngestionErrorCode.DOCX_UNSAFE_XML: "The DOCX package contains unsafe XML declarations.",
|
|
IngestionErrorCode.SENSITIVE_CONTENT_DETECTED: (
|
|
"The upload contains a credential-like value and was rejected."
|
|
),
|
|
IngestionErrorCode.INVALID_INGESTION_CONFIGURATION: (
|
|
"The document-ingestion configuration is invalid."
|
|
),
|
|
}
|
|
|
|
|
|
class DocumentIngestionError(ValueError):
|
|
"""A bounded error that never incorporates user-controlled values."""
|
|
|
|
def __init__(self, code: IngestionErrorCode) -> None:
|
|
self.code = code
|
|
super().__init__(_SAFE_ERROR_MESSAGES[code])
|
|
|
|
def __repr__(self) -> str:
|
|
return f"{type(self).__name__}(code={self.code.value!r})"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ChunkingConfig:
|
|
"""Token-window contract.
|
|
|
|
A chunk normally ends at ``target_tokens``. It may extend to the next
|
|
structural block boundary, but never past ``max_tokens``. Every subsequent
|
|
window replays exactly ``overlap_tokens`` when enough prior tokens exist.
|
|
"""
|
|
|
|
target_tokens: int = 512
|
|
max_tokens: int = 800
|
|
overlap_tokens: int = 64
|
|
|
|
def __post_init__(self) -> None:
|
|
if not (
|
|
self.target_tokens > 0
|
|
and self.max_tokens >= self.target_tokens
|
|
and 0 <= self.overlap_tokens < self.target_tokens
|
|
):
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_INGESTION_CONFIGURATION)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CloudTextPolicy:
|
|
"""Deterministic literal redactions applied only to outbound text."""
|
|
|
|
policy_id: str = "local-review-v1"
|
|
redact_literals: tuple[str, ...] = ()
|
|
replacement: str = "[REDACTED]"
|
|
|
|
def __post_init__(self) -> None:
|
|
if (
|
|
not _SAFE_POLICY_ID.fullmatch(self.policy_id)
|
|
or not self.replacement
|
|
or any(not value for value in self.redact_literals)
|
|
or _contains_secret(self.policy_id)
|
|
or _contains_secret(self.replacement)
|
|
):
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_INGESTION_CONFIGURATION)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SourceAnchor:
|
|
anchor_id: str
|
|
normalized_text_sha256: str
|
|
char_start: int
|
|
char_end: int
|
|
line_start: int
|
|
line_end: int
|
|
page_start: int | None
|
|
page_end: int | None
|
|
block_ids: tuple[str, ...]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ParsedPage:
|
|
page_id: str
|
|
ordinal: int
|
|
page_number: int | None
|
|
text: str
|
|
text_sha256: str
|
|
line_start: int
|
|
line_end: int
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ParsedBlock:
|
|
block_id: str
|
|
ordinal: int
|
|
kind: BlockKind
|
|
text: str
|
|
text_sha256: str
|
|
section_path: tuple[str, ...]
|
|
anchor: SourceAnchor
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PreparedChunk:
|
|
chunk_id: str
|
|
ordinal: int
|
|
display_text: str
|
|
cloud_text: str
|
|
embedding_prefix: str
|
|
embedding_text: str
|
|
display_text_sha256: str
|
|
cloud_text_sha256: str
|
|
embedding_text_sha256: str
|
|
token_count: int
|
|
section_path: tuple[str, ...]
|
|
anchor: SourceAnchor
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ManifestItem:
|
|
ordinal: int
|
|
chunk_id: str
|
|
cloud_text_sha256: str
|
|
embedding_text_sha256: str
|
|
anchor_id: str
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class OutboundManifest:
|
|
manifest_sha256: str
|
|
raw_sha256: str
|
|
normalized_text_sha256: str
|
|
parser_profile_hash: str
|
|
chunk_profile_hash: str
|
|
cloud_policy_id: str
|
|
cloud_policy_hash: str
|
|
items: tuple[ManifestItem, ...]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class IngestionArtifact:
|
|
status: IngestionStatus
|
|
document_format: DocumentFormat
|
|
media_type: str
|
|
raw_sha256: str
|
|
document_version_id: str
|
|
parser_profile_hash: str
|
|
chunk_profile_hash: str
|
|
normalized_text_sha256: str | None
|
|
pages: tuple[ParsedPage, ...]
|
|
blocks: tuple[ParsedBlock, ...]
|
|
chunks: tuple[PreparedChunk, ...]
|
|
manifest: OutboundManifest | None
|
|
limitations: tuple[str, ...] = ()
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _DraftBlock:
|
|
kind: BlockKind
|
|
text: str
|
|
section_path: tuple[str, ...]
|
|
page_number: int | None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _TokenSpan:
|
|
start: int
|
|
end: int
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _ParsedText:
|
|
encoding: str
|
|
blocks: tuple[_DraftBlock, ...]
|
|
|
|
|
|
def ingest_document(
|
|
*,
|
|
filename: str,
|
|
declared_mime_type: str,
|
|
content: bytes,
|
|
max_upload_bytes: int = DEFAULT_MAX_UPLOAD_BYTES,
|
|
chunking: ChunkingConfig | None = None,
|
|
cloud_policy: CloudTextPolicy | None = None,
|
|
) -> IngestionArtifact:
|
|
"""Validate, parse, chunk, and manifest one bounded local document.
|
|
|
|
User-controlled filenames and content are intentionally absent from errors.
|
|
The returned artifact is deterministic for identical bytes and configuration.
|
|
"""
|
|
|
|
effective_chunking = chunking or ChunkingConfig()
|
|
effective_policy = cloud_policy or CloudTextPolicy()
|
|
cloud_policy_hash = _cloud_policy_hash(effective_policy)
|
|
_validate_upload_envelope(filename, content, max_upload_bytes)
|
|
document_format, media_type = _resolve_format(filename, declared_mime_type)
|
|
_validate_content_signature(document_format, content)
|
|
raw_sha256 = _sha256_bytes(content)
|
|
|
|
if document_format is DocumentFormat.PDF:
|
|
parser_profile_hash = _profile_hash(
|
|
{"parser": "pdf-fail-closed", "version": 1, "mode": "ocr_required"}
|
|
)
|
|
chunk_profile_hash = _chunk_profile_hash(effective_chunking)
|
|
version_id = _stable_uuid(
|
|
"document-version",
|
|
raw_sha256,
|
|
parser_profile_hash,
|
|
chunk_profile_hash,
|
|
cloud_policy_hash,
|
|
)
|
|
return IngestionArtifact(
|
|
status=IngestionStatus.OCR_REQUIRED,
|
|
document_format=document_format,
|
|
media_type=media_type,
|
|
raw_sha256=raw_sha256,
|
|
document_version_id=version_id,
|
|
parser_profile_hash=parser_profile_hash,
|
|
chunk_profile_hash=chunk_profile_hash,
|
|
normalized_text_sha256=None,
|
|
pages=(),
|
|
blocks=(),
|
|
chunks=(),
|
|
manifest=None,
|
|
limitations=(
|
|
"PDF_PARSER_UNAVAILABLE",
|
|
"NO_MAP_OR_SPATIAL_UNDERSTANDING",
|
|
),
|
|
)
|
|
|
|
parsed = _parse(document_format, content)
|
|
if not parsed.blocks:
|
|
raise DocumentIngestionError(IngestionErrorCode.EMPTY_DOCUMENT_TEXT)
|
|
|
|
parser_profile_hash = _parser_profile_hash(document_format, parsed.encoding)
|
|
normalized_text, blocks = _materialize_blocks(
|
|
parsed.blocks,
|
|
raw_sha256=raw_sha256,
|
|
parser_profile_hash=parser_profile_hash,
|
|
)
|
|
_reject_secrets(filename, normalized_text)
|
|
normalized_text_sha256 = _sha256_text(normalized_text)
|
|
pages = _materialize_pages(blocks)
|
|
chunk_profile_hash = _chunk_profile_hash(effective_chunking)
|
|
version_id = _stable_uuid(
|
|
"document-version",
|
|
raw_sha256,
|
|
parser_profile_hash,
|
|
chunk_profile_hash,
|
|
cloud_policy_hash,
|
|
)
|
|
chunks = _build_chunks(
|
|
normalized_text,
|
|
blocks,
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
document_version_id=version_id,
|
|
config=effective_chunking,
|
|
policy=effective_policy,
|
|
)
|
|
if not chunks:
|
|
raise DocumentIngestionError(IngestionErrorCode.EMPTY_DOCUMENT_TEXT)
|
|
manifest = _build_manifest(
|
|
raw_sha256=raw_sha256,
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
parser_profile_hash=parser_profile_hash,
|
|
chunk_profile_hash=chunk_profile_hash,
|
|
policy=effective_policy,
|
|
cloud_policy_hash=cloud_policy_hash,
|
|
chunks=chunks,
|
|
)
|
|
return IngestionArtifact(
|
|
status=IngestionStatus.READY_FOR_LOCAL_REVIEW,
|
|
document_format=document_format,
|
|
media_type=media_type,
|
|
raw_sha256=raw_sha256,
|
|
document_version_id=version_id,
|
|
parser_profile_hash=parser_profile_hash,
|
|
chunk_profile_hash=chunk_profile_hash,
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
pages=pages,
|
|
blocks=blocks,
|
|
chunks=chunks,
|
|
manifest=manifest,
|
|
)
|
|
|
|
|
|
def _validate_upload_envelope(filename: str, content: bytes, max_upload_bytes: int) -> None:
|
|
if max_upload_bytes <= 0:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_INGESTION_CONFIGURATION)
|
|
if not filename or "\x00" in filename or "/" in filename or "\\" in filename:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_FILENAME)
|
|
if _contains_secret(filename):
|
|
raise DocumentIngestionError(IngestionErrorCode.SENSITIVE_CONTENT_DETECTED)
|
|
if not content:
|
|
raise DocumentIngestionError(IngestionErrorCode.EMPTY_FILE)
|
|
if len(content) > max_upload_bytes:
|
|
raise DocumentIngestionError(IngestionErrorCode.FILE_TOO_LARGE)
|
|
|
|
|
|
def _resolve_format(filename: str, declared_mime_type: str) -> tuple[DocumentFormat, str]:
|
|
suffix = PurePosixPath(filename).suffix.lower()
|
|
mime = declared_mime_type.split(";", maxsplit=1)[0].strip().lower()
|
|
allowed: dict[str, tuple[DocumentFormat, frozenset[str], str]] = {
|
|
".txt": (DocumentFormat.TEXT, frozenset({"text/plain"}), "text/plain"),
|
|
".md": (
|
|
DocumentFormat.MARKDOWN,
|
|
frozenset({"text/markdown", "text/plain"}),
|
|
"text/markdown",
|
|
),
|
|
".markdown": (
|
|
DocumentFormat.MARKDOWN,
|
|
frozenset({"text/markdown", "text/plain"}),
|
|
"text/markdown",
|
|
),
|
|
".docx": (DocumentFormat.DOCX, frozenset({_DOCX_MIME}), _DOCX_MIME),
|
|
".pdf": (DocumentFormat.PDF, frozenset({"application/pdf"}), "application/pdf"),
|
|
}
|
|
item = allowed.get(suffix)
|
|
if item is None:
|
|
raise DocumentIngestionError(IngestionErrorCode.UNSUPPORTED_MEDIA_TYPE)
|
|
document_format, accepted_mimes, canonical_mime = item
|
|
if mime not in accepted_mimes:
|
|
raise DocumentIngestionError(IngestionErrorCode.MIME_EXTENSION_MISMATCH)
|
|
return document_format, canonical_mime
|
|
|
|
|
|
def _validate_content_signature(document_format: DocumentFormat, content: bytes) -> None:
|
|
if document_format is DocumentFormat.PDF:
|
|
if not content.startswith(b"%PDF-"):
|
|
raise DocumentIngestionError(IngestionErrorCode.MIME_CONTENT_MISMATCH)
|
|
return
|
|
if document_format is DocumentFormat.DOCX:
|
|
if not content.startswith(b"PK"):
|
|
raise DocumentIngestionError(IngestionErrorCode.MIME_CONTENT_MISMATCH)
|
|
return
|
|
if content.startswith((b"%PDF-", b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")):
|
|
raise DocumentIngestionError(IngestionErrorCode.MIME_CONTENT_MISMATCH)
|
|
|
|
|
|
def _parse(document_format: DocumentFormat, content: bytes) -> _ParsedText:
|
|
if document_format is DocumentFormat.DOCX:
|
|
return _parse_docx(content)
|
|
text, encoding = _decode_text(content)
|
|
blocks = (
|
|
_parse_markdown_blocks(text)
|
|
if document_format is DocumentFormat.MARKDOWN
|
|
else _parse_plain_text_blocks(text)
|
|
)
|
|
return _ParsedText(encoding=encoding, blocks=blocks)
|
|
|
|
|
|
def _decode_text(content: bytes) -> tuple[str, str]:
|
|
try:
|
|
if content.startswith(b"\xef\xbb\xbf"):
|
|
text = content.decode("utf-8-sig", errors="strict")
|
|
encoding = "utf-8-sig"
|
|
elif content.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
text = content.decode("utf-16", errors="strict")
|
|
encoding = "utf-16"
|
|
else:
|
|
text = content.decode("utf-8", errors="strict")
|
|
encoding = "utf-8"
|
|
except UnicodeDecodeError:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_TEXT_ENCODING) from None
|
|
if "\x00" in text:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_TEXT_ENCODING)
|
|
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
if any(ord(character) < 32 and character not in "\t\n\f" for character in normalized):
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_TEXT_ENCODING)
|
|
if not normalized.strip():
|
|
raise DocumentIngestionError(IngestionErrorCode.EMPTY_DOCUMENT_TEXT)
|
|
return normalized, encoding
|
|
|
|
|
|
def _parse_plain_text_blocks(text: str) -> tuple[_DraftBlock, ...]:
|
|
blocks: list[_DraftBlock] = []
|
|
for page_number, page in enumerate(text.split("\f"), start=1):
|
|
paragraph: list[str] = []
|
|
for line in page.splitlines():
|
|
if line.strip():
|
|
paragraph.append(line.rstrip())
|
|
else:
|
|
_append_paragraph(blocks, paragraph, (), page_number)
|
|
_append_paragraph(blocks, paragraph, (), page_number)
|
|
return tuple(blocks)
|
|
|
|
|
|
def _parse_markdown_blocks(text: str) -> tuple[_DraftBlock, ...]:
|
|
blocks: list[_DraftBlock] = []
|
|
section_levels: list[str | None] = [None] * 6
|
|
fence_marker: str | None = None
|
|
|
|
for page_number, page in enumerate(text.split("\f"), start=1):
|
|
paragraph: list[str] = []
|
|
|
|
def current_section() -> tuple[str, ...]:
|
|
return tuple(value for value in section_levels if value is not None)
|
|
|
|
lines = page.splitlines()
|
|
for line in lines:
|
|
fence = _FENCE.match(line)
|
|
if fence:
|
|
marker = fence.group(1)[0]
|
|
if fence_marker is None:
|
|
fence_marker = marker
|
|
elif marker == fence_marker:
|
|
fence_marker = None
|
|
paragraph.append(line.rstrip())
|
|
continue
|
|
|
|
heading = _ATX_HEADING.match(line) if fence_marker is None else None
|
|
if heading:
|
|
_append_paragraph(blocks, paragraph, current_section(), page_number)
|
|
level = len(heading.group(1))
|
|
title = heading.group(2).strip()
|
|
if title:
|
|
section_levels[level - 1] = title
|
|
for index in range(level, len(section_levels)):
|
|
section_levels[index] = None
|
|
blocks.append(
|
|
_DraftBlock(
|
|
BlockKind.HEADING,
|
|
title,
|
|
current_section(),
|
|
page_number,
|
|
)
|
|
)
|
|
continue
|
|
|
|
setext = _SETEXT_HEADING.match(line) if fence_marker is None else None
|
|
if setext and len(paragraph) == 1:
|
|
title = paragraph.pop().strip()
|
|
if title:
|
|
level = 1 if setext.group(1).startswith("=") else 2
|
|
section_levels[level - 1] = title
|
|
for index in range(level, len(section_levels)):
|
|
section_levels[index] = None
|
|
blocks.append(
|
|
_DraftBlock(
|
|
BlockKind.HEADING,
|
|
title,
|
|
current_section(),
|
|
page_number,
|
|
)
|
|
)
|
|
continue
|
|
|
|
if line.strip():
|
|
paragraph.append(line.rstrip())
|
|
else:
|
|
_append_paragraph(blocks, paragraph, current_section(), page_number)
|
|
_append_paragraph(blocks, paragraph, current_section(), page_number)
|
|
return tuple(blocks)
|
|
|
|
|
|
def _append_paragraph(
|
|
blocks: list[_DraftBlock],
|
|
paragraph: list[str],
|
|
section_path: tuple[str, ...],
|
|
page_number: int,
|
|
) -> None:
|
|
value = "\n".join(paragraph).strip()
|
|
paragraph.clear()
|
|
if value:
|
|
blocks.append(_DraftBlock(BlockKind.PARAGRAPH, value, section_path, page_number))
|
|
|
|
|
|
def _parse_docx(content: bytes) -> _ParsedText:
|
|
try:
|
|
with zipfile.ZipFile(io.BytesIO(content), mode="r") as package:
|
|
infos = package.infolist()
|
|
_validate_docx_entries(infos)
|
|
names = {info.filename for info in infos}
|
|
required = {"[Content_Types].xml", "word/document.xml"}
|
|
if not required <= names:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE)
|
|
content_types = _read_bounded_zip_entry(
|
|
package, "[Content_Types].xml", DOCX_MAX_XML_BYTES
|
|
)
|
|
document_xml = _read_bounded_zip_entry(package, "word/document.xml", DOCX_MAX_XML_BYTES)
|
|
except DocumentIngestionError:
|
|
raise
|
|
except (zipfile.BadZipFile, RuntimeError, OSError, EOFError, zlib.error):
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE) from None
|
|
|
|
_reject_unsafe_xml(content_types)
|
|
_reject_unsafe_xml(document_xml)
|
|
try:
|
|
# Both inputs are bounded and DTD/entity declarations were rejected above.
|
|
content_root = ElementTree.fromstring(content_types) # noqa: S314
|
|
document_root = ElementTree.fromstring(document_xml) # noqa: S314
|
|
except ElementTree.ParseError:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE) from None
|
|
|
|
if not _has_word_document_content_type(content_root):
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE)
|
|
body = document_root.find(f".//{{{_WORD_NAMESPACE}}}body")
|
|
if body is None:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE)
|
|
|
|
blocks: list[_DraftBlock] = []
|
|
section_levels: list[str | None] = [None] * 6
|
|
|
|
def current_section() -> tuple[str, ...]:
|
|
return tuple(value for value in section_levels if value is not None)
|
|
|
|
for child in body:
|
|
if child.tag == f"{{{_WORD_NAMESPACE}}}p":
|
|
text = _docx_paragraph_text(child).strip()
|
|
if not text:
|
|
continue
|
|
heading_level = _docx_heading_level(child)
|
|
if heading_level is not None:
|
|
section_levels[heading_level - 1] = text
|
|
for index in range(heading_level, len(section_levels)):
|
|
section_levels[index] = None
|
|
kind = BlockKind.HEADING
|
|
else:
|
|
kind = BlockKind.PARAGRAPH
|
|
blocks.append(_DraftBlock(kind, text, current_section(), None))
|
|
elif child.tag == f"{{{_WORD_NAMESPACE}}}tbl":
|
|
for row in child.findall(f".//{{{_WORD_NAMESPACE}}}tr"):
|
|
cells: list[str] = []
|
|
for cell in row.findall(f"./{{{_WORD_NAMESPACE}}}tc"):
|
|
paragraphs = [
|
|
_docx_paragraph_text(paragraph).strip()
|
|
for paragraph in cell.findall(f".//{{{_WORD_NAMESPACE}}}p")
|
|
]
|
|
cells.append(" ".join(value for value in paragraphs if value))
|
|
row_text = "\t".join(cells).strip()
|
|
if row_text:
|
|
blocks.append(
|
|
_DraftBlock(
|
|
BlockKind.TABLE_ROW,
|
|
row_text,
|
|
current_section(),
|
|
None,
|
|
)
|
|
)
|
|
return _ParsedText(encoding="docx-xml", blocks=tuple(blocks))
|
|
|
|
|
|
def _validate_docx_entries(infos: list[zipfile.ZipInfo]) -> None:
|
|
if not infos or len(infos) > DOCX_MAX_ENTRIES:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
seen: set[str] = set()
|
|
total_size = 0
|
|
for info in infos:
|
|
name = info.filename
|
|
path = PurePosixPath(name)
|
|
if (
|
|
not name
|
|
or "\x00" in name
|
|
or "\\" in name
|
|
or name.startswith("/")
|
|
or path.is_absolute()
|
|
or ".." in path.parts
|
|
or any(":" in part for part in path.parts)
|
|
):
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PATH_TRAVERSAL)
|
|
if name in seen:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE)
|
|
seen.add(name)
|
|
if info.flag_bits & 0x1:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_ENCRYPTED)
|
|
file_mode = (info.external_attr >> 16) & 0xFFFF
|
|
if stat.S_ISLNK(file_mode):
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PATH_TRAVERSAL)
|
|
lower_name = name.lower()
|
|
if any(marker in lower_name for marker in ("vbaproject.bin", "/activex/", "/embeddings/")):
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_ACTIVE_CONTENT)
|
|
if info.is_dir():
|
|
continue
|
|
if info.file_size > DOCX_MAX_ENTRY_BYTES:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
total_size += info.file_size
|
|
if total_size > DOCX_MAX_TOTAL_UNCOMPRESSED_BYTES:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
if info.file_size > 0:
|
|
if info.compress_size <= 0:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
if info.file_size > info.compress_size * DOCX_MAX_COMPRESSION_RATIO:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
|
|
|
|
def _read_bounded_zip_entry(package: zipfile.ZipFile, name: str, maximum_bytes: int) -> bytes:
|
|
info = package.getinfo(name)
|
|
if info.file_size > maximum_bytes:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_PACKAGE_LIMIT)
|
|
value = package.read(info)
|
|
if len(value) != info.file_size or len(value) > maximum_bytes:
|
|
raise DocumentIngestionError(IngestionErrorCode.INVALID_DOCX_PACKAGE)
|
|
return value
|
|
|
|
|
|
def _reject_unsafe_xml(value: bytes) -> None:
|
|
upper_value = value.upper()
|
|
if b"<!DOCTYPE" in upper_value or b"<!ENTITY" in upper_value:
|
|
raise DocumentIngestionError(IngestionErrorCode.DOCX_UNSAFE_XML)
|
|
|
|
|
|
def _has_word_document_content_type(root: ElementTree.Element) -> bool:
|
|
for override in root.findall(f"{{{_CONTENT_TYPES_NAMESPACE}}}Override"):
|
|
if (
|
|
override.attrib.get("PartName") == "/word/document.xml"
|
|
and override.attrib.get("ContentType") == _WORD_DOCUMENT_CONTENT_TYPE
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _docx_paragraph_text(paragraph: ElementTree.Element) -> str:
|
|
values: list[str] = []
|
|
for element in paragraph.iter():
|
|
if element.tag == f"{{{_WORD_NAMESPACE}}}t":
|
|
values.append(element.text or "")
|
|
elif element.tag == f"{{{_WORD_NAMESPACE}}}tab":
|
|
values.append("\t")
|
|
elif element.tag == f"{{{_WORD_NAMESPACE}}}br":
|
|
values.append("\n")
|
|
return "".join(values)
|
|
|
|
|
|
def _docx_heading_level(paragraph: ElementTree.Element) -> int | None:
|
|
style = paragraph.find(f"./{{{_WORD_NAMESPACE}}}pPr/{{{_WORD_NAMESPACE}}}pStyle")
|
|
if style is None:
|
|
return None
|
|
value = style.attrib.get(f"{{{_WORD_NAMESPACE}}}val", "")
|
|
match = _HEADING_STYLE.fullmatch(value)
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def _materialize_blocks(
|
|
drafts: tuple[_DraftBlock, ...], *, raw_sha256: str, parser_profile_hash: str
|
|
) -> tuple[str, tuple[ParsedBlock, ...]]:
|
|
normalized_text = "\n\n".join(draft.text for draft in drafts)
|
|
normalized_hash = _sha256_text(normalized_text)
|
|
blocks: list[ParsedBlock] = []
|
|
cursor = 0
|
|
for ordinal, draft in enumerate(drafts):
|
|
char_start = cursor
|
|
char_end = char_start + len(draft.text)
|
|
line_start = _line_number(normalized_text, char_start)
|
|
line_end = _line_number(normalized_text, max(char_start, char_end - 1))
|
|
block_id = _stable_uuid(
|
|
"block",
|
|
raw_sha256,
|
|
parser_profile_hash,
|
|
str(ordinal),
|
|
_sha256_text(draft.text),
|
|
)
|
|
anchor = _source_anchor(
|
|
normalized_text_sha256=normalized_hash,
|
|
char_start=char_start,
|
|
char_end=char_end,
|
|
line_start=line_start,
|
|
line_end=line_end,
|
|
page_start=draft.page_number,
|
|
page_end=draft.page_number,
|
|
block_ids=(block_id,),
|
|
)
|
|
blocks.append(
|
|
ParsedBlock(
|
|
block_id=block_id,
|
|
ordinal=ordinal,
|
|
kind=draft.kind,
|
|
text=draft.text,
|
|
text_sha256=_sha256_text(draft.text),
|
|
section_path=draft.section_path,
|
|
anchor=anchor,
|
|
)
|
|
)
|
|
cursor = char_end + 2
|
|
return normalized_text, tuple(blocks)
|
|
|
|
|
|
def _materialize_pages(blocks: tuple[ParsedBlock, ...]) -> tuple[ParsedPage, ...]:
|
|
normalized_text_sha256 = blocks[0].anchor.normalized_text_sha256
|
|
page_numbers = sorted(
|
|
{block.anchor.page_start for block in blocks if block.anchor.page_start is not None}
|
|
)
|
|
if not page_numbers:
|
|
text = "\n\n".join(block.text for block in blocks)
|
|
return (
|
|
ParsedPage(
|
|
page_id=_stable_uuid(
|
|
"page", normalized_text_sha256, "unpaginated", _sha256_text(text)
|
|
),
|
|
ordinal=0,
|
|
page_number=None,
|
|
text=text,
|
|
text_sha256=_sha256_text(text),
|
|
line_start=1,
|
|
line_end=max(1, text.count("\n") + 1),
|
|
),
|
|
)
|
|
pages: list[ParsedPage] = []
|
|
for ordinal, page_number in enumerate(page_numbers):
|
|
page_blocks = [block for block in blocks if block.anchor.page_start == page_number]
|
|
text = "\n\n".join(block.text for block in page_blocks)
|
|
pages.append(
|
|
ParsedPage(
|
|
page_id=_stable_uuid(
|
|
"page", normalized_text_sha256, str(page_number), _sha256_text(text)
|
|
),
|
|
ordinal=ordinal,
|
|
page_number=page_number,
|
|
text=text,
|
|
text_sha256=_sha256_text(text),
|
|
line_start=min(block.anchor.line_start for block in page_blocks),
|
|
line_end=max(block.anchor.line_end for block in page_blocks),
|
|
)
|
|
)
|
|
return tuple(pages)
|
|
|
|
|
|
def _build_chunks(
|
|
normalized_text: str,
|
|
blocks: tuple[ParsedBlock, ...],
|
|
*,
|
|
normalized_text_sha256: str,
|
|
document_version_id: str,
|
|
config: ChunkingConfig,
|
|
policy: CloudTextPolicy,
|
|
) -> tuple[PreparedChunk, ...]:
|
|
tokens = tuple(
|
|
_TokenSpan(match.start(), match.end()) for match in _TOKEN_PATTERN.finditer(normalized_text)
|
|
)
|
|
if not tokens:
|
|
return ()
|
|
token_ends = [token.end for token in tokens]
|
|
block_boundaries = sorted(
|
|
{
|
|
bisect.bisect_right(token_ends, block.anchor.char_end)
|
|
for block in blocks
|
|
if block.anchor.char_end > 0
|
|
}
|
|
)
|
|
chunks: list[PreparedChunk] = []
|
|
start = 0
|
|
while start < len(tokens):
|
|
target_end = min(start + config.target_tokens, len(tokens))
|
|
max_end = min(start + config.max_tokens, len(tokens))
|
|
end = target_end
|
|
boundary_index = bisect.bisect_left(block_boundaries, target_end)
|
|
if boundary_index < len(block_boundaries):
|
|
candidate = block_boundaries[boundary_index]
|
|
if start < candidate <= max_end:
|
|
end = candidate
|
|
if end <= start:
|
|
end = min(start + config.target_tokens, len(tokens))
|
|
|
|
char_start = tokens[start].start
|
|
char_end = tokens[end - 1].end
|
|
while char_start < char_end and normalized_text[char_start].isspace():
|
|
char_start += 1
|
|
while char_end > char_start and normalized_text[char_end - 1].isspace():
|
|
char_end -= 1
|
|
display_text = normalized_text[char_start:char_end]
|
|
overlapping = tuple(
|
|
block
|
|
for block in blocks
|
|
if block.anchor.char_end > char_start and block.anchor.char_start < char_end
|
|
)
|
|
block_ids = tuple(block.block_id for block in overlapping)
|
|
page_values = [
|
|
block.anchor.page_start for block in overlapping if block.anchor.page_start is not None
|
|
]
|
|
page_is_complete = len(page_values) == len(overlapping)
|
|
section_path = _common_section_path(tuple(block.section_path for block in overlapping))
|
|
anchor = _source_anchor(
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
char_start=char_start,
|
|
char_end=char_end,
|
|
line_start=_line_number(normalized_text, char_start),
|
|
line_end=_line_number(normalized_text, max(char_start, char_end - 1)),
|
|
page_start=min(page_values) if page_values and page_is_complete else None,
|
|
page_end=max(page_values) if page_values and page_is_complete else None,
|
|
block_ids=block_ids,
|
|
)
|
|
cloud_text = _apply_cloud_policy(display_text, policy)
|
|
embedding_prefix = _embedding_prefix(section_path)
|
|
embedding_text = embedding_prefix + cloud_text
|
|
_reject_secrets(display_text, cloud_text, embedding_text)
|
|
ordinal = len(chunks)
|
|
display_hash = _sha256_text(display_text)
|
|
chunk_id = _stable_uuid(
|
|
"chunk", document_version_id, str(ordinal), display_hash, anchor.anchor_id
|
|
)
|
|
chunks.append(
|
|
PreparedChunk(
|
|
chunk_id=chunk_id,
|
|
ordinal=ordinal,
|
|
display_text=display_text,
|
|
cloud_text=cloud_text,
|
|
embedding_prefix=embedding_prefix,
|
|
embedding_text=embedding_text,
|
|
display_text_sha256=display_hash,
|
|
cloud_text_sha256=_sha256_text(cloud_text),
|
|
embedding_text_sha256=_sha256_text(embedding_text),
|
|
token_count=end - start,
|
|
section_path=section_path,
|
|
anchor=anchor,
|
|
)
|
|
)
|
|
if end == len(tokens):
|
|
break
|
|
start = max(start + 1, end - config.overlap_tokens)
|
|
return tuple(chunks)
|
|
|
|
|
|
def _build_manifest(
|
|
*,
|
|
raw_sha256: str,
|
|
normalized_text_sha256: str,
|
|
parser_profile_hash: str,
|
|
chunk_profile_hash: str,
|
|
policy: CloudTextPolicy,
|
|
cloud_policy_hash: str,
|
|
chunks: tuple[PreparedChunk, ...],
|
|
) -> OutboundManifest:
|
|
items = tuple(
|
|
ManifestItem(
|
|
ordinal=chunk.ordinal,
|
|
chunk_id=chunk.chunk_id,
|
|
cloud_text_sha256=chunk.cloud_text_sha256,
|
|
embedding_text_sha256=chunk.embedding_text_sha256,
|
|
anchor_id=chunk.anchor.anchor_id,
|
|
)
|
|
for chunk in chunks
|
|
)
|
|
payload = {
|
|
"schema_version": 1,
|
|
"raw_sha256": raw_sha256,
|
|
"normalized_text_sha256": normalized_text_sha256,
|
|
"parser_profile_hash": parser_profile_hash,
|
|
"chunk_profile_hash": chunk_profile_hash,
|
|
"cloud_policy_id": policy.policy_id,
|
|
"cloud_policy_hash": cloud_policy_hash,
|
|
"items": [
|
|
{
|
|
"ordinal": item.ordinal,
|
|
"chunk_id": item.chunk_id,
|
|
"cloud_text_sha256": item.cloud_text_sha256,
|
|
"embedding_text_sha256": item.embedding_text_sha256,
|
|
"anchor_id": item.anchor_id,
|
|
}
|
|
for item in items
|
|
],
|
|
}
|
|
return OutboundManifest(
|
|
manifest_sha256=_profile_hash(payload),
|
|
raw_sha256=raw_sha256,
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
parser_profile_hash=parser_profile_hash,
|
|
chunk_profile_hash=chunk_profile_hash,
|
|
cloud_policy_id=policy.policy_id,
|
|
cloud_policy_hash=cloud_policy_hash,
|
|
items=items,
|
|
)
|
|
|
|
|
|
def _source_anchor(
|
|
*,
|
|
normalized_text_sha256: str,
|
|
char_start: int,
|
|
char_end: int,
|
|
line_start: int,
|
|
line_end: int,
|
|
page_start: int | None,
|
|
page_end: int | None,
|
|
block_ids: tuple[str, ...],
|
|
) -> SourceAnchor:
|
|
payload = {
|
|
"normalized_text_sha256": normalized_text_sha256,
|
|
"char_start": char_start,
|
|
"char_end": char_end,
|
|
"line_start": line_start,
|
|
"line_end": line_end,
|
|
"page_start": page_start,
|
|
"page_end": page_end,
|
|
"block_ids": block_ids,
|
|
}
|
|
return SourceAnchor(
|
|
anchor_id=_profile_hash(payload),
|
|
normalized_text_sha256=normalized_text_sha256,
|
|
char_start=char_start,
|
|
char_end=char_end,
|
|
line_start=line_start,
|
|
line_end=line_end,
|
|
page_start=page_start,
|
|
page_end=page_end,
|
|
block_ids=block_ids,
|
|
)
|
|
|
|
|
|
def _parser_profile_hash(document_format: DocumentFormat, encoding: str) -> str:
|
|
payload: dict[str, object] = {
|
|
"service": "document-ingestion",
|
|
"version": 1,
|
|
"format": document_format.value,
|
|
"encoding": encoding,
|
|
}
|
|
if document_format is DocumentFormat.DOCX:
|
|
payload["zip_limits"] = {
|
|
"entries": DOCX_MAX_ENTRIES,
|
|
"entry_bytes": DOCX_MAX_ENTRY_BYTES,
|
|
"total_bytes": DOCX_MAX_TOTAL_UNCOMPRESSED_BYTES,
|
|
"compression_ratio": DOCX_MAX_COMPRESSION_RATIO,
|
|
"xml_bytes": DOCX_MAX_XML_BYTES,
|
|
}
|
|
return _profile_hash(payload)
|
|
|
|
|
|
def _chunk_profile_hash(config: ChunkingConfig) -> str:
|
|
return _profile_hash(
|
|
{
|
|
"chunker": "deterministic-token-window-v1",
|
|
"target_tokens": config.target_tokens,
|
|
"max_tokens": config.max_tokens,
|
|
"overlap_tokens": config.overlap_tokens,
|
|
"token_pattern_version": 1,
|
|
}
|
|
)
|
|
|
|
|
|
def _cloud_policy_hash(policy: CloudTextPolicy) -> str:
|
|
return _profile_hash(
|
|
{
|
|
"policy_id": policy.policy_id,
|
|
"redact_literals_sha256": [
|
|
_sha256_text(value) for value in sorted(set(policy.redact_literals))
|
|
],
|
|
"replacement_sha256": _sha256_text(policy.replacement),
|
|
}
|
|
)
|
|
|
|
|
|
def _profile_hash(value: object) -> str:
|
|
encoded = json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return _sha256_bytes(encoded)
|
|
|
|
|
|
def _stable_uuid(*parts: str) -> str:
|
|
return str(uuid.uuid5(_ARTIFACT_NAMESPACE, "\x1f".join(parts)))
|
|
|
|
|
|
def _sha256_bytes(value: bytes) -> str:
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def _sha256_text(value: str) -> str:
|
|
return _sha256_bytes(value.encode("utf-8"))
|
|
|
|
|
|
def _line_number(text: str, position: int) -> int:
|
|
return text.count("\n", 0, position) + 1
|
|
|
|
|
|
def _common_section_path(paths: tuple[tuple[str, ...], ...]) -> tuple[str, ...]:
|
|
if not paths:
|
|
return ()
|
|
common_length = len(paths[0])
|
|
for path in paths[1:]:
|
|
common_length = min(common_length, len(path))
|
|
for index in range(common_length):
|
|
if paths[0][index] != path[index]:
|
|
common_length = index
|
|
break
|
|
if common_length == 0:
|
|
break
|
|
return paths[0][:common_length]
|
|
|
|
|
|
def _apply_cloud_policy(text: str, policy: CloudTextPolicy) -> str:
|
|
value = text
|
|
for literal in sorted(set(policy.redact_literals), key=lambda item: (-len(item), item)):
|
|
value = value.replace(literal, policy.replacement)
|
|
return value
|
|
|
|
|
|
def _embedding_prefix(section_path: tuple[str, ...]) -> str:
|
|
section = " / ".join(section_path) if section_path else "UNSPECIFIED"
|
|
return f"SECTION: {section}\nCONTENT: "
|
|
|
|
|
|
def _contains_secret(value: str) -> bool:
|
|
return any(pattern.search(value) is not None for pattern in _SECRET_PATTERNS)
|
|
|
|
|
|
def _reject_secrets(*values: str) -> None:
|
|
if any(_contains_secret(value) for value in values):
|
|
raise DocumentIngestionError(IngestionErrorCode.SENSITIVE_CONTENT_DETECTED)
|