chore: bootstrap repository with governance docs

Initialize git, add Apache-2.0 LICENSE, .gitattributes (LF line
endings), AGENTS.md (entry points, stack, discovery order, baseline
checks), RUNBOOK.md (dev boot, prod deploy with overlay, ingestion,
failures, rollback, scaling notes), .env.prod.example with rotated
credential placeholders, and dev-only warnings on .env.example.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

app/utils/hashing.py

@@ -0,0 +1,21 @@
+"""Streaming SHA256 hashing utilities for large files."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+_CHUNK = 1024 * 1024  # 1 MiB
+
+
+def sha256_file(path: Path | str) -> str:
+    """Compute SHA256 of a file in streaming mode (constant memory)."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(_CHUNK), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()

app/utils/language.py

@@ -0,0 +1,24 @@
+"""Language detection helper - tolerant to short / mixed text."""
+
+from __future__ import annotations
+
+from langdetect import DetectorFactory, LangDetectException, detect_langs
+
+DetectorFactory.seed = 42
+
+
+def detect_language(text: str, min_chars: int = 40) -> str | None:
+    """Return ISO 639-1 language code or ``None`` if undetectable."""
+    if not text or len(text.strip()) < min_chars:
+        return None
+    try:
+        ranked = detect_langs(text)
+    except LangDetectException:
+        return None
+    if not ranked:
+        return None
+    return ranked[0].lang
+
+
+def has_cyrillic(text: str) -> bool:
+    return any("Ѐ" <= ch <= "ӿ" for ch in text)

app/utils/pdf.py

@@ -0,0 +1,36 @@
+"""PDF inspection helpers - decide whether OCR is required."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pikepdf
+from pdfminer.high_level import extract_text
+
+
+def page_count(path: Path | str) -> int:
+    with pikepdf.open(str(path)) as pdf:
+        return len(pdf.pages)
+
+
+def has_searchable_text(path: Path | str, sample_pages: int = 3, min_chars: int = 80) -> bool:
+    """Cheap check: extract text from first ``sample_pages`` and require ``min_chars``.
+
+    Returns False on any extraction error - safer to OCR than to skip.
+    """
+    try:
+        text = extract_text(str(path), maxpages=sample_pages) or ""
+    except Exception:
+        return False
+    return len(text.strip()) >= min_chars
+
+
+def is_pdf(path: Path | str) -> bool:
+    p = Path(path)
+    if not p.is_file() or p.suffix.lower() != ".pdf":
+        return False
+    try:
+        with open(p, "rb") as f:
+            return f.read(5) == b"%PDF-"
+    except OSError:
+        return False

app/utils/text_cleaning.py

@@ -0,0 +1,69 @@
+"""Conservative OCR text cleaning.
+
+Goals:
+- Drop hyphenation across line breaks (``инвен-\\nтарный`` -> ``инвентарный``).
+- Collapse runs of whitespace.
+- Strip control chars.
+- Preserve all non-letter characters that may carry meaning in legacy/technical
+  documents: digits, punctuation, slashes, dashes, dots, parentheses, etc.
+
+We do NOT lowercase, transliterate, or strip punctuation here. ``normalize_for_search``
+produces a more aggressive form for indexing, but the original ``text`` is always
+kept untouched for citation/display.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_CONTROL_CHARS = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]")
+_SOFT_HYPHEN = "­"
+_MULTI_WS = re.compile(r"[ \t ]+")
+_MULTI_NL = re.compile(r"\n{3,}")
+_HYPHEN_LINEBREAK = re.compile(r"(\w)[-‐‑‒–]\n(\w)")
+_TRAILING_WS = re.compile(r"[ \t]+\n")
+
+
+def clean_ocr_text(text: str) -> str:
+    if not text:
+        return ""
+    # Normalize unicode (NFC) to merge combining marks.
+    text = unicodedata.normalize("NFC", text)
+    text = text.replace(_SOFT_HYPHEN, "")
+    text = _CONTROL_CHARS.sub("", text)
+    text = _HYPHEN_LINEBREAK.sub(r"\1\2", text)
+    text = _TRAILING_WS.sub("\n", text)
+    text = _MULTI_WS.sub(" ", text)
+    text = _MULTI_NL.sub("\n\n", text)
+    return text.strip()
+
+
+_PUNCT_RUN = re.compile(r"[^\w\s/\-.,№#:()\[\]]+", flags=re.UNICODE)
+_WS_RUN = re.compile(r"\s+")
+
+
+def normalize_for_search(text: str) -> str:
+    """Lowercase + light normalization for full-text indexing.
+
+    Preserves digits, alphanumerics, slashes, dashes, dots, commas, ``№``, ``#``,
+    colons and brackets - all of which appear in document/serial/standard codes.
+    """
+    if not text:
+        return ""
+    text = clean_ocr_text(text)
+    text = text.lower()
+    text = _PUNCT_RUN.sub(" ", text)
+    text = _WS_RUN.sub(" ", text)
+    return text.strip()
+
+
+def looks_garbled(text: str, threshold: float = 0.35) -> bool:
+    """Heuristic: ratio of non-alphanumeric, non-whitespace chars."""
+    if not text:
+        return False
+    total = len(text)
+    if total < 20:
+        return False
+    bad = sum(1 for c in text if not (c.isalnum() or c.isspace() or c in ".,;:!?-/()[]№#"))
+    return (bad / total) > threshold