fix(pipeline): tolerate duplicate ORIGINAL_PDF artifacts in document lookup
Some checks failed
CI / Backend (lint + tests + compose) (push) Has been cancelled
CI / Frontend (lint + type-check + build) (push) Has been cancelled

Repeated knowledge-ingest of identical content (same sha256, new
asset_id -> new canonical object key) reuses the Document row but
appends a second ORIGINAL_PDF artifact, because ensure_artifact identity
is (document_id, storage_key). process_document_id then crashed with
MultipleResultsFound on scalar_one_or_none and Celery retried forever.
New latest_artifact helper picks the newest row deterministically
(created_at DESC, id DESC) — every duplicate references byte-identical
objects (sha256 verified at ingest), so the latest reference is always
safe.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Vadim Malanov
2026-07-11 20:14:52 +03:00
parent 576662ebe3
commit fe346d983b
3 changed files with 126 additions and 8 deletions

View File

@@ -20,7 +20,6 @@ from app.db.models import (
ArtifactType,
Chunk,
Document,
DocumentArtifact,
DocumentStatus,
Page,
ProcessingEvent,
@@ -35,7 +34,7 @@ from app.ingestion.knowledge_ingest import load_asset_metadata_for_document
from app.ingestion.ocr import run_ocr
from app.ingestion.table_processor import persist_tables
from app.logging_config import get_logger
from app.storage.artifacts import ensure_artifact
from app.storage.artifacts import ensure_artifact, latest_artifact
from app.storage.local_paths import (
key_docling_json,
key_markdown,
@@ -65,12 +64,9 @@ def process_document_id( # noqa: PLR0911, PLR0912, PLR0915
source_path = Path(doc.source_path)
sha = doc.sha256
original_artifact = db.execute(
select(DocumentArtifact).where(
DocumentArtifact.document_id == doc.id,
DocumentArtifact.artifact_type == ArtifactType.ORIGINAL_PDF,
)
).scalar_one_or_none()
original_artifact = latest_artifact(
db, document_id=doc.id, artifact_type=ArtifactType.ORIGINAL_PDF
)
work_dir = work_dir_for(document_id)
local_pdf = work_dir / f"{sha}.pdf"

View File

@@ -51,3 +51,32 @@ def ensure_artifact(
)
db.add(artifact)
return artifact
def latest_artifact(
db: Session,
*,
document_id: uuid.UUID,
artifact_type: str,
) -> DocumentArtifact | None:
"""Newest artifact of the given type, or ``None``.
Duplicates of one type are legal: repeated knowledge-ingest of identical
content (same sha256, new asset_id → new canonical object key) appends a
second ORIGINAL_PDF row because ensure_artifact identity is
(document_id, storage_key). All such rows reference byte-identical objects
(sha256 verified at ingest), so the newest reference is always a safe,
deterministic pick — never scalar_one_or_none here (G6 MultipleResultsFound).
"""
return (
db.execute(
select(DocumentArtifact)
.where(
DocumentArtifact.document_id == document_id,
DocumentArtifact.artifact_type == artifact_type,
)
.order_by(DocumentArtifact.created_at.desc(), DocumentArtifact.id.desc())
)
.scalars()
.first()
)