fix(pipeline): tolerate duplicate ORIGINAL_PDF artifacts in document lookup
Repeated knowledge-ingest of identical content (same sha256, new asset_id -> new canonical object key) reuses the Document row but appends a second ORIGINAL_PDF artifact, because ensure_artifact identity is (document_id, storage_key). process_document_id then crashed with MultipleResultsFound on scalar_one_or_none and Celery retried forever. New latest_artifact helper picks the newest row deterministically (created_at DESC, id DESC) — every duplicate references byte-identical objects (sha256 verified at ingest), so the latest reference is always safe. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -20,7 +20,6 @@ from app.db.models import (
|
||||
ArtifactType,
|
||||
Chunk,
|
||||
Document,
|
||||
DocumentArtifact,
|
||||
DocumentStatus,
|
||||
Page,
|
||||
ProcessingEvent,
|
||||
@@ -35,7 +34,7 @@ from app.ingestion.knowledge_ingest import load_asset_metadata_for_document
|
||||
from app.ingestion.ocr import run_ocr
|
||||
from app.ingestion.table_processor import persist_tables
|
||||
from app.logging_config import get_logger
|
||||
from app.storage.artifacts import ensure_artifact
|
||||
from app.storage.artifacts import ensure_artifact, latest_artifact
|
||||
from app.storage.local_paths import (
|
||||
key_docling_json,
|
||||
key_markdown,
|
||||
@@ -65,12 +64,9 @@ def process_document_id( # noqa: PLR0911, PLR0912, PLR0915
|
||||
|
||||
source_path = Path(doc.source_path)
|
||||
sha = doc.sha256
|
||||
original_artifact = db.execute(
|
||||
select(DocumentArtifact).where(
|
||||
DocumentArtifact.document_id == doc.id,
|
||||
DocumentArtifact.artifact_type == ArtifactType.ORIGINAL_PDF,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
original_artifact = latest_artifact(
|
||||
db, document_id=doc.id, artifact_type=ArtifactType.ORIGINAL_PDF
|
||||
)
|
||||
|
||||
work_dir = work_dir_for(document_id)
|
||||
local_pdf = work_dir / f"{sha}.pdf"
|
||||
|
||||
Reference in New Issue
Block a user