refactor: extract ensure_artifact into app/storage/artifacts.py
The artifact-upsert helper was duplicated four times (scanner.py, table_processor.py, figure_processor.py, pipeline.py) with slightly different signatures. Consolidates into a single keyword-only function keyed on (document_id, storage_key) - the identity the schema already enforces - so re-running the pipeline never creates duplicate rows. scanner / table_processor / figure_processor now import the shared helper directly. pipeline.py keeps a thin local wrapper to preserve the positional call sites at three artifact upsert points (OCR_PDF, MARKDOWN, DOCLING_JSON). Tests: 24 passed (5 health + 19 original). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,9 +7,10 @@ import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.models import ArtifactType, DocumentArtifact, Table
|
||||
from app.db.models import ArtifactType, Table
|
||||
from app.ingestion.docling_extractor import ExtractedTable
|
||||
from app.logging_config import get_logger
|
||||
from app.storage.artifacts import ensure_artifact
|
||||
from app.storage.local_paths import key_table_json
|
||||
from app.storage.minio_client import MinioStorage
|
||||
|
||||
@@ -52,7 +53,14 @@ def persist_tables(
|
||||
data=json.dumps(t.json_data, ensure_ascii=False).encode("utf-8"),
|
||||
content_type="application/json",
|
||||
)
|
||||
_ensure_artifact(db, document_id, ArtifactType.TABLE_JSON, storage.derived_bucket, key, t.page_number)
|
||||
ensure_artifact(
|
||||
db,
|
||||
document_id=document_id,
|
||||
artifact_type=ArtifactType.TABLE_JSON,
|
||||
bucket=storage.derived_bucket,
|
||||
key=key,
|
||||
page_number=t.page_number,
|
||||
)
|
||||
|
||||
count += 1
|
||||
return count
|
||||
@@ -62,23 +70,3 @@ def _summary(t: ExtractedTable) -> str:
|
||||
md = t.markdown or ""
|
||||
n_rows = max(0, sum(1 for ln in md.splitlines() if ln.startswith("|")) - 2)
|
||||
return f"Table {t.table_index} on page {t.page_number} ({n_rows} rows)."
|
||||
|
||||
|
||||
def _ensure_artifact(db, document_id: uuid.UUID, artifact_type: str, bucket: str, key: str, page: int | None) -> None:
|
||||
existing = db.execute(
|
||||
select(DocumentArtifact).where(
|
||||
DocumentArtifact.document_id == document_id,
|
||||
DocumentArtifact.storage_key == key,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
return
|
||||
db.add(
|
||||
DocumentArtifact(
|
||||
document_id=document_id,
|
||||
artifact_type=artifact_type,
|
||||
storage_bucket=bucket,
|
||||
storage_key=key,
|
||||
page_number=page,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user