From 2806aaf441ed626c1a2ee765304cbcbab8fd19e7 Mon Sep 17 00:00:00 2001 From: Vadim Malanov Date: Sun, 14 Jun 2026 13:41:53 +0300 Subject: [PATCH] fix(ingestion): materialize scanner generator before inline processing The folder scanner commits each document's row inside a per-document session_scope that only finalizes when the generator advances past its yield. In inline mode scripts/ingest_folder.py invoked the pipeline (which opens a fresh session) before that commit landed, so process_document_id saw "document_missing" and the pipeline aborted at STORED_ORIGINAL. Draining the generator with list() forces every scanner commit before any pipeline run. Verified end to end: a generated PDF now progresses DISCOVERED -> ... -> CHUNKING_COMPLETED and is searchable. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/ingest_folder.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/ingest_folder.py b/scripts/ingest_folder.py index 220e7df..09f43a5 100644 --- a/scripts/ingest_folder.py +++ b/scripts/ingest_folder.py @@ -28,7 +28,14 @@ def main(path: Path, recursive: bool, force: bool, mode: str) -> None: run_id = uuid.uuid4() discovered = queued = dups = invalid = 0 - for record in discover_documents(path, recursive=recursive, force=force): + # Materialize generator first: scanner's per-document session_scope only + # commits when the generator advances past the yield. Inline mode invokes + # the pipeline (which opens a fresh session) before that commit happens, + # so the document row is invisible. List() drains the generator end to end, + # forcing all commits before any processing. + records = list(discover_documents(path, recursive=recursive, force=force)) + + for record in records: discovered += 1 if record.duplicate and not force: dups += 1