2806aaf441
The folder scanner commits each document's row inside a per-document session_scope that only finalizes when the generator advances past its yield. In inline mode scripts/ingest_folder.py invoked the pipeline (which opens a fresh session) before that commit landed, so process_document_id saw "document_missing" and the pipeline aborted at STORED_ORIGINAL. Draining the generator with list() forces every scanner commit before any pipeline run. Verified end to end: a generated PDF now progresses DISCOVERED -> ... -> CHUNKING_COMPLETED and is searchable. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
68 lines
2.4 KiB
Python
68 lines
2.4 KiB
Python
"""Synchronous CLI ingestion: discover -> queue -> process inline.
|
|
|
|
Use ``--async`` to push tasks to Celery instead of running inline (default
|
|
inline mode is convenient for ad-hoc runs without a worker container).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import click
|
|
|
|
from app.ingestion.scanner import discover_documents
|
|
from app.logging_config import configure_logging, get_logger
|
|
|
|
configure_logging()
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
@click.command()
|
|
@click.option("--path", required=True, type=click.Path(exists=True, file_okay=True, dir_okay=True, path_type=Path))
|
|
@click.option("--recursive/--no-recursive", default=True)
|
|
@click.option("--force", is_flag=True, default=False, help="Re-process even if SHA already exists")
|
|
@click.option("--mode", type=click.Choice(["inline", "celery"]), default="inline")
|
|
def main(path: Path, recursive: bool, force: bool, mode: str) -> None:
|
|
run_id = uuid.uuid4()
|
|
discovered = queued = dups = invalid = 0
|
|
|
|
# Materialize generator first: scanner's per-document session_scope only
|
|
# commits when the generator advances past the yield. Inline mode invokes
|
|
# the pipeline (which opens a fresh session) before that commit happens,
|
|
# so the document row is invisible. List() drains the generator end to end,
|
|
# forcing all commits before any processing.
|
|
records = list(discover_documents(path, recursive=recursive, force=force))
|
|
|
|
for record in records:
|
|
discovered += 1
|
|
if record.duplicate and not force:
|
|
dups += 1
|
|
continue
|
|
if not record.document_id:
|
|
invalid += 1
|
|
continue
|
|
|
|
if mode == "celery":
|
|
from app.workers.tasks import process_document
|
|
process_document.delay(str(record.document_id), str(run_id))
|
|
else:
|
|
from app.ingestion.pipeline import process_document_id
|
|
try:
|
|
result = process_document_id(record.document_id, run_id)
|
|
logger.info("ingest.cli.processed", path=str(record.path), result=result)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.exception("ingest.cli.failed", path=str(record.path), error=str(exc))
|
|
invalid += 1
|
|
continue
|
|
queued += 1
|
|
|
|
click.echo(
|
|
f"discovered={discovered} queued={queued} duplicates={dups} invalid={invalid} run={run_id}"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(standalone_mode=True) or 0)
|