feat: align LegacyHUB with TeamHUB platform contract (D2/D4, assets, security)
Close 12 audit-driven platform-compliance gaps on a single branch. - D4 dispatch: app/integrations/dispatch_client.py participant `legacyhub`, emits LegacyhubDocumentIndexed + AssetDerivativeReady after the indexing commit (idempotent uuid5), http_inbox route (reindex/tombstone) with audit-based dedupe; docs/dispatch-contract.md. Celery+Redis stays intra-module. - D2 SSO: app/integrations/identity.py validates X-TeamHub-* + role/scope mapper; security.py adds trusted-header enforcement (AUTH_REQUIRE_IDENTITY) and a scope check on /search; docker-compose.teamhub.yml (external teamhub_net + internal db net, api not host-published); RUNBOOK network/firewall section. - Asset standard: SearchHit/Citation carry asset_id/owner_module; buckets renamed teamhub-legacyhub-* (+quarantine/tmp/exports); purge-by-asset_id with legal-hold guard (app/indexing/projection.py); OCR-markdown derivative event. - audit_log model + Alembic 0003 + record_audit on writes (same transaction). - Secret masking: app/common/json_logger.py recursive mask wired into structlog (+ensure_ascii=False); event payloads redacted before persistence. - Service X-API-Key mandatory on ingest endpoints (defence-in-depth). - Port: host API 8000->8050 (collision with SalesHUB/MailHUB resolved), container still listens on 8000. - Config: no plaintext secret defaults; fail-loud in non-dev (no value leak). - Docs drift: README PG 5440, layered-auth note, 5173 removed from CORS; ingest/folder gated by ENABLE_FOLDER_INGEST (410 by default). - ADRs: layers mapping, shared-core extraction, UI locale (RU-first). Tests: 78 passing (ruff, compileall, pytest, tsc, vite build, compose config). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -23,7 +23,9 @@ from app.api.schemas import (
|
||||
KnowledgeIngestOptions,
|
||||
KnowledgeIngestResponse,
|
||||
)
|
||||
from app.common.json_logger import redact_secrets
|
||||
from app.config import settings
|
||||
from app.db.audit import record_audit
|
||||
from app.db.models import (
|
||||
ArtifactType,
|
||||
AssetIngestJob,
|
||||
@@ -177,6 +179,22 @@ class SqlAlchemyKnowledgeIngestRepository:
|
||||
)
|
||||
)
|
||||
|
||||
record_audit(
|
||||
db,
|
||||
action="asset.ingest.registered",
|
||||
actor=manifest.asset.created_by,
|
||||
app_code=manifest.asset.owner_module,
|
||||
entity_type="document",
|
||||
entity_id=str(document_id),
|
||||
details={
|
||||
"asset_id": manifest.asset.asset_id,
|
||||
"job_id": str(job_id),
|
||||
"sha256": manifest.asset.sha256.lower(),
|
||||
"owner_record_type": manifest.asset.owner_record_type,
|
||||
"owner_record_id": manifest.asset.owner_record_id,
|
||||
},
|
||||
)
|
||||
|
||||
return KnowledgeIngestRegistration(
|
||||
job_id=job_id,
|
||||
document_id=document_id,
|
||||
@@ -305,6 +323,23 @@ def load_asset_metadata_for_document(document_id: uuid.UUID) -> dict[str, object
|
||||
return _metadata_from_manifest(job.manifest_json)
|
||||
|
||||
|
||||
def load_asset_ingest_options(document_id: uuid.UUID) -> dict[str, object]:
|
||||
"""Return the latest asset ingest job's options (e.g. ``emit_events``)."""
|
||||
with session_scope() as db:
|
||||
job = (
|
||||
db.execute(
|
||||
select(AssetIngestJob)
|
||||
.where(AssetIngestJob.document_id == document_id)
|
||||
.order_by(AssetIngestJob.created_at.desc())
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
if job is None:
|
||||
return {}
|
||||
return dict(job.ingest_options or {})
|
||||
|
||||
|
||||
def _identity_query(manifest: AssetManifest):
|
||||
return select(AssetIngestJob).where(
|
||||
AssetIngestJob.asset_id == manifest.asset.asset_id,
|
||||
@@ -347,17 +382,21 @@ def _original_filename(manifest: AssetManifest) -> str:
|
||||
|
||||
|
||||
def _event_data(manifest: AssetManifest, idempotency_key: str | None) -> dict[str, object]:
|
||||
return {
|
||||
"asset_id": manifest.asset.asset_id,
|
||||
"manifest_version": manifest.manifest_version,
|
||||
"owner_module": manifest.asset.owner_module,
|
||||
"owner_record_type": manifest.asset.owner_record_type,
|
||||
"owner_record_id": manifest.asset.owner_record_id,
|
||||
"idempotency_key": idempotency_key,
|
||||
"storage": manifest.storage.model_dump(mode="json"),
|
||||
"security": manifest.security.model_dump(mode="json"),
|
||||
"retention": manifest.retention.model_dump(mode="json"),
|
||||
}
|
||||
# redact_secrets masks any secret-bearing field (e.g. storage.kms_key_id)
|
||||
# before the payload is persisted to processing_events.data.
|
||||
return redact_secrets(
|
||||
{
|
||||
"asset_id": manifest.asset.asset_id,
|
||||
"manifest_version": manifest.manifest_version,
|
||||
"owner_module": manifest.asset.owner_module,
|
||||
"owner_record_type": manifest.asset.owner_record_type,
|
||||
"owner_record_id": manifest.asset.owner_record_id,
|
||||
"idempotency_key": idempotency_key,
|
||||
"storage": manifest.storage.model_dump(mode="json"),
|
||||
"security": manifest.security.model_dump(mode="json"),
|
||||
"retention": manifest.retention.model_dump(mode="json"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _metadata_from_manifest(manifest: dict[str, object]) -> dict[str, object]:
|
||||
@@ -378,6 +417,7 @@ def _metadata_from_manifest(manifest: dict[str, object]) -> dict[str, object]:
|
||||
"owner_module": asset.get("owner_module"),
|
||||
"owner_record_type": asset.get("owner_record_type"),
|
||||
"owner_record_id": asset.get("owner_record_id"),
|
||||
"sha256": asset.get("sha256"),
|
||||
"manifest_version": manifest.get("manifest_version"),
|
||||
"retention": retention,
|
||||
"legal_hold": retention.get("legal_hold"),
|
||||
|
||||
@@ -216,6 +216,14 @@ def process_document_id( # noqa: PLR0911, PLR0912, PLR0915
|
||||
)
|
||||
)
|
||||
|
||||
# Inter-module event (D4) - after commit, best-effort; never fails the doc.
|
||||
try:
|
||||
from app.integrations.dispatch_client import emit_after_indexing # noqa: PLC0415
|
||||
|
||||
emit_after_indexing(document_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("pipeline.emit_failed", document_id=str(document_id), error=str(exc))
|
||||
|
||||
return {"status": DocumentStatus.INDEXING_COMPLETED, "chunks": len(chunk_records)}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user