feat: align LegacyHUB with TeamHUB platform contract (D2/D4, assets, security)

Close 12 audit-driven platform-compliance gaps on a single branch.

- D4 dispatch: app/integrations/dispatch_client.py participant `legacyhub`,
  emits LegacyhubDocumentIndexed + AssetDerivativeReady after the indexing
  commit (idempotent uuid5), http_inbox route (reindex/tombstone) with
  audit-based dedupe; docs/dispatch-contract.md. Celery+Redis stays intra-module.
- D2 SSO: app/integrations/identity.py validates X-TeamHub-* + role/scope
  mapper; security.py adds trusted-header enforcement (AUTH_REQUIRE_IDENTITY)
  and a scope check on /search; docker-compose.teamhub.yml (external teamhub_net
  + internal db net, api not host-published); RUNBOOK network/firewall section.
- Asset standard: SearchHit/Citation carry asset_id/owner_module; buckets
  renamed teamhub-legacyhub-* (+quarantine/tmp/exports); purge-by-asset_id with
  legal-hold guard (app/indexing/projection.py); OCR-markdown derivative event.
- audit_log model + Alembic 0003 + record_audit on writes (same transaction).
- Secret masking: app/common/json_logger.py recursive mask wired into structlog
  (+ensure_ascii=False); event payloads redacted before persistence.
- Service X-API-Key mandatory on ingest endpoints (defence-in-depth).
- Port: host API 8000->8050 (collision with SalesHUB/MailHUB resolved),
  container still listens on 8000.
- Config: no plaintext secret defaults; fail-loud in non-dev (no value leak).
- Docs drift: README PG 5440, layered-auth note, 5173 removed from CORS;
  ingest/folder gated by ENABLE_FOLDER_INGEST (410 by default).
- ADRs: layers mapping, shared-core extraction, UI locale (RU-first).

Tests: 78 passing (ruff, compileall, pytest, tsc, vite build, compose config).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Vadim Malanov
2026-06-15 11:44:15 +03:00
parent 9af557c26a
commit d27dd0ffbb
45 changed files with 2149 additions and 61 deletions

View File

@@ -23,7 +23,9 @@ from app.api.schemas import (
KnowledgeIngestOptions,
KnowledgeIngestResponse,
)
from app.common.json_logger import redact_secrets
from app.config import settings
from app.db.audit import record_audit
from app.db.models import (
ArtifactType,
AssetIngestJob,
@@ -177,6 +179,22 @@ class SqlAlchemyKnowledgeIngestRepository:
)
)
record_audit(
db,
action="asset.ingest.registered",
actor=manifest.asset.created_by,
app_code=manifest.asset.owner_module,
entity_type="document",
entity_id=str(document_id),
details={
"asset_id": manifest.asset.asset_id,
"job_id": str(job_id),
"sha256": manifest.asset.sha256.lower(),
"owner_record_type": manifest.asset.owner_record_type,
"owner_record_id": manifest.asset.owner_record_id,
},
)
return KnowledgeIngestRegistration(
job_id=job_id,
document_id=document_id,
@@ -305,6 +323,23 @@ def load_asset_metadata_for_document(document_id: uuid.UUID) -> dict[str, object
return _metadata_from_manifest(job.manifest_json)
def load_asset_ingest_options(document_id: uuid.UUID) -> dict[str, object]:
"""Return the latest asset ingest job's options (e.g. ``emit_events``)."""
with session_scope() as db:
job = (
db.execute(
select(AssetIngestJob)
.where(AssetIngestJob.document_id == document_id)
.order_by(AssetIngestJob.created_at.desc())
)
.scalars()
.first()
)
if job is None:
return {}
return dict(job.ingest_options or {})
def _identity_query(manifest: AssetManifest):
return select(AssetIngestJob).where(
AssetIngestJob.asset_id == manifest.asset.asset_id,
@@ -347,17 +382,21 @@ def _original_filename(manifest: AssetManifest) -> str:
def _event_data(manifest: AssetManifest, idempotency_key: str | None) -> dict[str, object]:
return {
"asset_id": manifest.asset.asset_id,
"manifest_version": manifest.manifest_version,
"owner_module": manifest.asset.owner_module,
"owner_record_type": manifest.asset.owner_record_type,
"owner_record_id": manifest.asset.owner_record_id,
"idempotency_key": idempotency_key,
"storage": manifest.storage.model_dump(mode="json"),
"security": manifest.security.model_dump(mode="json"),
"retention": manifest.retention.model_dump(mode="json"),
}
# redact_secrets masks any secret-bearing field (e.g. storage.kms_key_id)
# before the payload is persisted to processing_events.data.
return redact_secrets(
{
"asset_id": manifest.asset.asset_id,
"manifest_version": manifest.manifest_version,
"owner_module": manifest.asset.owner_module,
"owner_record_type": manifest.asset.owner_record_type,
"owner_record_id": manifest.asset.owner_record_id,
"idempotency_key": idempotency_key,
"storage": manifest.storage.model_dump(mode="json"),
"security": manifest.security.model_dump(mode="json"),
"retention": manifest.retention.model_dump(mode="json"),
}
)
def _metadata_from_manifest(manifest: dict[str, object]) -> dict[str, object]:
@@ -378,6 +417,7 @@ def _metadata_from_manifest(manifest: dict[str, object]) -> dict[str, object]:
"owner_module": asset.get("owner_module"),
"owner_record_type": asset.get("owner_record_type"),
"owner_record_id": asset.get("owner_record_id"),
"sha256": asset.get("sha256"),
"manifest_version": manifest.get("manifest_version"),
"retention": retention,
"legal_hold": retention.get("legal_hold"),

View File

@@ -216,6 +216,14 @@ def process_document_id( # noqa: PLR0911, PLR0912, PLR0915
)
)
# Inter-module event (D4) - after commit, best-effort; never fails the doc.
try:
from app.integrations.dispatch_client import emit_after_indexing # noqa: PLC0415
emit_after_indexing(document_id)
except Exception as exc: # noqa: BLE001
logger.warning("pipeline.emit_failed", document_id=str(document_id), error=str(exc))
return {"status": DocumentStatus.INDEXING_COMPLETED, "chunks": len(chunk_records)}