feat: align LegacyHUB with TeamHUB platform contract (D2/D4, assets, security)
Close 12 audit-driven platform-compliance gaps on a single branch. - D4 dispatch: app/integrations/dispatch_client.py participant `legacyhub`, emits LegacyhubDocumentIndexed + AssetDerivativeReady after the indexing commit (idempotent uuid5), http_inbox route (reindex/tombstone) with audit-based dedupe; docs/dispatch-contract.md. Celery+Redis stays intra-module. - D2 SSO: app/integrations/identity.py validates X-TeamHub-* + role/scope mapper; security.py adds trusted-header enforcement (AUTH_REQUIRE_IDENTITY) and a scope check on /search; docker-compose.teamhub.yml (external teamhub_net + internal db net, api not host-published); RUNBOOK network/firewall section. - Asset standard: SearchHit/Citation carry asset_id/owner_module; buckets renamed teamhub-legacyhub-* (+quarantine/tmp/exports); purge-by-asset_id with legal-hold guard (app/indexing/projection.py); OCR-markdown derivative event. - audit_log model + Alembic 0003 + record_audit on writes (same transaction). - Secret masking: app/common/json_logger.py recursive mask wired into structlog (+ensure_ascii=False); event payloads redacted before persistence. - Service X-API-Key mandatory on ingest endpoints (defence-in-depth). - Port: host API 8000->8050 (collision with SalesHUB/MailHUB resolved), container still listens on 8000. - Config: no plaintext secret defaults; fail-loud in non-dev (no value leak). - Docs drift: README PG 5440, layered-auth note, 5173 removed from CORS; ingest/folder gated by ENABLE_FOLDER_INGEST (410 by default). - ADRs: layers mapping, shared-core extraction, UI locale (RU-first). Tests: 78 passing (ruff, compileall, pytest, tsc, vite build, compose config). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
103
app/indexing/projection.py
Normal file
103
app/indexing/projection.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Purge LegacyHUB search projections by ``asset_id`` (10 §6 / §8.4).
|
||||
|
||||
When an owner module deletes an asset (tombstone event) LegacyHUB removes its
|
||||
projection - chunks plus OpenSearch and Qdrant entries - for that asset's
|
||||
documents. A legal hold blocks the purge (10 §6: legal_hold overrides deletion
|
||||
and applies to index chunks/embeddings).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.db.audit import record_audit
|
||||
from app.db.models import AssetIngestJob, Chunk
|
||||
from app.db.session import session_scope
|
||||
from app.indexing import opensearch_client, qdrant_client
|
||||
from app.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssetPurgeTarget:
|
||||
found: bool
|
||||
document_ids: list[str] = field(default_factory=list)
|
||||
legal_hold: bool = False
|
||||
|
||||
|
||||
class AssetProjectionRepository(Protocol):
|
||||
def find_asset_documents(self, asset_id: str) -> AssetPurgeTarget: ...
|
||||
|
||||
def purge_chunks(self, asset_id: str, document_ids: list[str]) -> None: ...
|
||||
|
||||
|
||||
class SqlAlchemyAssetProjectionRepository:
|
||||
def find_asset_documents(self, asset_id: str) -> AssetPurgeTarget:
|
||||
with session_scope() as db:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(AssetIngestJob)
|
||||
.where(AssetIngestJob.asset_id == asset_id)
|
||||
.order_by(AssetIngestJob.created_at.desc())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not jobs:
|
||||
return AssetPurgeTarget(found=False)
|
||||
legal_hold = bool((jobs[0].manifest_json or {}).get("retention", {}).get("legal_hold"))
|
||||
document_ids = sorted({str(j.document_id) for j in jobs})
|
||||
return AssetPurgeTarget(found=True, document_ids=document_ids, legal_hold=legal_hold)
|
||||
|
||||
def purge_chunks(self, asset_id: str, document_ids: list[str]) -> None:
|
||||
with session_scope() as db:
|
||||
for did in document_ids:
|
||||
db.execute(delete(Chunk).where(Chunk.document_id == uuid.UUID(did)))
|
||||
record_audit(
|
||||
db,
|
||||
action="asset.projection.purged",
|
||||
entity_type="asset",
|
||||
entity_id=asset_id,
|
||||
details={"documents": document_ids},
|
||||
)
|
||||
|
||||
|
||||
def _default_search_purge(document_ids: list[str]) -> None:
|
||||
for did in document_ids:
|
||||
try:
|
||||
opensearch_client.delete_by_document(did)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("projection.os_purge_failed", document_id=did, error=str(exc))
|
||||
try:
|
||||
qdrant_client.delete_by_document(did)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("projection.qdrant_purge_failed", document_id=did, error=str(exc))
|
||||
|
||||
|
||||
def purge_projection_by_asset_id(
|
||||
asset_id: str,
|
||||
*,
|
||||
repository: AssetProjectionRepository | None = None,
|
||||
search_purge=None,
|
||||
) -> dict[str, object]:
|
||||
"""Remove the search projection for ``asset_id``. Blocked by a legal hold."""
|
||||
repo = repository or SqlAlchemyAssetProjectionRepository()
|
||||
target = repo.find_asset_documents(asset_id)
|
||||
if not target.found:
|
||||
return {"asset_id": asset_id, "purged": False, "reason": "unknown_asset"}
|
||||
if target.legal_hold:
|
||||
logger.info("projection.purge_blocked_legal_hold", asset_id=asset_id)
|
||||
return {"asset_id": asset_id, "purged": False, "reason": "legal_hold"}
|
||||
|
||||
repo.purge_chunks(asset_id, target.document_ids)
|
||||
(search_purge or _default_search_purge)(target.document_ids)
|
||||
logger.info("projection.purged", asset_id=asset_id, documents=len(target.document_ids))
|
||||
return {"asset_id": asset_id, "purged": True, "documents": target.document_ids}
|
||||
|
||||
|
||||
__all__ = ["AssetPurgeTarget", "purge_projection_by_asset_id"]
|
||||
Reference in New Issue
Block a user