438 lines
20 KiB
Python
438 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import NAMESPACE_URL, uuid4, uuid5
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from hub_core.runtime.postgres_store import PostgresPortStore
|
|
from hub_core.runtime.tables import (
|
|
compat_api_consumers,
|
|
compat_api_keys,
|
|
compat_hubs,
|
|
compat_manifests,
|
|
compat_migration_runs,
|
|
compat_widgets,
|
|
runtime_audit_ledger,
|
|
runtime_import_runs,
|
|
runtime_interaction_events,
|
|
)
|
|
|
|
SCHEMA_VERSION = "core-hub.migration.v1"
|
|
COLLECTIONS = (
|
|
"hubs",
|
|
"hubCapabilityManifests",
|
|
"apiConsumers",
|
|
"apiKeys",
|
|
"widgets",
|
|
"interactionEvents",
|
|
"migrationRuns",
|
|
)
|
|
SECRET_FIELDS = {"fullKey", "rawKey", "apiKey", "secret", "token", "authorization"}
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _json(value: Any) -> str:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=_json_default)
|
|
|
|
|
|
def _json_default(value: Any) -> str:
|
|
if isinstance(value, datetime):
|
|
return _timestamp(value)
|
|
raise TypeError(f"cannot encode {type(value).__name__}")
|
|
|
|
|
|
def _timestamp(value: datetime) -> str:
|
|
if value.tzinfo is None:
|
|
value = value.replace(tzinfo=timezone.utc)
|
|
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _datetime(value: Any, default: datetime | None = None) -> datetime:
|
|
if isinstance(value, datetime):
|
|
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
|
if isinstance(value, str) and value:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
return default or _now()
|
|
|
|
|
|
def _records(bundle: dict[str, Any], collection: str) -> list[dict[str, Any]]:
|
|
records = bundle.get("records", {})
|
|
values = records.get(collection, []) if isinstance(records, dict) else []
|
|
return [dict(value) for value in values] if isinstance(values, list) else []
|
|
|
|
|
|
def _contains_secret(value: Any) -> bool:
|
|
if isinstance(value, dict):
|
|
return any(key in SECRET_FIELDS or _contains_secret(child) for key, child in value.items())
|
|
if isinstance(value, list):
|
|
return any(_contains_secret(child) for child in value)
|
|
return False
|
|
|
|
|
|
def _canonical_record(record: dict[str, Any]) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
for key, value in record.items():
|
|
if key in {"createdAt", "updatedAt", "occurredAt", "recordedAt"} and value:
|
|
result[key] = _timestamp(_datetime(value))
|
|
elif isinstance(value, dict):
|
|
result[key] = _canonical_record(value)
|
|
elif isinstance(value, list):
|
|
result[key] = [
|
|
_canonical_record(item) if isinstance(item, dict) else item for item in value
|
|
]
|
|
else:
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def _identity(record: dict[str, Any]) -> str:
|
|
return str(record.get("id") or record.get("slug") or record.get("keyHash") or "")
|
|
|
|
|
|
def content_hashes(bundle: dict[str, Any]) -> dict[str, str]:
|
|
hashes: dict[str, str] = {}
|
|
for collection in COLLECTIONS:
|
|
values = [_canonical_record(value) for value in _records(bundle, collection)]
|
|
values.sort(key=_identity)
|
|
hashes[collection] = hashlib.sha256(_json(values).encode()).hexdigest()
|
|
return hashes
|
|
|
|
|
|
def identity_sets(bundle: dict[str, Any]) -> dict[str, list[str]]:
|
|
return {
|
|
collection: sorted(_identity(value) for value in _records(bundle, collection))
|
|
for collection in COLLECTIONS
|
|
}
|
|
|
|
|
|
def bundle_digest(bundle: dict[str, Any]) -> str:
|
|
value = {key: child for key, child in bundle.items() if key != "bundleSha256"}
|
|
return hashlib.sha256(_json(value).encode()).hexdigest()
|
|
|
|
|
|
def validate_bundle(bundle: dict[str, Any]) -> dict[str, Any]:
|
|
errors: list[str] = []
|
|
warnings: list[str] = []
|
|
unknown = set(bundle) - {
|
|
"schemaVersion",
|
|
"source",
|
|
"sourceRevision",
|
|
"exportedAt",
|
|
"highWaterMark",
|
|
"bundleSha256",
|
|
"records",
|
|
}
|
|
if unknown:
|
|
errors.append(f"unknown bundle fields: {', '.join(sorted(unknown))}")
|
|
if bundle.get("schemaVersion") != SCHEMA_VERSION:
|
|
errors.append(f"schemaVersion must be {SCHEMA_VERSION}")
|
|
if not isinstance(bundle.get("records"), dict):
|
|
errors.append("records must be an object")
|
|
else:
|
|
unknown_collections = set(bundle["records"]) - set(COLLECTIONS)
|
|
if unknown_collections:
|
|
errors.append(f"unknown collections: {', '.join(sorted(unknown_collections))}")
|
|
claimed = bundle.get("bundleSha256")
|
|
calculated = bundle_digest(bundle)
|
|
if claimed and claimed != calculated:
|
|
errors.append("bundleSha256 does not match canonical bundle content")
|
|
|
|
required = {
|
|
"hubs": ("slug", "name"),
|
|
"hubCapabilityManifests": ("id",),
|
|
"apiConsumers": ("name",),
|
|
"apiKeys": ("id", "keyPrefix", "keyHash"),
|
|
"widgets": ("id", "name"),
|
|
"interactionEvents": ("id", "widgetId", "eventType"),
|
|
"migrationRuns": ("id", "source", "schemaVersion", "bundleSha256"),
|
|
}
|
|
for collection in COLLECTIONS:
|
|
seen: set[str] = set()
|
|
for index, record in enumerate(_records(bundle, collection)):
|
|
path = f"records.{collection}[{index}]"
|
|
for field in required[collection]:
|
|
if record.get(field) in (None, ""):
|
|
errors.append(f"{path}.{field} is required")
|
|
identity = _identity(record)
|
|
if identity and identity in seen:
|
|
errors.append(f"duplicate {collection} identity: {identity}")
|
|
seen.add(identity)
|
|
if _contains_secret(record):
|
|
errors.append(f"{path} contains secret-shaped material")
|
|
for record in _records(bundle, "hubCapabilityManifests"):
|
|
if not record.get("hubId") and not record.get("hubSlug"):
|
|
errors.append("manifest must include hubId or hubSlug")
|
|
for record in _records(bundle, "apiConsumers"):
|
|
if not record.get("id") and not record.get("slug"):
|
|
errors.append("API consumer must include id or slug")
|
|
for record in _records(bundle, "apiKeys"):
|
|
if not record.get("apiConsumerId") and not record.get("apiConsumerSlug"):
|
|
errors.append("API key must include apiConsumerId or apiConsumerSlug")
|
|
for record in _records(bundle, "widgets"):
|
|
if not record.get("hubId") and not record.get("hubSlug"):
|
|
errors.append("widget must include hubId or hubSlug")
|
|
|
|
return {
|
|
"ok": not errors,
|
|
"schemaVersion": bundle.get("schemaVersion"),
|
|
"source": bundle.get("source", "unknown"),
|
|
"bundleSha256": calculated,
|
|
"highWaterMark": bundle.get("highWaterMark"),
|
|
"counts": {name: {"input": len(_records(bundle, name))} for name in COLLECTIONS},
|
|
"identitySets": identity_sets(bundle),
|
|
"contentHashes": content_hashes(bundle),
|
|
"errors": errors,
|
|
"warnings": warnings,
|
|
}
|
|
|
|
|
|
async def import_bundle(
|
|
store: PostgresPortStore, bundle: dict[str, Any], *, dry_run: bool = False
|
|
) -> dict[str, Any]:
|
|
report = validate_bundle(bundle)
|
|
report["dryRun"] = dry_run
|
|
report["idempotent"] = False
|
|
report["counts"] = {
|
|
name: {"input": len(_records(bundle, name)), "created": 0, "updated": 0, "skipped": 0}
|
|
for name in COLLECTIONS
|
|
}
|
|
if not report["ok"]:
|
|
return report
|
|
|
|
async with store.sessions() as session:
|
|
prior = (
|
|
await session.execute(
|
|
sa.select(runtime_import_runs).where(
|
|
runtime_import_runs.c.bundle_sha256 == report["bundleSha256"]
|
|
)
|
|
)
|
|
).mappings().first()
|
|
if prior and not dry_run:
|
|
report["idempotent"] = True
|
|
report["migrationRunId"] = prior["id"]
|
|
for name in COLLECTIONS:
|
|
report["counts"][name]["skipped"] = report["counts"][name]["input"]
|
|
return report
|
|
|
|
async with store.sessions.begin() as session:
|
|
hubs = await _maps(session, compat_hubs, "id", "slug")
|
|
await _import_hubs(session, bundle, report, hubs, dry_run)
|
|
hubs = await _maps(session, compat_hubs, "id", "slug") if not dry_run else hubs
|
|
await _import_manifests(session, bundle, report, hubs, dry_run)
|
|
consumers = await _maps(session, compat_api_consumers, "id", "slug")
|
|
await _import_consumers(session, bundle, report, consumers, dry_run)
|
|
consumers = (
|
|
await _maps(session, compat_api_consumers, "id", "slug") if not dry_run else consumers
|
|
)
|
|
await _import_keys(session, bundle, report, consumers, dry_run)
|
|
await _import_widgets(session, bundle, report, hubs, dry_run)
|
|
await _import_events(session, bundle, report, dry_run)
|
|
await _import_source_runs(session, bundle, report, dry_run)
|
|
if dry_run:
|
|
await session.rollback()
|
|
return report
|
|
run_id = str(uuid4())
|
|
await session.execute(
|
|
runtime_import_runs.insert().values(
|
|
id=run_id,
|
|
source=report["source"],
|
|
schema_version=SCHEMA_VERSION,
|
|
bundle_sha256=report["bundleSha256"],
|
|
high_water_mark=report["highWaterMark"],
|
|
counts=report["counts"],
|
|
content_hashes=report["contentHashes"],
|
|
status="imported",
|
|
created_at=_now(),
|
|
)
|
|
)
|
|
await session.execute(
|
|
runtime_audit_ledger.insert().values(
|
|
id=str(uuid4()),
|
|
action="migration.imported",
|
|
subject_type="migration_bundle",
|
|
subject_id=report["bundleSha256"],
|
|
correlation_id=None,
|
|
payload_hash=hashlib.sha256(_json(report["contentHashes"]).encode()).hexdigest(),
|
|
detail={"source": report["source"], "counts": report["counts"]},
|
|
recorded_at=_now(),
|
|
)
|
|
)
|
|
report["migrationRunId"] = run_id
|
|
return report
|
|
|
|
|
|
async def _maps(session: Any, table: sa.Table, *keys: str) -> dict[str, dict[str, Any]]:
|
|
rows = (await session.execute(sa.select(table))).mappings().all()
|
|
return {str(row[key]): dict(row) for row in rows for key in keys if row[key]}
|
|
|
|
|
|
def _mark(report: dict[str, Any], name: str, existing: Any) -> None:
|
|
report["counts"][name]["updated" if existing else "created"] += 1
|
|
|
|
|
|
async def _upsert(session: Any, table: sa.Table, row: dict[str, Any], existing: Any) -> None:
|
|
if existing:
|
|
await session.execute(table.update().where(table.c.id == existing["id"]).values(**row))
|
|
else:
|
|
await session.execute(table.insert().values(**row))
|
|
|
|
|
|
def _parent_id(record: dict[str, Any], rows: dict[str, dict[str, Any]], kind: str) -> str:
|
|
parent = record.get(f"{kind}Id") or record.get(f"{kind}Slug")
|
|
if parent in rows:
|
|
return str(rows[str(parent)]["id"])
|
|
raise ValueError(f"unresolved {kind} reference: {parent}")
|
|
|
|
|
|
async def _import_hubs(session: Any, bundle: dict[str, Any], report: dict, rows: dict, dry: bool) -> None:
|
|
for record in _records(bundle, "hubs"):
|
|
existing = rows.get(str(record.get("id"))) or rows.get(str(record.get("slug")))
|
|
_mark(report, "hubs", existing)
|
|
if dry:
|
|
continue
|
|
now = _now()
|
|
row = {
|
|
"id": existing["id"] if existing else str(record.get("id") or uuid4()),
|
|
"slug": record["slug"], "name": record["name"], "domain": record.get("domain"),
|
|
"hub_kind": record.get("hubKind"), "hub_family": record.get("hubFamily"),
|
|
"vsm_function": record.get("vsmFunction"), "vsm_system": record.get("vsmSystem"),
|
|
"status": record.get("status", "active"), "description": record.get("description"),
|
|
"body": record, "created_at": _datetime(record.get("createdAt"), now),
|
|
"updated_at": _datetime(record.get("updatedAt"), now),
|
|
}
|
|
await _upsert(session, compat_hubs, row, existing)
|
|
|
|
|
|
async def _import_manifests(session: Any, bundle: dict, report: dict, hubs: dict, dry: bool) -> None:
|
|
current = await _maps(session, compat_manifests, "id")
|
|
for record in _records(bundle, "hubCapabilityManifests"):
|
|
existing = current.get(str(record["id"])); _mark(report, "hubCapabilityManifests", existing)
|
|
if dry: continue
|
|
now = _now(); hub_id = _parent_id(record, hubs, "hub")
|
|
row = {"id": record["id"], "hub_id": hub_id, "hub_slug": hubs[hub_id]["slug"],
|
|
"manifest_version": record.get("manifestVersion", "0.1.0"),
|
|
"status": record.get("status", "draft"), "body": record,
|
|
"created_at": _datetime(record.get("createdAt"), now),
|
|
"updated_at": _datetime(record.get("updatedAt"), now)}
|
|
await _upsert(session, compat_manifests, row, existing)
|
|
|
|
|
|
async def _import_consumers(session: Any, bundle: dict, report: dict, rows: dict, dry: bool) -> None:
|
|
for record in _records(bundle, "apiConsumers"):
|
|
existing = rows.get(str(record.get("id"))) or rows.get(str(record.get("slug")))
|
|
_mark(report, "apiConsumers", existing)
|
|
if dry: continue
|
|
row = {"id": existing["id"] if existing else str(record.get("id") or uuid4()),
|
|
"slug": record.get("slug"), "name": record["name"],
|
|
"description": record.get("description"),
|
|
"hub_capability_manifest_id": record.get("hubCapabilityManifestId"),
|
|
"rate_limit_per_minute": record.get("rateLimitPerMinute"),
|
|
"quota_per_day": record.get("quotaPerDay"), "key_prefix": record.get("keyPrefix"),
|
|
"status": record.get("status", "active"), "body": record,
|
|
"created_at": _datetime(record.get("createdAt"))}
|
|
await _upsert(session, compat_api_consumers, row, existing)
|
|
|
|
|
|
async def _import_keys(session: Any, bundle: dict, report: dict, consumers: dict, dry: bool) -> None:
|
|
current = await _maps(session, compat_api_keys, "id", "key_hash")
|
|
for record in _records(bundle, "apiKeys"):
|
|
existing = current.get(str(record["id"])) or current.get(str(record["keyHash"]))
|
|
_mark(report, "apiKeys", existing)
|
|
if dry: continue
|
|
consumer_id = _parent_id(record, consumers, "apiConsumer")
|
|
row = {"id": record["id"], "api_consumer_id": consumer_id,
|
|
"key_prefix": record["keyPrefix"], "key_hash": record["keyHash"],
|
|
"scopes": record.get("scopes"), "status": record.get("status", "active"),
|
|
"created_at": _datetime(record.get("createdAt"))}
|
|
await _upsert(session, compat_api_keys, row, existing)
|
|
|
|
|
|
async def _import_widgets(session: Any, bundle: dict, report: dict, hubs: dict, dry: bool) -> None:
|
|
current = await _maps(session, compat_widgets, "id")
|
|
for record in _records(bundle, "widgets"):
|
|
existing = current.get(str(record["id"])); _mark(report, "widgets", existing)
|
|
if dry: continue
|
|
row = {"id": record["id"], "hub_id": _parent_id(record, hubs, "hub"),
|
|
"name": record["name"], "widget_type": record.get("widgetType"),
|
|
"capability_ref": record.get("capabilityRef"),
|
|
"view_context": record.get("viewContext"), "policy_scope": record.get("policyScope"),
|
|
"status": record.get("status", "active"), "body": record,
|
|
"created_at": _datetime(record.get("createdAt"))}
|
|
await _upsert(session, compat_widgets, row, existing)
|
|
|
|
|
|
async def _import_events(session: Any, bundle: dict, report: dict, dry: bool) -> None:
|
|
current = await _maps(session, runtime_interaction_events, "id")
|
|
for record in _records(bundle, "interactionEvents"):
|
|
existing = current.get(str(record["id"])); _mark(report, "interactionEvents", existing)
|
|
if dry: continue
|
|
created = _datetime(record.get("createdAt") or record.get("occurredAt"))
|
|
row = {"id": record["id"], "schema_version": "0.1.0",
|
|
"correlation_id": str(record.get("correlationId") or uuid5(NAMESPACE_URL, record["id"])),
|
|
"event_type": record["eventType"], "occurred_at": created,
|
|
"subject_refs": {"widget": record["widgetId"]}, "payload": {"legacy": record},
|
|
"recorded_at": _datetime(record.get("recordedAt"), created)}
|
|
await _upsert(session, runtime_interaction_events, row, existing)
|
|
|
|
|
|
async def _import_source_runs(session: Any, bundle: dict, report: dict, dry: bool) -> None:
|
|
current = await _maps(session, compat_migration_runs, "id")
|
|
for record in _records(bundle, "migrationRuns"):
|
|
existing = current.get(str(record["id"])); _mark(report, "migrationRuns", existing)
|
|
if dry: continue
|
|
row = {"id": record["id"], "source": record["source"],
|
|
"schema_version": record["schemaVersion"], "bundle_sha256": record["bundleSha256"],
|
|
"dry_run": bool(record.get("dryRun", False)), "status": record.get("status", "imported"),
|
|
"counts": record.get("counts", {}), "diagnostics": record.get("diagnostics", {}),
|
|
"created_at": _datetime(record.get("createdAt"))}
|
|
await _upsert(session, compat_migration_runs, row, existing)
|
|
|
|
|
|
async def export_bundle(store: PostgresPortStore, *, source_revision: str | None = None) -> dict[str, Any]:
|
|
async with store.sessions() as session:
|
|
hubs = (await session.execute(sa.select(compat_hubs).order_by(compat_hubs.c.id))).mappings().all()
|
|
manifests = (await session.execute(sa.select(compat_manifests).order_by(compat_manifests.c.id))).mappings().all()
|
|
consumers = (await session.execute(sa.select(compat_api_consumers).order_by(compat_api_consumers.c.id))).mappings().all()
|
|
keys = (await session.execute(sa.select(compat_api_keys).order_by(compat_api_keys.c.id))).mappings().all()
|
|
widgets = (await session.execute(sa.select(compat_widgets).order_by(compat_widgets.c.id))).mappings().all()
|
|
events = (await session.execute(sa.select(runtime_interaction_events).order_by(runtime_interaction_events.c.id))).mappings().all()
|
|
runs = (await session.execute(sa.select(compat_migration_runs).order_by(compat_migration_runs.c.id))).mappings().all()
|
|
records = {
|
|
"hubs": [_body(row, created=True, updated=True) for row in hubs],
|
|
"hubCapabilityManifests": [_body(row, created=True, updated=True) for row in manifests],
|
|
"apiConsumers": [_body(row, created=True) for row in consumers],
|
|
"apiKeys": [{"id": row["id"], "apiConsumerId": row["api_consumer_id"], "keyPrefix": row["key_prefix"], "keyHash": row["key_hash"], "scopes": row["scopes"], "status": row["status"], "createdAt": _timestamp(row["created_at"])} for row in keys],
|
|
"widgets": [_body(row, created=True) for row in widgets],
|
|
"interactionEvents": [{**dict((row["payload"] or {}).get("legacy") or {}), "id": row["id"], "widgetId": (row["subject_refs"] or {}).get("widget"), "eventType": row["event_type"], "occurredAt": _timestamp(row["occurred_at"]), "recordedAt": _timestamp(row["recorded_at"]), "correlationId": row["correlation_id"]} for row in events],
|
|
"migrationRuns": [{"id": row["id"], "source": row["source"], "schemaVersion": row["schema_version"], "bundleSha256": row["bundle_sha256"], "dryRun": row["dry_run"], "status": row["status"], "counts": row["counts"], "diagnostics": row["diagnostics"], "createdAt": _timestamp(row["created_at"])} for row in runs],
|
|
}
|
|
high_water = max((_timestamp(row["recorded_at"]) for row in events), default=None)
|
|
bundle: dict[str, Any] = {"schemaVersion": SCHEMA_VERSION, "source": "hub-core", "exportedAt": _timestamp(_now()), "highWaterMark": high_water, "records": records}
|
|
if source_revision:
|
|
bundle["sourceRevision"] = source_revision
|
|
bundle["bundleSha256"] = bundle_digest(bundle)
|
|
return bundle
|
|
|
|
|
|
def _body(row: Any, *, created: bool = False, updated: bool = False) -> dict[str, Any]:
|
|
value = dict(row["body"] or {})
|
|
if created:
|
|
value["createdAt"] = _timestamp(row["created_at"])
|
|
if updated:
|
|
value["updatedAt"] = _timestamp(row["updated_at"])
|
|
return value
|
|
|
|
|
|
def load_bundle(path: Path) -> dict[str, Any]:
|
|
value = json.loads(path.read_text())
|
|
if not isinstance(value, dict):
|
|
raise ValueError("migration bundle must be a JSON object")
|
|
return value
|