secrets-engine/src/secrets_engine/evidence.py

"""Non-secret evidence writer.

Every privileged or noteworthy action emits an evidence record to a local
append-only JSONL log and, best-effort, to the State Hub progress API. Records
are scrubbed of anything that looks like a secret value before they are written.
"""
from __future__ import annotations

import json
import os
import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

from secrets_engine.errors import DecisionError, SecretsEngineError
from secrets_engine.evidence_class import KIND_ATTRIBUTIVE, classify
from secrets_engine.redact import looks_secret, redact_text

# Keys that must never carry a value into evidence regardless of nesting.
_FORBIDDEN_VALUE_KEYS = {"value", "secret", "token", "password", "raw"}


def _scrub(obj: Any) -> Any:
    """Recursively drop secret-looking keys and redact token shapes in strings."""
    if isinstance(obj, dict):
        out = {}
        for k, v in obj.items():
            if k.lower() in _FORBIDDEN_VALUE_KEYS or looks_secret(k):
                out[k] = "<omitted: non-secret evidence only>"
            else:
                out[k] = _scrub(v)
        return out
    if isinstance(obj, list):
        return [_scrub(v) for v in obj]
    if isinstance(obj, str):
        return redact_text(obj)
    return obj


@dataclass
class EvidenceWriter:
    evidence_dir: Path
    hub_url: str = ""
    topic_id: str = ""
    workstream_id: str = ""
    author: str = "secrets-engine"
    repo_slug: str = "secrets-engine"
    actor: str = field(default_factory=lambda: os.environ.get("USER", "unknown"))

    def __post_init__(self) -> None:
        self.evidence_dir = Path(self.evidence_dir)

    def _log_path(self) -> Path:
        self.evidence_dir.mkdir(parents=True, exist_ok=True)
        day = datetime.now(timezone.utc).strftime("%Y-%m-%d")
        return self.evidence_dir / f"evidence-{day}.jsonl"

    def _append_local(self, record: dict[str, Any]) -> None:
        """Append one record before any best-effort external delivery."""
        path = self._log_path()
        with path.open("a", encoding="utf-8") as fh:
            fh.write(json.dumps(record, sort_keys=True) + "\n")

    def outbox_dir(self) -> Path:
        path = self.evidence_dir / "outbox"
        path.mkdir(parents=True, exist_ok=True)
        return path

    def _queue_outbox(self, record: dict[str, Any]) -> str:
        """Durably queue a load-bearing record. Never talks to audit-core."""
        record_id = str(record.get("record_id") or uuid.uuid4())
        path = self.outbox_dir() / f"{record_id}.json"
        tmp = path.with_suffix(".tmp")
        tmp.write_text(json.dumps(record, sort_keys=True) + "\n", encoding="utf-8")
        os.chmod(tmp, 0o600)
        tmp.replace(path)
        return record_id

    def record(
        self,
        action: str,
        *,
        result: str,
        catalog_id: str = "",
        stage: str = "",
        decision_id: str = "",
        detail: dict[str, Any] | None = None,
        hub: bool = True,
    ) -> dict[str, Any]:
        """Append one non-secret evidence record. Returns the stored record.

        Load-bearing records are queued locally first. An audit-core outage
        cannot occur here because this method never contacts audit-core.
        Completeness is never claimed. Presence or absence of a record is
        not consulted as a permission.
        """
        record_id = str(uuid.uuid4())
        evidence_class = classify(action, stage)
        hub_requested = bool(
            hub and self.hub_url and evidence_class.kind == KIND_ATTRIBUTIVE
        )
        record = {
            "record_id": record_id,
            "ts": datetime.now(timezone.utc).isoformat(),
            "action": action,
            "result": result,
            "actor": self.actor,
            "catalog_id": catalog_id,
            "stage": stage,
            "decision_id": decision_id,
            "detail": _scrub(detail or {}),
            "hub_delivery_requested": hub_requested,
            "evidence_kind": evidence_class.kind,
            "evidence_rule": evidence_class.rule_id,
            "completeness_claimed": False,
        }
        if evidence_class.queued_locally:
            self._queue_outbox(record)
            record["outbox_queued"] = True
        self._append_local(record)
        if hub_requested:
            delivery = self._post_hub(
                action,
                result,
                catalog_id,
                stage,
                decision_id,
                record_id=record_id,
            )
            # Append-only companion evidence makes an unavailable State Hub
            # visible without rewriting or delaying the primary local record.
            self._append_local(
                {
                    "record_id": str(uuid.uuid4()),
                    "related_record_id": record_id,
                    "ts": datetime.now(timezone.utc).isoformat(),
                    "action": "evidence-delivery",
                    "result": delivery.status,
                    "actor": self.actor,
                    "catalog_id": catalog_id,
                    "stage": stage,
                    "decision_id": decision_id,
                    "detail": {"outbox_id": delivery.outbox_id}
                    if delivery.outbox_id
                    else {},
                    "hub_delivery_requested": False,
                }
            )
        return record

    def _post_hub(
        self,
        action: str,
        result: str,
        catalog_id: str,
        stage: str,
        decision_id: str,
        *,
        record_id: str,
    ) -> "HubDelivery":
        """Best-effort progress note; return a non-secret delivery outcome."""
        if not self.topic_id:
            return HubDelivery("skipped-no-topic")
        summary = f"secrets-engine {action}: {result}"
        if catalog_id:
            summary += f" [{catalog_id}{'/' + stage if stage else ''}]"
        payload: dict[str, Any] = {
            "topic_id": self.topic_id,
            "event_type": "note",
            "summary": summary,
            "author": self.author,
        }
        if self.workstream_id:
            payload["workplan_id"] = self.workstream_id
        if decision_id:
            payload["detail"] = {"decision_id": decision_id, "catalog_id": catalog_id}
        try:
            idempotency_key = f"secrets-engine:{record_id}"
            req = urllib.request.Request(
                self.hub_url.rstrip("/") + "/progress/",
                data=json.dumps(payload).encode(),
                headers={
                    "Content-Type": "application/json",
                    "Idempotency-Key": idempotency_key,
                    "X-StateHub-Source-Agent": self.author,
                    "X-StateHub-Repo-Slug": self.repo_slug,
                },
                method="POST",
            )
            response = urllib.request.urlopen(req, timeout=3)
            body = response.read()
            status = getattr(response, "status", 200)
            if status == 202:
                try:
                    receipt = json.loads(body or b"{}")
                except (json.JSONDecodeError, TypeError):
                    receipt = {}
                if isinstance(receipt, dict) and receipt.get("queued") is True:
                    outbox_id = receipt.get("outbox_id", "")
                    if isinstance(outbox_id, str):
                        try:
                            if str(uuid.UUID(outbox_id)) == outbox_id:
                                return HubDelivery("queued", outbox_id=outbox_id)
                        except ValueError:
                            pass
                return HubDelivery("failed")
            return HubDelivery("delivered")
        except (urllib.error.URLError, OSError, ValueError):
            # Hub being offline must never block secret work or leak anything.
            return HubDelivery("failed")


@dataclass(frozen=True)
class HubDelivery:
    """Non-secret outcome returned by State Hub or its edge relay."""

    status: str
    outbox_id: str = ""


@dataclass
class PrivilegedActionEvidence:
    """Record one privileged command's attempt, approval, and terminal result."""

    writer: EvidenceWriter
    action: str
    catalog_id: str
    stage: str
    decision_ref: str = ""
    approval_required: bool = False
    detail: dict[str, Any] = field(default_factory=dict)
    decision_id: str = ""
    approval_status: str = "pending"
    completed: bool = False
    stance: dict[str, Any] = field(default_factory=dict)
    consume: dict[str, Any] = field(default_factory=dict)

    def __post_init__(self) -> None:
        if not self.approval_required:
            self.approval_status = "not-required"

    def _detail(self, extra: dict[str, Any] | None = None) -> dict[str, Any]:
        merged = dict(self.detail)
        merged.update(
            {
                "approval_status": self.approval_status,
                "decision_ref": self.decision_ref,
            }
        )
        if self.stance:
            merged.update(self.stance)
        if self.consume:
            merged.update(self.consume)
        if extra:
            merged.update(extra)
        return merged

    def __enter__(self) -> "PrivilegedActionEvidence":
        self.writer.record(
            self.action,
            result="attempt",
            catalog_id=self.catalog_id,
            stage=self.stage,
            detail=self._detail(),
        )
        return self

    def mark_approved(self, decision: object | None) -> None:
        if decision is None:
            self.approval_status = "not-required"
            return
        self.decision_id = str(getattr(decision, "id", ""))
        self.approval_status = "approved"

    def mark_consumed(self, consumed: object | None) -> None:
        if consumed is None:
            return
        if hasattr(consumed, "as_evidence"):
            payload = consumed.as_evidence()
        elif isinstance(consumed, dict):
            payload = consumed
        else:
            return
        self.consume = {key: value for key, value in payload.items() if value != ""}

    def mark_stance(self, stance: object | None) -> None:
        if stance is None:
            return
        if hasattr(stance, "as_evidence"):
            payload = stance.as_evidence()
        elif isinstance(stance, dict):
            payload = stance
        else:
            return
        self.stance = {key: value for key, value in payload.items() if value != ""}

    def finish(
        self, result: str, *, detail: dict[str, Any] | None = None
    ) -> dict[str, Any]:
        self.completed = True
        return self.writer.record(
            self.action,
            result=result,
            catalog_id=self.catalog_id,
            stage=self.stage,
            decision_id=self.decision_id,
            detail=self._detail(detail),
        )

    def __exit__(self, exc_type, exc, _traceback) -> bool:
        if exc_type is None:
            if not self.completed:
                self.finish("failed-incomplete")
            return False
        if isinstance(exc, DecisionError):
            self.approval_status = "rejected"
            if getattr(exc, "stance", None):
                self.mark_stance(exc.stance)
        if isinstance(exc, (KeyboardInterrupt, SystemExit)):
            result = "interrupted"
        elif isinstance(exc, SecretsEngineError):
            result = f"failed-{type(exc).__name__}"
        else:
            result = "failed-unexpected"
        self.writer.record(
            self.action,
            result=result,
            catalog_id=self.catalog_id,
            stage=self.stage,
            decision_id=self.decision_id,
            detail=self._detail({"error_type": type(exc).__name__}),
        )
        return False


def write_heartbeat(writer: EvidenceWriter, *, stage: str = "prod") -> dict[str, Any]:
    """Positive claim that can go missing. Not a permission and not silence."""
    queued = 0
    outbox = writer.evidence_dir / "outbox"
    if outbox.is_dir():
        queued = sum(1 for path in outbox.glob("*.json") if path.is_file())
    return writer.record(
        "evidence-heartbeat",
        result="nothing-to-report",
        stage=stage,
        detail={
            "form": "heartbeat",
            "outbox_depth": queued,
            "completeness_claimed": False,
        },
        hub=False,
    )


def drain_outbox(
    writer: EvidenceWriter,
    *,
    audit_core_url: str = "",
) -> dict[str, Any]:
    """Best-effort delivery of queued load-bearing records.

    Never called from a mutation path. An audit-core outage leaves files in
    place and does not raise into a revoke/destroy handler.
    """
    outbox = writer.evidence_dir / "outbox"
    if not outbox.is_dir():
        return {
            "queued": 0,
            "delivered": 0,
            "failed": 0,
            "skipped": 0,
            "completeness_claimed": False,
        }
    files = sorted(path for path in outbox.glob("*.json") if path.is_file())
    queued = len(files)
    if not audit_core_url:
        return {
            "queued": queued,
            "delivered": 0,
            "failed": 0,
            "skipped": queued,
            "completeness_claimed": False,
        }
    delivered = 0
    failed = 0
    for path in files:
        try:
            payload = path.read_text(encoding="utf-8").encode()
            req = urllib.request.Request(
                audit_core_url.rstrip("/") + "/v1/events",
                data=payload,
                headers={"Content-Type": "application/json"},
                method="POST",
            )
            urllib.request.urlopen(req, timeout=3)
            path.unlink()
            delivered += 1
        except (urllib.error.URLError, OSError, ValueError):
            failed += 1
    return {
        "queued": queued,
        "delivered": delivered,
        "failed": failed,
        "skipped": 0,
        "completeness_claimed": False,
    }