From 2f4e1adf6606f6a94db774c5136d4e1c9c6503f3 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 10 Aug 2026 17:42:43 +0200 Subject: [PATCH] Add deployment manifests, custody-class guard and request counters AUDIT-WP-0005-T03 (progress). Manifests validated --dry-run=server --validate=strict against railiance01; not applied, since deployment is gated on RAPP-POSTGRES-WP-0002 and T02 credentials. Nothing here mutates the cluster. Conventions read off the deployed user-engine workload rather than invented: digest-pinned image from forgejo.coulomb.social, runAsNonRoot with RuntimeDefault seccomp, no privilege escalation, all capabilities dropped, readOnlyRootFilesystem, probes on a named http port, same resource envelope. The namespace carries railiance.io/postgres-client: platform-pg, which is what platform-pg-consumer-ingress in rapp-postgres admits; without that label the pod cannot reach the database at all. NetworkPolicies default-deny both directions, then permit ingress from the user-engine namespace only, a separately labelled operator read path, and egress to PostgreSQL in databases plus DNS. Three decisions worth naming. Liveness is /healthz while readiness is /readyz, so a database outage drops the pod from the Service rather than restarting it in a loop. readOnlyRootFilesystem enforces the empty-filesystem property rather than trusting it, so the SQLite fallback physically cannot accumulate audit records on ephemeral storage. AUDIT_CORE_REQUIRE_CUSTODY_CLASS=archive makes a missing database URL a startup failure instead of a silent downgrade to the development store. Counters deferred from WP-0004-T06 are exposed as JSON at /v1/stats behind the read privilege, not as Prometheus exposition format: the cluster runs no Prometheus, no ServiceMonitor CRD and no other scrape target, so an exposition endpoint would target a scrape path that does not exist. Usable with curl now and a small step from /metrics later. Tests 77 -> 80. Co-Authored-By: Claude Opus 5 --- WORK-RECORDS.md | 2 +- audit_core/ingestion.py | 67 +++++++- deploy/audit-core.yaml | 157 ++++++++++++++++++ deploy/networkpolicies.yaml | 85 ++++++++++ tests/test_ingestion.py | 34 ++++ ...ostgres-store-and-production-deployment.md | 53 +++++- 6 files changed, 393 insertions(+), 5 deletions(-) create mode 100644 deploy/audit-core.yaml create mode 100644 deploy/networkpolicies.yaml diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 9cc0a87..f98ea11 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -31,6 +31,6 @@ | task | AUDIT-WP-0005-T01 | done | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | | task | AUDIT-WP-0005-T02 | todo | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | | task | AUDIT-WP-0005-T03 | todo | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | -| task | AUDIT-WP-0005-T04 | todo | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | +| task | AUDIT-WP-0005-T04 | done | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | | task | AUDIT-WP-0005-T05 | todo | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | | task | AUDIT-WP-0005-T06 | todo | — | workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md | diff --git a/audit_core/ingestion.py b/audit_core/ingestion.py index e51343f..c0a4497 100644 --- a/audit_core/ingestion.py +++ b/audit_core/ingestion.py @@ -26,6 +26,7 @@ import logging import os import signal import sys +import threading from datetime import datetime, timezone from http import HTTPStatus from typing import Any @@ -52,6 +53,40 @@ MAX_BODY_BYTES = 256 * 1024 log = logging.getLogger("audit_core.ingestion") +class Counters: + """In-process request counters (AUDIT-WP-0005-T03). + + Exposed as JSON at ``/v1/stats`` rather than in Prometheus exposition + format, because railiance01 currently runs no Prometheus, ServiceMonitor + CRD, or any other scrape target. Building an exposition endpoint for a + scrape path that does not exist would be guessing; this is usable by an + operator with curl today and is a small step from a /metrics endpoint when + a metrics stack lands. + + These reset on restart, which is correct for rate signals. The counters + that must survive a restart — secret-shaped field findings — are persisted + by the backend instead. + """ + + FIELDS = ("accepted", "duplicate", "conflict", "rejected", "unauthorized", + "forbidden", "unavailable", "error") + + def __init__(self) -> None: + self._lock = threading.Lock() + self._counts = {name: 0 for name in self.FIELDS} + self._started = datetime.now(timezone.utc).replace(microsecond=0) + + def hit(self, name: str) -> None: + with self._lock: + if name in self._counts: + self._counts[name] += 1 + + def snapshot(self) -> dict[str, Any]: + with self._lock: + counts = dict(self._counts) + return {"since": self._started.isoformat(), "counts": counts} + + class IngestionApplication: """WSGI application accepting user-engine outbox events. @@ -61,9 +96,20 @@ class IngestionApplication: """ def __init__( - self, backend: IdempotentAuditBackend, senders: SenderRegistry | str + self, + backend: IdempotentAuditBackend, + senders: SenderRegistry | str, + require_custody_class: str | None = None, ) -> None: policy = backend.retention_policy + if require_custody_class and policy.custody_class != require_custody_class: + # Production sets this. Without it, losing AUDIT_CORE_DATABASE_URL + # silently downgrades custody to the development store instead of + # failing to start. + raise ValueError( + f"backend custody_class={policy.custody_class!r} does not meet the " + f"required {require_custody_class!r}; refusing to start" + ) if not policy.durable: # The mock file backend declares durable=False. Refusing it here is # what stops a development sink from silently becoming the @@ -78,6 +124,7 @@ class IngestionApplication: senders = development_registry(senders) self.backend = backend self.senders = senders + self.counters = Counters() def __call__(self, environ, start_response): try: @@ -87,6 +134,7 @@ class IngestionApplication: # start_response is never called and the sender sees a dropped # connection it cannot classify. log.exception("unhandled error in ingestion request") + self.counters.hit("error") return self._json( start_response, HTTPStatus.INTERNAL_SERVER_ERROR, {"error": "internal_error"} ) @@ -100,13 +148,14 @@ class IngestionApplication: method = environ.get("REQUEST_METHOD") identity = self.senders.authenticate(environ.get("HTTP_AUTHORIZATION")) if identity is None: + self.counters.hit("unauthorized") return self._json( start_response, HTTPStatus.UNAUTHORIZED, {"error": "unauthorized"} ) if method == "GET" and ( path.startswith("/v1/events") - or path in ("/v1/dead-letters", "/v1/secret-findings") + or path in ("/v1/dead-letters", "/v1/secret-findings", "/v1/stats") ): return self._read(start_response, environ, path, identity) @@ -114,6 +163,7 @@ class IngestionApplication: return self._json(start_response, HTTPStatus.NOT_FOUND, {"error": "not_found"}) if not identity.may_write: + self.counters.hit("forbidden") return self._json(start_response, HTTPStatus.FORBIDDEN, {"error": "write_forbidden"}) raw = b"" @@ -125,9 +175,11 @@ class IngestionApplication: except SecretFieldRejection as exc: self._count_secrets(payload, identity, "rejected", exc.findings) self._dead_letter(raw, str(exc), identity) + self.counters.hit("rejected") return self._json(start_response, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) except (ValueError, TypeError, KeyError, json.JSONDecodeError) as exc: self._dead_letter(raw, str(exc), identity) + self.counters.hit("rejected") return self._json(start_response, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) redaction = event.details.get("redaction") @@ -141,15 +193,18 @@ class IngestionApplication: result = self.backend.accept(event, hashlib.sha256(raw).hexdigest()) except EventConflictError as exc: log.warning("event conflict: %s", exc) + self.counters.hit("conflict") return self._json(start_response, HTTPStatus.CONFLICT, {"error": "event_id_conflict"}) except EventValidationError as exc: return self._json(start_response, HTTPStatus.BAD_REQUEST, {"error": str(exc)}) except BackendUnavailableError as exc: log.error("backend unavailable: %s", exc) + self.counters.hit("unavailable") return self._json( start_response, HTTPStatus.SERVICE_UNAVAILABLE, {"error": "backend_unavailable"} ) + self.counters.hit("duplicate" if result.duplicate else "accepted") return self._json( start_response, HTTPStatus.OK if result.duplicate else HTTPStatus.ACCEPTED, @@ -175,6 +230,8 @@ class IngestionApplication: start_response, HTTPStatus.OK, {"dead_letters": self.backend.dead_letters(_limit(query))}, ) + if path == "/v1/stats": + return self._json(start_response, HTTPStatus.OK, self.counters.snapshot()) if path == "/v1/secret-findings": return self._json( start_response, HTTPStatus.OK, @@ -451,7 +508,11 @@ def main() -> None: format='{"ts":"%(asctime)s","level":"%(levelname)s","logger":"%(name)s","msg":"%(message)s"}', stream=sys.stdout, ) - app = IngestionApplication(build_backend(), SenderRegistry.from_env()) + app = IngestionApplication( + build_backend(), + SenderRegistry.from_env(), + require_custody_class=os.environ.get("AUDIT_CORE_REQUIRE_CUSTODY_CLASS") or None, + ) serve( app, host=os.environ.get("AUDIT_CORE_HOST", "0.0.0.0"), diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml new file mode 100644 index 0000000..907893b --- /dev/null +++ b/deploy/audit-core.yaml @@ -0,0 +1,157 @@ +# audit-core receiver — railiance01 (AUDIT-WP-0005-T03). +# +# Conventions match the deployed user-engine workload: digest-pinned image from +# forgejo.coulomb.social, non-root with a read-only root filesystem, and probes +# on a named http port. +# +# Apply order matters: the namespace label railiance.io/postgres-client is what +# platform-pg-consumer-ingress in the databases namespace admits, so without it +# the pod cannot reach the database at all. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: audit-core + labels: + kubernetes.io/metadata.name: audit-core + railiance.io/workload-class: platform + # Admitted by NetworkPolicy platform-pg-consumer-ingress (rapp-postgres). + railiance.io/postgres-client: platform-pg +--- +apiVersion: v1 +kind: Service +metadata: + name: audit-core + namespace: audit-core + labels: + app.kubernetes.io/name: audit-core +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: audit-core + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: audit-core + namespace: audit-core + labels: + app.kubernetes.io/name: audit-core + annotations: + # Rollback position. Update both together; `kubectl rollout undo` returns to + # the previous digest, and the schema note records whether that is safe. + audit-core.railiance.io/rollback-note: >- + Migrations 0001-0004 are additive (CREATE TABLE/INDEX/TRIGGER IF NOT + EXISTS) and are not reversed by a rollback. An older image runs against + the newer schema without harm. A future migration that drops or narrows a + column breaks that property and must state its own rollback position + before it is released. +spec: + replicas: 1 + revisionHistoryLimit: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: audit-core + template: + metadata: + labels: + app.kubernetes.io/name: audit-core + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + # In-flight events must not be lost on rollout; the app shuts down + # gracefully on SIGTERM (AUDIT-WP-0004-T06). + terminationGracePeriodSeconds: 30 + containers: + - name: audit-core + # REPLACE at release time with the built digest. A mutable tag is not + # an immutable image, and `:latest` must never be the only reference. + image: forgejo.coulomb.social/coulomb/audit-core@sha256:REPLACE_AT_RELEASE + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + env: + - name: AUDIT_CORE_HOST + value: "0.0.0.0" + - name: AUDIT_CORE_HTTP_PORT + value: "8080" + # Refuses to start on anything but archive-class custody, so a + # missing database URL fails loudly instead of silently downgrading + # to the development store. + - name: AUDIT_CORE_REQUIRE_CUSTODY_CLASS + value: archive + - name: AUDIT_CORE_DATABASE_SCHEMA + value: audit_core + - name: AUDIT_CORE_THREADS + value: "8" + - name: AUDIT_CORE_REQUEST_TIMEOUT + value: "30" + - name: AUDIT_CORE_DB_STATEMENT_TIMEOUT_MS + value: "30000" + # Both secrets are delivered through the OpenBao lane + # (AUDIT-WP-0005-T02) — never committed, never set by hand. + - name: AUDIT_CORE_DATABASE_URL + valueFrom: + secretKeyRef: + name: audit-core-database + key: url + - name: AUDIT_CORE_SENDERS + valueFrom: + secretKeyRef: + name: audit-core-senders + key: senders.json + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + # Custody lives in PostgreSQL. Nothing of value is written to the + # pod filesystem, and a read-only root enforces that rather than + # trusting it — the SQLite fallback cannot silently start + # accumulating audit records on ephemeral storage. + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + startupProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 3 + failureThreshold: 20 + readinessProbe: + # /readyz checks the backend is reachable and durable, so the pod + # leaves the Service when custody is unavailable rather than + # accepting events it cannot store. + httpGet: {path: /readyz, port: http} + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + livenessProbe: + # Deliberately /healthz, not /readyz: a database outage must not + # restart the pod in a loop. Losing readiness is the correct + # response; restarting solves nothing and loses in-flight work. + httpGet: {path: /healthz, port: http} + periodSeconds: 20 + timeoutSeconds: 2 + failureThreshold: 3 + volumes: + - name: tmp + emptyDir: {} diff --git a/deploy/networkpolicies.yaml b/deploy/networkpolicies.yaml new file mode 100644 index 0000000..7a74919 --- /dev/null +++ b/deploy/networkpolicies.yaml @@ -0,0 +1,85 @@ +# Default-deny plus the narrowest set of exceptions (AUDIT-WP-0005-T03). +# +# The receiver holds the audit trail, so reachability is part of its threat +# model: only the declared sender may write, and only the declared operator +# path may read. +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: audit-core-default-deny + namespace: audit-core +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + # No rules: everything not permitted below is denied. +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: audit-core-sender-ingress + namespace: audit-core +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: audit-core + policyTypes: [Ingress] + ingress: + # user-engine is the only sender. A second sender is a deliberate change + # here and a matching entry in AUDIT_CORE_SENDERS — the network rule and + # the credential binding must move together. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: user-engine + ports: + - {protocol: TCP, port: 8080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: audit-core-operator-ingress + namespace: audit-core +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: audit-core + policyTypes: [Ingress] + ingress: + # Operator read path: lookup, dead letters, secret findings, stats. + # Namespace-scoped rather than open, and still gated on a credential + # carrying may_read — the network rule is the outer of two checks, not the + # only one. + - from: + - namespaceSelector: + matchLabels: + railiance.io/audit-core-reader: "true" + ports: + - {protocol: TCP, port: 8080} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: audit-core-egress + namespace: audit-core +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: audit-core + policyTypes: [Egress] + egress: + # PostgreSQL custody store. This is the only destination the receiver needs; + # it calls no other service. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: databases + ports: + - {protocol: TCP, port: 5432} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - {protocol: UDP, port: 53} + - {protocol: TCP, port: 53} diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 6548c1d..cbd0344 100644 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -400,3 +400,37 @@ def test_counters_survive_restart(tmp_path): reopened = IngestionApplication(SQLiteAuditBackend(path), "opaque") _, body = invoke(reopened, None, path="/v1/secret-findings", method="GET", body=b"") assert body["secret_findings"][0]["occurrences"] == 1 + + +# --- deployment guards and counters (WP-0005-T03) --------------------------- + +def test_required_custody_class_refuses_a_development_backend(tmp_path): + """Losing AUDIT_CORE_DATABASE_URL must fail to start, not silently + downgrade custody to the development store.""" + backend = SQLiteAuditBackend(str(tmp_path / "dev.db")) + with pytest.raises(ValueError, match="does not meet the required"): + IngestionApplication(backend, "opaque", require_custody_class="archive") + + +def test_counters_track_each_outcome(tmp_path): + app, _ = bound_app(tmp_path, may_read=True) + invoke(app, event()) # accepted + invoke(app, event()) # duplicate + invoke(app, event(subject="other")) # conflict + invoke(app, event(id="e2", tenant="tenant:coulomb"), key="e2") # rejected + invoke(app, event(), token="nope") # unauthorized + + _, body = invoke(app, None, path="/v1/stats", method="GET", body=b"") + counts = body["counts"] + assert counts["accepted"] == 1 + assert counts["duplicate"] == 1 + assert counts["conflict"] == 1 + assert counts["rejected"] == 1 + assert counts["unauthorized"] == 1 + assert body["since"] + + +def test_stats_require_the_read_privilege(tmp_path): + app, _ = bound_app(tmp_path, may_read=False) + status, _ = invoke(app, None, path="/v1/stats", method="GET", body=b"") + assert status.startswith("403") diff --git a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md index eec4f1d..f67ca50 100644 --- a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md +++ b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md @@ -137,7 +137,7 @@ events. ```task id: AUDIT-WP-0005-T03 -status: todo +status: progress priority: high state_hub_task_id: "598af2ac-e772-4a4e-9a65-dde9d4ca167f" ``` @@ -162,6 +162,57 @@ Done when the receiver is reachable only by its declared peers, survives pod restart and rescheduling without loss, and has a tested path back to the previous version. +Progress 2026-08-10: manifests written and validated `--dry-run=server +--validate=strict` against railiance01. Not applied — deployment is gated on +RAPP-POSTGRES-WP-0002 and on T02's credentials, and nothing here mutates the +cluster. + +Conventions were read off the deployed `user-engine` workload rather than +invented: digest-pinned image from `forgejo.coulomb.social/coulomb/`, +`runAsNonRoot` + `RuntimeDefault` seccomp, `allowPrivilegeEscalation: false`, +all capabilities dropped, `readOnlyRootFilesystem: true`, probes on a named +`http` port, and the same resource envelope. + +`deploy/audit-core.yaml` — namespace, Service, Deployment. The namespace +carries `railiance.io/postgres-client: platform-pg`, which is what +`platform-pg-consumer-ingress` in rapp-postgres admits; without that label the +pod cannot reach the database at all. + +`deploy/networkpolicies.yaml` — default-deny both directions, then the +narrowest exceptions: ingress from the `user-engine` namespace only, a +separately labelled operator read path, and egress restricted to PostgreSQL in +`databases` plus DNS. + +Three decisions worth recording: + +- **Liveness is `/healthz`, readiness is `/readyz`.** A database outage must + drop the pod out of the Service, not restart it in a loop — restarting solves + nothing and discards in-flight work. +- **`readOnlyRootFilesystem: true` enforces the empty-filesystem property** + rather than trusting it. Custody is in PostgreSQL, and a read-only root means + the SQLite fallback physically cannot start accumulating audit records on + ephemeral storage. +- **`AUDIT_CORE_REQUIRE_CUSTODY_CLASS=archive`** makes a missing database URL a + startup failure instead of a silent downgrade to the development store. Added + in this task; covered by a test. + +Rollback position is stated on the Deployment: migrations 0001-0004 are +additive, so an older image runs against the newer schema without harm and +`kubectl rollout undo` is safe. A future migration that drops or narrows a +column breaks that property and must state its own rollback position before +release. + +Counters deferred here from WP-0004-T06 are exposed as JSON at `/v1/stats` +(accepted, duplicate, conflict, rejected, unauthorized, forbidden, unavailable, +error), behind the read privilege. Deliberately not Prometheus exposition +format: railiance01 currently runs no Prometheus, no ServiceMonitor CRD, and no +other scrape target, so an exposition endpoint would be built for a scrape path +that does not exist. This is usable with curl today and a small step from +`/metrics` when a metrics stack lands. + +Remaining before done: build and publish the image to get a real digest, apply, +and verify restart/rescheduling and the rollback path on the cluster. + ## T04 - Migrate existing SQLite records ```task