Redact secret-shaped fields by default, countable per field path

AUDIT-WP-0004-T04, closing the workplan.

Decision (Bernd): default to redaction, allow rejection per sender. Losing an
audit record over one field is worse than storing it masked, but a
higher-assurance channel must be able to refuse rather than mask. secret_policy
is set per sender identity in AUDIT_CORE_SENDERS and defaults to redact.

Detection now covers the whole payload at any depth, including lists, rather
than only the top level of data. Under redaction the value is masked and the
key is preserved: dropping the key would hide that the sender transmitted the
field at all, which is exactly what an operator needs in order to stop it. The
stored record carries details.redaction with policy and affected paths, so a
reader never has to infer whether what they see is what was sent.

Idempotency is unaffected - the payload hash is taken over the original request
body, so redaction is deterministic and a resubmission still reconciles as a
duplicate.

Both outcomes are counted durably by sender, source, action and field path,
exposed at GET /v1/secret-findings. Per-path aggregation is the point: the
actionable unit is "stop emitting data.auth.token on membership.added", not
"there were 47 redactions". Counters survive restart because the fix they drive
lives in another service.

Contract doc updated to match. Tests 46 -> 50. WP-0004 is finished.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_ingestion.py

@@ -100,7 +100,6 @@ def test_non_ascii_credential_is_unauthorized_not_a_crash(app):
 # --- validation (T02, T07) --------------------------------------------------
 
 @pytest.mark.parametrize("payload,expected", [
-    (event(data={"password": "never"}), "secret_shaped_field"),
     (event(source="somewhere-else"), "source_not_allowed"),
     (event(occurred_at="2026-08-09T00:00:00"), "timestamp_missing_timezone"),
     (event(occurred_at="not-a-date"), "invalid_timestamp"),
@@ -235,6 +234,7 @@ def bound_app(tmp_path, **kw):
         sources=frozenset(kw.get("sources", {"user-engine"})),
         tenants=frozenset(kw.get("tenants", {"tenant:friendly:binky"})),
         may_read=kw.get("may_read", False),
+        secret_policy=kw.get("secret_policy", "redact"),
     )
     backend = SQLiteAuditBackend(str(tmp_path / "bound.db"))
     return IngestionApplication(backend, SenderRegistry([identity])), backend
@@ -323,9 +323,10 @@ def test_rejected_events_appear_as_dead_letters(app):
     assert entry["payload"] is not None
 
 
-def test_secret_rejection_withholds_the_payload(app):
+def test_secret_rejection_withholds_the_payload(tmp_path):
     """Storing the body of an event rejected for carrying secret-shaped
     material would write that material into the audit store."""
+    app, _ = bound_app(tmp_path, secret_policy="reject", may_read=True)
     assert invoke(app, event(data={"password": "hunter2"}))[0].startswith("400")
     _, body = invoke(app, None, path="/v1/dead-letters", method="GET", body=b"")
     entry = body["dead_letters"][0]
@@ -333,3 +334,69 @@ def test_secret_rejection_withholds_the_payload(app):
     assert entry["payload_withheld"] is True
     assert entry["payload"] is None
     assert entry["payload_hash"]
+
+
+# --- redaction policy (T04) -------------------------------------------------
+
+def test_default_policy_redacts_and_accepts(app):
+    """Default is redact: losing the whole audit record over one field is
+    worse than storing it with that field masked."""
+    status, _ = invoke(app, event(data={"membership_id": "m-1", "auth_token": "s3cret"}))
+    assert status.startswith("202")
+
+    _, record = invoke(app, None, path="/v1/events/evt-1", method="GET", body=b"")
+    assert record["details"]["data"]["auth_token"] == "[redacted]"
+    assert record["details"]["data"]["membership_id"] == "m-1"
+    # The record must admit it was modified.
+    assert record["details"]["redaction"]["policy"] == "redact"
+    assert record["details"]["redaction"]["paths"] == ["data.auth_token"]
+
+
+def test_reject_policy_is_available_per_sender(tmp_path):
+    app, _ = bound_app(tmp_path, secret_policy="reject")
+    status, body = invoke(app, event(data={"password": "x"}))
+    assert status.startswith("400")
+    assert body["error"] == "secret_shaped_field"
+
+
+def test_nested_and_listed_secrets_are_redacted(app):
+    payload = event(data={"items": [{"api_secret": "a"}, {"ok": 1}], "n": {"private_key": "k"}})
+    assert invoke(app, payload)[0].startswith("202")
+    _, record = invoke(app, None, path="/v1/events/evt-1", method="GET", body=b"")
+    data = record["details"]["data"]
+    assert data["items"][0]["api_secret"] == "[redacted]"
+    assert data["items"][1]["ok"] == 1
+    assert data["n"]["private_key"] == "[redacted]"
+    assert set(record["details"]["redaction"]["paths"]) == {
+        "data.items[0].api_secret", "data.n.private_key",
+    }
+
+
+def test_findings_are_counted_by_path_for_both_outcomes(tmp_path):
+    """Counters name the field to fix, not just a total."""
+    app, backend = bound_app(tmp_path, may_read=True)
+    for i in range(3):
+        invoke(app, event(id=f"e{i}", data={"auth_token": "x"}), key=f"e{i}")
+
+    strict, _ = bound_app(tmp_path, secret_policy="reject")
+    invoke(strict, event(id="r1", data={"auth_token": "x"}), key="r1")
+
+    _, body = invoke(app, None, path="/v1/secret-findings", method="GET", body=b"")
+    rows = {(r["outcome"], r["field_path"]): r for r in body["secret_findings"]}
+    assert rows[("redacted", "data.auth_token")]["occurrences"] == 3
+    assert rows[("redacted", "data.auth_token")]["action"] == "membership.added"
+    assert rows[("redacted", "data.auth_token")]["source"] == "user-engine"
+    assert rows[("redacted", "data.auth_token")]["persisted"] is True
+    assert rows[("rejected", "data.auth_token")]["occurrences"] == 1
+
+
+def test_counters_survive_restart(tmp_path):
+    """The counters drive a fix in the sending service; that work outlives a
+    pod restart, so they are durable rather than in-memory."""
+    path = str(tmp_path / "counters.db")
+    first = IngestionApplication(SQLiteAuditBackend(path), "opaque")
+    invoke(first, event(data={"auth_token": "x"}))
+
+    reopened = IngestionApplication(SQLiteAuditBackend(path), "opaque")
+    _, body = invoke(reopened, None, path="/v1/secret-findings", method="GET", body=b"")
+    assert body["secret_findings"][0]["occurrences"] == 1