fluid-core/internal/observation/redact.go

// Package observation implements the FLUID observation plane: telemetry
// normalization, redaction, cohorts, interaction topology, and interface
// pressure classification.
//
// ArchitectureBlueprint.md section 6.2 sets the boundary this package works
// inside: telemetry should be designed for interface learning without becoming
// an unrestricted behavioural capture layer. Raw payload capture is never the
// default, and semantic learning relies on minimized evidence where it can.
package observation

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/tegwick/fluid-core/internal/contract"
)

// RedactionPolicy governs what may reach the evidence store.
//
// It is deterministic configuration, not a heuristic. A privacy filter that
// guesses is a privacy filter that will one day guess wrong, and the evidence
// store is append-only: anything it accepts cannot be taken back out.
type RedactionPolicy struct {
	// Salt keys the pseudonymization of consumer identities. It must be stable
	// for the life of the interface: a rotated salt breaks every longitudinal
	// comparison, because the same consumer starts looking like a new one.
	Salt []byte

	// AllowRawPayload permits request and response bodies into telemetry.
	// Off by default, and Blueprint 6.2 says it should stay that way.
	AllowRawPayload bool

	// DropQueryParams removes named query parameters from recorded routes.
	DropQueryParams []string

	// DropHeaders removes named headers from recorded evidence.
	DropHeaders []string

	// SensitivePatterns match values that must never be stored, wherever they
	// appear. Anything matching is replaced rather than dropped, so the shape
	// of the evidence survives while the content does not.
	SensitivePatterns []*regexp.Regexp

	// RetentionDays bounds how long telemetry is kept. Zero means unbounded,
	// which should be a deliberate choice rather than an oversight.
	RetentionDays int

	// CohortMinimumSize is the smallest population that may be reported
	// separately. Below it, a "cohort" identifies individuals.
	CohortMinimumSize int
}

// DefaultRedactionPolicy returns a conservative policy.
//
// The defaults assume the interface handles something worth protecting. An
// operator who knows otherwise can loosen them explicitly; an operator who has
// not thought about it gets the safe behaviour.
func DefaultRedactionPolicy(salt []byte) RedactionPolicy {
	return RedactionPolicy{
		Salt:            salt,
		AllowRawPayload: false,
		DropQueryParams: []string{"token", "api_key", "apikey", "access_token", "signature", "password"},
		DropHeaders:     []string{"authorization", "cookie", "set-cookie", "proxy-authorization", "x-api-key"},
		SensitivePatterns: []*regexp.Regexp{
			// Bearer tokens and basic credentials appearing in free text.
			regexp.MustCompile(`(?i)bearer\s+[A-Za-z0-9._~+/-]+=*`),
			// Anything that looks like an email address.
			regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}`),
			// Connection strings with embedded credentials.
			regexp.MustCompile(`[a-z][a-z0-9+.-]*://[^\s:@/]+:[^\s@/]+@`),
		},
		RetentionDays:     90,
		CohortMinimumSize: 5,
	}
}

// ErrNoSalt reports a policy that would pseudonymize with an empty key.
var ErrNoSalt = errors.New("redaction policy has no salt; consumer identities would be trivially reversible")

// Validate checks a policy is usable.
func (p RedactionPolicy) Validate() error {
	if len(p.Salt) < 16 {
		return ErrNoSalt
	}
	return nil
}

// Pseudonymize maps a consumer identity to a stable opaque reference.
//
// HMAC rather than a plain hash: a bare SHA-256 of a short identifier — an
// account id, an email — is reversible by anyone willing to enumerate the
// input space, which for most identifier schemes is cheap.
func (p RedactionPolicy) Pseudonymize(identity string) string {
	if identity == "" {
		return ""
	}
	mac := hmac.New(sha256.New, p.Salt)
	_, _ = mac.Write([]byte(identity))
	// Twelve bytes is ample to keep collisions negligible at interface scale
	// while keeping the value short enough to read in a terminal.
	return "psu-" + hex.EncodeToString(mac.Sum(nil)[:12])
}

// Scrub removes sensitive substrings from free text.
func (p RedactionPolicy) Scrub(s string) (string, bool) {
	redacted := false
	for _, pattern := range p.SensitivePatterns {
		if pattern.MatchString(s) {
			s = pattern.ReplaceAllString(s, "[redacted]")
			redacted = true
		}
	}
	return s, redacted
}

// CleanRoute strips sensitive query parameters from a recorded route.
//
// The parameter is kept with an emptied value rather than removed. Which
// parameters a consumer sent is itself interface evidence — it tells you what
// they were trying to do — and deleting the key loses that.
func (p RedactionPolicy) CleanRoute(route string) (string, bool) {
	idx := strings.IndexByte(route, '?')
	if idx < 0 {
		return route, false
	}

	path, rawQuery := route[:idx], route[idx+1:]
	values, err := url.ParseQuery(rawQuery)
	if err != nil {
		// An unparseable query is dropped entirely: it cannot be inspected, so
		// it cannot be shown to be safe.
		return path, true
	}

	drop := map[string]bool{}
	for _, k := range p.DropQueryParams {
		drop[strings.ToLower(k)] = true
	}

	redacted := false
	keys := make([]string, 0, len(values))
	for k := range values {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	cleaned := url.Values{}
	for _, k := range keys {
		if drop[strings.ToLower(k)] {
			cleaned.Set(k, "[redacted]")
			redacted = true
			continue
		}
		for _, v := range values[k] {
			scrubbed, hit := p.Scrub(v)
			if hit {
				redacted = true
			}
			cleaned.Add(k, scrubbed)
		}
	}

	if len(cleaned) == 0 {
		return path, redacted
	}
	return path + "?" + cleaned.Encode(), redacted
}

// Apply redacts a telemetry event in place and records what it did.
//
// The applied rules are recorded on the event so that later analysis knows what
// it cannot see. Silent redaction would let an analyst mistake an absence of
// evidence for evidence of absence.
func (p RedactionPolicy) Apply(ev *contract.FluidTelemetry) {
	var rules []string

	if ev.ConsumerRef != "" && !strings.HasPrefix(ev.ConsumerRef, "psu-") {
		ev.ConsumerRef = p.Pseudonymize(ev.ConsumerRef)
		rules = append(rules, "pseudonymize-consumer")
	}

	if ev.Request != nil {
		if cleaned, hit := p.CleanRoute(ev.Request.Route); hit {
			ev.Request.Route = cleaned
			rules = append(rules, "clean-route")
		}
	}

	if ev.Error != nil && ev.Error.Detail != "" {
		if scrubbed, hit := p.Scrub(ev.Error.Detail); hit {
			ev.Error.Detail = scrubbed
			rules = append(rules, "scrub-error-detail")
		}
	}

	if ev.Sequence != nil && ev.Sequence.Pattern != "" {
		if scrubbed, hit := p.Scrub(ev.Sequence.Pattern); hit {
			ev.Sequence.Pattern = scrubbed
			rules = append(rules, "scrub-sequence-pattern")
		}
	}

	sort.Strings(rules)
	ev.Redaction = &contract.FluidTelemetryRedaction{
		Applied: len(rules) > 0,
		Rules:   rules,
	}
}

// Expired reports whether an event has outlived the retention policy.
func (p RedactionPolicy) Expired(ev contract.FluidTelemetry, now time.Time) bool {
	if p.RetentionDays <= 0 {
		return false
	}
	return ev.OccurredAt.Before(now.AddDate(0, 0, -p.RetentionDays))
}

// SuppressSmallCohort reports whether a population is too small to report on
// separately.
//
// Blueprint 6.2 requires cohort minimum sizes because a cohort of one is not a
// cohort; it is a named individual with extra steps.
func (p RedactionPolicy) SuppressSmallCohort(size int) bool {
	if p.CohortMinimumSize <= 0 {
		return false
	}
	return size < p.CohortMinimumSize
}