feat(posture): add deterministic feedback proposals
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02929-244b-7391-b933-c04010e8eedb
This commit is contained in:
tegwick 2026-08-23 13:16:34 +02:00
parent dc8da422f8
commit cfc9e7d0cb
19 changed files with 1428 additions and 16 deletions

View file

@ -0,0 +1,22 @@
# Posture Feedback Evaluator
This tool implements the proposal-only feedback boundary in
`canon/standards/posture-feedback_v0.1.md`. It reads tenancy declarations and
emits deterministic JSON. It never writes State Hub, edits a declaration,
changes policy, or executes remediation.
Evaluate the checked-in reference at an explicit time:
```bash
uv run tools/posture-feedback/posture_feedback.py \
--as-of 2026-08-23T12:00:00Z \
--horizon-days 14 \
--fail-on none \
examples/posture-feedback/expired-e2.yaml
```
Run tests:
```bash
uv run pytest tools/posture-feedback/tests
```

View file

@ -0,0 +1,440 @@
#!/usr/bin/env python3
# /// script
# dependencies = ["jsonschema>=4.23,<5", "PyYAML>=6,<7"]
# ///
"""Emit deterministic, proposal-only feedback for tenancy posture declarations."""
from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import importlib.util
import json
import pathlib
import sys
from collections import Counter
from typing import Any
import jsonschema
import yaml
ROOT = pathlib.Path(__file__).resolve().parents[2]
TENANCY_SCHEMA = ROOT / "canon/schemas/tenancy-posture_v0.1.schema.json"
REPORT_SCHEMA = ROOT / "canon/schemas/posture-feedback-report_v0.1.schema.json"
TENANCY_VALIDATOR_PATH = ROOT / "tools/tenancy-posture/validate.py"
API_VERSION = "netkingdom.io/posture-feedback/v0.1"
KIND = "PostureFeedbackReport"
ADVERSARIAL_LEVELS = {"E2", "R4", "V2", "V3", "V4"}
SEVERITY_RANK = {"low": 1, "medium": 2, "high": 3}
SPEC = importlib.util.spec_from_file_location(
"tenancy_posture_validator_for_feedback", TENANCY_VALIDATOR_PATH
)
assert SPEC and SPEC.loader
TENANCY_VALIDATOR = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(TENANCY_VALIDATOR)
def parse_timestamp(value: str) -> dt.datetime:
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.tzinfo is None or parsed.utcoffset() is None:
raise ValueError("timestamp must include an explicit UTC offset")
return parsed.astimezone(dt.timezone.utc)
def format_timestamp(value: dt.datetime) -> str:
return value.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def load_document(path: pathlib.Path) -> dict[str, Any]:
document = yaml.safe_load(path.read_text(encoding="utf-8"))
if not isinstance(document, dict):
raise ValueError("document must be a YAML object")
return document
def service_entries(document: dict[str, Any]) -> list[dict[str, Any]]:
if "services" in document:
return document["services"]
return [document]
def source_identifier(path: pathlib.Path) -> str:
"""Return a portable declaration id without treating the path as ownership."""
resolved = path.resolve()
try:
return resolved.relative_to(ROOT.parent).as_posix()
except ValueError:
return resolved.as_posix()
def finding_id(
source: str,
service: str,
finding_class: str,
control: str,
due: str | None,
) -> str:
identity = "\x1f".join((source, service, finding_class, control, due or ""))
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16]
return f"NKFB-{digest}"
def finding(
*,
source: str,
service: str,
finding_class: str,
severity: str,
control: str,
owner: str,
evidence_state: str,
reason: str,
recommended_action: str,
due: str | None = None,
) -> dict[str, Any]:
result = {
"id": finding_id(source, service, finding_class, control, due),
"class": finding_class,
"severity": severity,
"source": source,
"service": service,
"control": control,
"owner": owner,
"evidence_state": evidence_state,
"reason": reason,
"recommended_action": recommended_action,
}
if due is not None:
result["due"] = due
return result
def review_finding(
*,
source: str,
service: str,
owner: str,
control: str,
due: dt.date,
as_of: dt.datetime,
horizon_days: int,
prefix: str,
) -> dict[str, Any] | None:
due_text = due.isoformat()
if as_of.date() > due:
return finding(
source=source,
service=service,
finding_class=f"{prefix}-review-overdue",
severity="high",
control=control,
owner=owner,
evidence_state="overdue",
due=due_text,
reason=f"{control} review was due on {due_text} and is overdue at {format_timestamp(as_of)}.",
recommended_action=f"Review {control}, update its declaration and evidence, and retain the review record.",
)
days = (due - as_of.date()).days
if days <= horizon_days:
return finding(
source=source,
service=service,
finding_class=f"{prefix}-review-due-soon",
severity="medium",
control=control,
owner=owner,
evidence_state="due-soon",
due=due_text,
reason=f"{control} review is due in {days} day(s), within the {horizon_days}-day horizon.",
recommended_action=f"Schedule and evidence the {control} review before {due_text}.",
)
return None
def current_evidence_keys(entry: dict[str, Any]) -> set[str]:
current = entry["tenancy"]["current"]
return {
f"{axis}{level}"
for axis, level in current.items()
if isinstance(level, int)
}
def evidence_findings(
*,
source: str,
entry: dict[str, Any],
as_of: dt.datetime,
horizon_days: int,
) -> list[dict[str, Any]]:
service = entry["service"]
freshness = entry.get("evidence_freshness", {})
current_keys = current_evidence_keys(entry)
results: list[dict[str, Any]] = []
for key in sorted(ADVERSARIAL_LEVELS & current_keys):
if key not in freshness:
results.append(
finding(
source=source,
service=service,
finding_class="evidence-freshness-unknown",
severity="high",
control=key,
owner="unknown",
evidence_state="unknown",
reason=f"Current adversarial claim {key} has no authoritative evidence_freshness entry.",
recommended_action=(
f"Declare the {key} observation, expiry, bounded scope, responsible repository, "
"and replacement action; do not infer freshness from prose."
),
)
)
for key in sorted(current_keys & set(freshness)):
metadata = freshness[key]
valid_until_raw = metadata.get("valid_until")
if not valid_until_raw:
continue
valid_until = parse_timestamp(valid_until_raw)
owner = metadata["responsible_repo"]
if as_of > valid_until:
results.append(
finding(
source=source,
service=service,
finding_class="evidence-expired",
severity="high",
control=key,
owner=owner,
evidence_state="expired",
due=format_timestamp(valid_until),
reason=f"Evidence for {key} expired before {format_timestamp(as_of)}.",
recommended_action=metadata["remediation"],
)
)
continue
remaining = valid_until - as_of
if remaining <= dt.timedelta(days=horizon_days):
results.append(
finding(
source=source,
service=service,
finding_class="evidence-due-soon",
severity="medium",
control=key,
owner=owner,
evidence_state="due-soon",
due=format_timestamp(valid_until),
reason=(
f"Evidence for {key} expires within the {horizon_days}-day horizon "
f"at {format_timestamp(valid_until)}."
),
recommended_action=metadata["remediation"],
)
)
return results
def evaluate_entry(
source: str,
entry: dict[str, Any],
as_of: dt.datetime,
horizon_days: int,
) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
service = entry["service"]
posture = entry["tenancy"]
posture_owner = entry.get("responsible_repo", "unknown")
posture_review = review_finding(
source=source,
service=service,
owner=posture_owner,
control="tenancy-posture",
due=dt.date.fromisoformat(posture["review_due"]),
as_of=as_of,
horizon_days=horizon_days,
prefix="posture",
)
if posture_review:
results.append(posture_review)
zones = entry.get("zones")
if zones:
zone_review = review_finding(
source=source,
service=service,
owner=zones["responsible_party"],
control="security-zones",
due=dt.date.fromisoformat(zones["review_due"]),
as_of=as_of,
horizon_days=horizon_days,
prefix="zone",
)
if zone_review:
results.append(zone_review)
results.extend(
evidence_findings(
source=source,
entry=entry,
as_of=as_of,
horizon_days=horizon_days,
)
)
current = posture["current"]
for axis, level in sorted(posture.get("implemented", {}).items()):
current_level = current[axis]
results.append(
finding(
source=source,
service=service,
finding_class="implemented-not-evidenced",
severity="medium",
control=f"{axis}{level}",
owner=posture_owner,
evidence_state="implemented",
reason=f"{axis}{level} is implemented while the current evidenced level is {axis}{current_level}.",
recommended_action=f"Produce the canon-required evidence for {axis}{level} or remove the implemented claim.",
)
)
for axis, description in sorted(posture.get("gap", {}).items()):
results.append(
finding(
source=source,
service=service,
finding_class="declared-gap",
severity="low",
control=axis,
owner=posture_owner,
evidence_state="gap",
reason=description,
recommended_action=f"Review the declared {axis}-axis gap and retain, schedule, or resolve it explicitly.",
)
)
return results
def report_digest(report_without_digest: dict[str, Any]) -> str:
encoded = json.dumps(
report_without_digest,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
def build_report(
paths: list[pathlib.Path],
*,
as_of: dt.datetime,
horizon_days: int,
fail_on: str,
) -> dict[str, Any]:
tenancy_schema = json.loads(TENANCY_SCHEMA.read_text(encoding="utf-8"))
validation_errors: list[str] = []
findings: list[dict[str, Any]] = []
for path in paths:
source = source_identifier(path)
try:
document = load_document(path)
except Exception as exc:
validation_errors.append(f"{path}: {exc}")
continue
errors = TENANCY_VALIDATOR.validate(path, tenancy_schema)
if errors:
validation_errors.extend(errors)
continue
for entry in service_entries(document):
findings.extend(evaluate_entry(source, entry, as_of, horizon_days))
findings.sort(
key=lambda item: (
item["source"],
item["service"],
item["class"],
item["control"],
item.get("due", ""),
)
)
counts = Counter(item["severity"] for item in findings)
threshold_failed = False
if fail_on != "none":
threshold = SEVERITY_RANK[fail_on]
threshold_failed = any(
SEVERITY_RANK[item["severity"]] >= threshold for item in findings
)
report: dict[str, Any] = {
"apiVersion": API_VERSION,
"kind": KIND,
"ok": not validation_errors and not threshold_failed,
"as_of": format_timestamp(as_of),
"horizon_days": horizon_days,
"fail_on": fail_on,
"automation": {
"mode": "proposal-only",
"external_write_permitted": False,
"policy_mutation_permitted": False,
"declaration_mutation_permitted": False,
},
"summary": {
"total": len(findings),
"high": counts["high"],
"medium": counts["medium"],
"low": counts["low"],
"unknown_owner": sum(item["owner"] == "unknown" for item in findings),
},
"validation_errors": sorted(validation_errors),
"findings": findings,
}
report["report_digest"] = report_digest(report)
report_schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8"))
jsonschema.Draft202012Validator(
report_schema,
format_checker=jsonschema.FormatChecker(),
).validate(report)
return report
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Emit deterministic, proposal-only NetKingdom posture feedback."
)
parser.add_argument("declarations", nargs="+", type=pathlib.Path)
parser.add_argument("--as-of", required=True, help="RFC 3339 timestamp with explicit offset")
parser.add_argument("--horizon-days", type=int, default=30)
parser.add_argument("--fail-on", choices=("none", "low", "medium", "high"), default="high")
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
if args.horizon_days < 0:
raise SystemExit("--horizon-days must be non-negative")
try:
as_of = parse_timestamp(args.as_of)
except ValueError as exc:
raise SystemExit(f"invalid --as-of: {exc}") from exc
report = build_report(
args.declarations,
as_of=as_of,
horizon_days=args.horizon_days,
fail_on=args.fail_on,
)
print(json.dumps(report, indent=2, sort_keys=True))
if report["validation_errors"]:
return 2
return 0 if report["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -0,0 +1,261 @@
from __future__ import annotations
import copy
import importlib.util
import json
import pathlib
import sys
import yaml
TOOL_PATH = pathlib.Path(__file__).resolve().parents[1] / "posture_feedback.py"
SPEC = importlib.util.spec_from_file_location("posture_feedback", TOOL_PATH)
feedback = importlib.util.module_from_spec(SPEC)
assert SPEC and SPEC.loader
sys.modules[SPEC.name] = feedback
SPEC.loader.exec_module(feedback)
def declaration(service="example") -> dict:
return {
"schema_version": "0.1",
"framework": "netkingdom-tenancy-posture",
"service": service,
"role": "test-service",
"responsible_repo": "example-owner",
"tenancy": {
"current": {"I": 1, "A": 1, "E": 0, "P": 0, "R": 1, "V": 0},
"target": {"I": 1, "A": 1, "E": 0, "P": 0, "R": 1, "V": 0},
"reviewed": "2026-08-01",
"review_due": "2026-12-31",
"service_class": "interactive",
"reason": {
"I": "floor explained",
"A": "floor explained",
"E": "floor explained",
"P": "floor explained",
"R": "floor explained",
"V": "floor explained",
},
},
}
def write_declaration(tmp_path: pathlib.Path, document: dict, name="tenancy.yaml") -> pathlib.Path:
path = tmp_path / name
path.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8")
return path
def build(path: pathlib.Path, *, as_of="2026-08-23T12:00:00Z", horizon=14, fail_on="none"):
return feedback.build_report(
[path],
as_of=feedback.parse_timestamp(as_of),
horizon_days=horizon,
fail_on=fail_on,
)
def classes(report: dict) -> list[str]:
return [item["class"] for item in report["findings"]]
def test_checked_in_reference_emits_five_proposals_and_no_authority():
repo_root = pathlib.Path(__file__).resolve().parents[3]
path = repo_root / "examples/posture-feedback/expired-e2.yaml"
report = build(path)
assert report["summary"] == {
"total": 5,
"high": 1,
"medium": 3,
"low": 1,
"unknown_owner": 0,
}
assert report["automation"] == {
"mode": "proposal-only",
"external_write_permitted": False,
"policy_mutation_permitted": False,
"declaration_mutation_permitted": False,
}
def test_missing_adversarial_freshness_is_unknown_without_owner_inference(tmp_path):
document = declaration()
document["tenancy"]["current"]["E"] = 2
document["tenancy"]["target"]["E"] = 2
document["tenancy"]["reason"].pop("E")
document["evidence"] = {"E2": "docs/evidence/e2.md"}
path = write_declaration(tmp_path, document)
report = build(path)
finding = next(item for item in report["findings"] if item["class"] == "evidence-freshness-unknown")
assert finding["owner"] == "unknown"
assert finding["evidence_state"] == "unknown"
def test_exact_evidence_expiry_is_still_valid_but_due(tmp_path):
document = declaration()
document["tenancy"]["current"]["E"] = 2
document["tenancy"]["target"]["E"] = 2
document["tenancy"]["reason"].pop("E")
document["evidence"] = {"E2": "docs/evidence/e2.md"}
document["evidence_freshness"] = {
"E2": {
"kind": "adversarial",
"observed_at": "2026-08-22T12:00:00Z",
"valid_until": "2026-08-23T12:00:00Z",
"responsible_repo": "evidence-owner",
"scope": "bounded",
"remediation": "repeat",
}
}
path = write_declaration(tmp_path, document)
report = build(path, horizon=0)
assert "evidence-expired" not in classes(report)
assert "evidence-due-soon" in classes(report)
def test_review_due_date_expires_after_utc_calendar_day(tmp_path):
document = declaration()
document["tenancy"]["review_due"] = "2026-08-23"
path = write_declaration(tmp_path, document)
due_today = build(path, as_of="2026-08-23T23:59:59Z", horizon=0)
overdue = build(path, as_of="2026-08-24T00:00:00Z", horizon=0)
assert "posture-review-due-soon" in classes(due_today)
assert "posture-review-overdue" not in classes(due_today)
assert "posture-review-overdue" in classes(overdue)
def test_posture_owner_is_unknown_when_not_declared(tmp_path):
document = declaration()
del document["responsible_repo"]
document["tenancy"]["review_due"] = "2026-08-01"
path = write_declaration(tmp_path, document)
report = build(path)
finding = next(item for item in report["findings"] if item["class"] == "posture-review-overdue")
assert finding["owner"] == "unknown"
assert report["summary"]["unknown_owner"] == 1
def test_multi_service_declaration_is_evaluated_per_service(tmp_path):
first = declaration("first")
second = declaration("second")
for entry in (first, second):
entry.pop("schema_version")
entry.pop("framework")
first["tenancy"]["gap"] = {"A": "first gap"}
second["tenancy"]["gap"] = {"V": "second gap"}
document = {
"schema_version": "0.1",
"framework": "netkingdom-tenancy-posture",
"services": [first, second],
}
path = write_declaration(tmp_path, document)
report = build(path)
assert {(item["service"], item["control"]) for item in report["findings"]} == {
("first", "A"),
("second", "V"),
}
def test_mechanical_evidence_without_expiry_creates_no_freshness_finding(tmp_path):
document = declaration()
document["tenancy"]["current"]["A"] = 2
document["tenancy"]["target"]["A"] = 2
document["tenancy"]["reason"].pop("A")
document["evidence"] = {"A2": "tests/authorization.py"}
document["evidence_freshness"] = {
"A2": {
"kind": "mechanical",
"observed_at": "2026-08-22T12:00:00Z",
"responsible_repo": "example-owner",
"scope": "continuous test",
"remediation": "repair the test",
}
}
path = write_declaration(tmp_path, document)
report = build(path)
assert not any(item["class"].startswith("evidence-") for item in report["findings"])
def test_report_digest_is_stable_for_identical_inputs(tmp_path):
path = write_declaration(tmp_path, declaration())
first = build(path)
second = build(path)
assert first == second
assert first["report_digest"].startswith("sha256:")
def test_workspace_source_identifier_is_portable_and_not_an_owner_inference():
repo_root = pathlib.Path(__file__).resolve().parents[3]
path = repo_root / "examples/posture-feedback/expired-e2.yaml"
report = build(path)
assert {item["source"] for item in report["findings"]} == {
"net-kingdom/examples/posture-feedback/expired-e2.yaml"
}
gap = next(item for item in report["findings"] if item["class"] == "declared-gap")
zone = next(item for item in report["findings"] if item["class"].startswith("zone-"))
assert gap["owner"] == "net-kingdom"
assert zone["owner"] == "team:platform-security"
assert all(item["owner"] != item["source"] for item in report["findings"])
def test_fail_on_threshold_changes_ok_not_findings(tmp_path):
document = declaration()
document["tenancy"]["gap"] = {"R": "declared gap"}
path = write_declaration(tmp_path, document)
report_only = build(path, fail_on="none")
failing = build(path, fail_on="low")
assert report_only["ok"] is True
assert failing["ok"] is False
assert report_only["findings"] == failing["findings"]
def test_invalid_declaration_is_reported_and_cli_exits_two(tmp_path, capsys):
document = declaration()
del document["role"]
path = write_declaration(tmp_path, document)
exit_code = feedback.main(
[
"--as-of",
"2026-08-23T12:00:00Z",
"--fail-on",
"none",
str(path),
]
)
payload = json.loads(capsys.readouterr().out)
assert exit_code == 2
assert payload["ok"] is False
assert payload["validation_errors"]
def test_timestamp_without_offset_is_rejected():
try:
feedback.parse_timestamp("2026-08-23T12:00:00")
except ValueError as exc:
assert "explicit UTC offset" in str(exc)
else:
raise AssertionError("timezone-naive timestamp was accepted")

View file

@ -89,6 +89,41 @@ class SemanticValidationTests(unittest.TestCase):
document["tenancy"]["review_due"] = "2026-08-16"
self.assertIn("review_due precedes reviewed", self.validate(document)[0])
def test_evidence_freshness_must_reference_evidence_key(self) -> None:
document = declaration()
document["evidence_freshness"] = {
"E2": {
"kind": "adversarial",
"observed_at": "2026-08-22T22:10:25Z",
"valid_until": "2026-08-23T22:10:25Z",
"responsible_repo": "example",
"scope": "bounded tenant probes",
"remediation": "repeat the bounded run",
}
}
self.assertIn(
"evidence_freshness E2 has no evidence entry",
self.validate(document)[0],
)
def test_evidence_freshness_expiry_must_follow_observation(self) -> None:
document = declaration()
document["evidence"] = {"E2": "docs/evidence/e2.md"}
document["evidence_freshness"] = {
"E2": {
"kind": "adversarial",
"observed_at": "2026-08-22T22:10:25Z",
"valid_until": "2026-08-22T22:10:25Z",
"responsible_repo": "example",
"scope": "bounded tenant probes",
"remediation": "repeat the bounded run",
}
}
self.assertIn(
"valid_until must be after observed_at",
self.validate(document)[0],
)
def test_service_names_are_unique(self) -> None:
entry = declaration()
document = {

View file

@ -67,6 +67,7 @@ def validate_semantics(document: dict[str, Any], path: pathlib.Path) -> list[str
reason = posture.get("reason", {})
gap = posture.get("gap", {})
evidence = entry.get("evidence", {})
evidence_freshness = entry.get("evidence_freshness", {})
reviewed = dt.date.fromisoformat(posture["reviewed"])
review_due = dt.date.fromisoformat(posture["review_due"])
@ -92,6 +93,22 @@ def validate_semantics(document: dict[str, Any], path: pathlib.Path) -> list[str
f"{name}: implemented {axis}{level} must be above current {axis}{current_level}"
)
for key, freshness in evidence_freshness.items():
if key not in evidence:
errors.append(f"{name}: evidence_freshness {key} has no evidence entry")
observed_at = dt.datetime.fromisoformat(
freshness["observed_at"].replace("Z", "+00:00")
)
valid_until_raw = freshness.get("valid_until")
if valid_until_raw:
valid_until = dt.datetime.fromisoformat(
valid_until_raw.replace("Z", "+00:00")
)
if valid_until <= observed_at:
errors.append(
f"{name}: evidence_freshness {key} valid_until must be after observed_at"
)
provider = entry.get("provider", {})
for axis, reach in provider.get("axes", {}).items():
available = reach["available"]