feat(posture): add deterministic feedback proposals
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02929-244b-7391-b933-c04010e8eedb
This commit is contained in:
parent
dc8da422f8
commit
cfc9e7d0cb
19 changed files with 1428 additions and 16 deletions
22
tools/posture-feedback/README.md
Normal file
22
tools/posture-feedback/README.md
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Posture Feedback Evaluator
|
||||
|
||||
This tool implements the proposal-only feedback boundary in
|
||||
`canon/standards/posture-feedback_v0.1.md`. It reads tenancy declarations and
|
||||
emits deterministic JSON. It never writes State Hub, edits a declaration,
|
||||
changes policy, or executes remediation.
|
||||
|
||||
Evaluate the checked-in reference at an explicit time:
|
||||
|
||||
```bash
|
||||
uv run tools/posture-feedback/posture_feedback.py \
|
||||
--as-of 2026-08-23T12:00:00Z \
|
||||
--horizon-days 14 \
|
||||
--fail-on none \
|
||||
examples/posture-feedback/expired-e2.yaml
|
||||
```
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
uv run pytest tools/posture-feedback/tests
|
||||
```
|
||||
440
tools/posture-feedback/posture_feedback.py
Normal file
440
tools/posture-feedback/posture_feedback.py
Normal file
|
|
@ -0,0 +1,440 @@
|
|||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# dependencies = ["jsonschema>=4.23,<5", "PyYAML>=6,<7"]
|
||||
# ///
|
||||
"""Emit deterministic, proposal-only feedback for tenancy posture declarations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
import jsonschema
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
TENANCY_SCHEMA = ROOT / "canon/schemas/tenancy-posture_v0.1.schema.json"
|
||||
REPORT_SCHEMA = ROOT / "canon/schemas/posture-feedback-report_v0.1.schema.json"
|
||||
TENANCY_VALIDATOR_PATH = ROOT / "tools/tenancy-posture/validate.py"
|
||||
API_VERSION = "netkingdom.io/posture-feedback/v0.1"
|
||||
KIND = "PostureFeedbackReport"
|
||||
ADVERSARIAL_LEVELS = {"E2", "R4", "V2", "V3", "V4"}
|
||||
SEVERITY_RANK = {"low": 1, "medium": 2, "high": 3}
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"tenancy_posture_validator_for_feedback", TENANCY_VALIDATOR_PATH
|
||||
)
|
||||
assert SPEC and SPEC.loader
|
||||
TENANCY_VALIDATOR = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(TENANCY_VALIDATOR)
|
||||
|
||||
|
||||
def parse_timestamp(value: str) -> dt.datetime:
|
||||
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise ValueError("timestamp must include an explicit UTC offset")
|
||||
return parsed.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
def format_timestamp(value: dt.datetime) -> str:
|
||||
return value.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def load_document(path: pathlib.Path) -> dict[str, Any]:
|
||||
document = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(document, dict):
|
||||
raise ValueError("document must be a YAML object")
|
||||
return document
|
||||
|
||||
|
||||
def service_entries(document: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if "services" in document:
|
||||
return document["services"]
|
||||
return [document]
|
||||
|
||||
|
||||
def source_identifier(path: pathlib.Path) -> str:
|
||||
"""Return a portable declaration id without treating the path as ownership."""
|
||||
resolved = path.resolve()
|
||||
try:
|
||||
return resolved.relative_to(ROOT.parent).as_posix()
|
||||
except ValueError:
|
||||
return resolved.as_posix()
|
||||
|
||||
|
||||
def finding_id(
|
||||
source: str,
|
||||
service: str,
|
||||
finding_class: str,
|
||||
control: str,
|
||||
due: str | None,
|
||||
) -> str:
|
||||
identity = "\x1f".join((source, service, finding_class, control, due or ""))
|
||||
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16]
|
||||
return f"NKFB-{digest}"
|
||||
|
||||
|
||||
def finding(
|
||||
*,
|
||||
source: str,
|
||||
service: str,
|
||||
finding_class: str,
|
||||
severity: str,
|
||||
control: str,
|
||||
owner: str,
|
||||
evidence_state: str,
|
||||
reason: str,
|
||||
recommended_action: str,
|
||||
due: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result = {
|
||||
"id": finding_id(source, service, finding_class, control, due),
|
||||
"class": finding_class,
|
||||
"severity": severity,
|
||||
"source": source,
|
||||
"service": service,
|
||||
"control": control,
|
||||
"owner": owner,
|
||||
"evidence_state": evidence_state,
|
||||
"reason": reason,
|
||||
"recommended_action": recommended_action,
|
||||
}
|
||||
if due is not None:
|
||||
result["due"] = due
|
||||
return result
|
||||
|
||||
|
||||
def review_finding(
|
||||
*,
|
||||
source: str,
|
||||
service: str,
|
||||
owner: str,
|
||||
control: str,
|
||||
due: dt.date,
|
||||
as_of: dt.datetime,
|
||||
horizon_days: int,
|
||||
prefix: str,
|
||||
) -> dict[str, Any] | None:
|
||||
due_text = due.isoformat()
|
||||
if as_of.date() > due:
|
||||
return finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class=f"{prefix}-review-overdue",
|
||||
severity="high",
|
||||
control=control,
|
||||
owner=owner,
|
||||
evidence_state="overdue",
|
||||
due=due_text,
|
||||
reason=f"{control} review was due on {due_text} and is overdue at {format_timestamp(as_of)}.",
|
||||
recommended_action=f"Review {control}, update its declaration and evidence, and retain the review record.",
|
||||
)
|
||||
days = (due - as_of.date()).days
|
||||
if days <= horizon_days:
|
||||
return finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class=f"{prefix}-review-due-soon",
|
||||
severity="medium",
|
||||
control=control,
|
||||
owner=owner,
|
||||
evidence_state="due-soon",
|
||||
due=due_text,
|
||||
reason=f"{control} review is due in {days} day(s), within the {horizon_days}-day horizon.",
|
||||
recommended_action=f"Schedule and evidence the {control} review before {due_text}.",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def current_evidence_keys(entry: dict[str, Any]) -> set[str]:
|
||||
current = entry["tenancy"]["current"]
|
||||
return {
|
||||
f"{axis}{level}"
|
||||
for axis, level in current.items()
|
||||
if isinstance(level, int)
|
||||
}
|
||||
|
||||
|
||||
def evidence_findings(
|
||||
*,
|
||||
source: str,
|
||||
entry: dict[str, Any],
|
||||
as_of: dt.datetime,
|
||||
horizon_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
service = entry["service"]
|
||||
freshness = entry.get("evidence_freshness", {})
|
||||
current_keys = current_evidence_keys(entry)
|
||||
results: list[dict[str, Any]] = []
|
||||
|
||||
for key in sorted(ADVERSARIAL_LEVELS & current_keys):
|
||||
if key not in freshness:
|
||||
results.append(
|
||||
finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class="evidence-freshness-unknown",
|
||||
severity="high",
|
||||
control=key,
|
||||
owner="unknown",
|
||||
evidence_state="unknown",
|
||||
reason=f"Current adversarial claim {key} has no authoritative evidence_freshness entry.",
|
||||
recommended_action=(
|
||||
f"Declare the {key} observation, expiry, bounded scope, responsible repository, "
|
||||
"and replacement action; do not infer freshness from prose."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
for key in sorted(current_keys & set(freshness)):
|
||||
metadata = freshness[key]
|
||||
valid_until_raw = metadata.get("valid_until")
|
||||
if not valid_until_raw:
|
||||
continue
|
||||
valid_until = parse_timestamp(valid_until_raw)
|
||||
owner = metadata["responsible_repo"]
|
||||
if as_of > valid_until:
|
||||
results.append(
|
||||
finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class="evidence-expired",
|
||||
severity="high",
|
||||
control=key,
|
||||
owner=owner,
|
||||
evidence_state="expired",
|
||||
due=format_timestamp(valid_until),
|
||||
reason=f"Evidence for {key} expired before {format_timestamp(as_of)}.",
|
||||
recommended_action=metadata["remediation"],
|
||||
)
|
||||
)
|
||||
continue
|
||||
remaining = valid_until - as_of
|
||||
if remaining <= dt.timedelta(days=horizon_days):
|
||||
results.append(
|
||||
finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class="evidence-due-soon",
|
||||
severity="medium",
|
||||
control=key,
|
||||
owner=owner,
|
||||
evidence_state="due-soon",
|
||||
due=format_timestamp(valid_until),
|
||||
reason=(
|
||||
f"Evidence for {key} expires within the {horizon_days}-day horizon "
|
||||
f"at {format_timestamp(valid_until)}."
|
||||
),
|
||||
recommended_action=metadata["remediation"],
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def evaluate_entry(
|
||||
source: str,
|
||||
entry: dict[str, Any],
|
||||
as_of: dt.datetime,
|
||||
horizon_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
service = entry["service"]
|
||||
posture = entry["tenancy"]
|
||||
posture_owner = entry.get("responsible_repo", "unknown")
|
||||
|
||||
posture_review = review_finding(
|
||||
source=source,
|
||||
service=service,
|
||||
owner=posture_owner,
|
||||
control="tenancy-posture",
|
||||
due=dt.date.fromisoformat(posture["review_due"]),
|
||||
as_of=as_of,
|
||||
horizon_days=horizon_days,
|
||||
prefix="posture",
|
||||
)
|
||||
if posture_review:
|
||||
results.append(posture_review)
|
||||
|
||||
zones = entry.get("zones")
|
||||
if zones:
|
||||
zone_review = review_finding(
|
||||
source=source,
|
||||
service=service,
|
||||
owner=zones["responsible_party"],
|
||||
control="security-zones",
|
||||
due=dt.date.fromisoformat(zones["review_due"]),
|
||||
as_of=as_of,
|
||||
horizon_days=horizon_days,
|
||||
prefix="zone",
|
||||
)
|
||||
if zone_review:
|
||||
results.append(zone_review)
|
||||
|
||||
results.extend(
|
||||
evidence_findings(
|
||||
source=source,
|
||||
entry=entry,
|
||||
as_of=as_of,
|
||||
horizon_days=horizon_days,
|
||||
)
|
||||
)
|
||||
|
||||
current = posture["current"]
|
||||
for axis, level in sorted(posture.get("implemented", {}).items()):
|
||||
current_level = current[axis]
|
||||
results.append(
|
||||
finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class="implemented-not-evidenced",
|
||||
severity="medium",
|
||||
control=f"{axis}{level}",
|
||||
owner=posture_owner,
|
||||
evidence_state="implemented",
|
||||
reason=f"{axis}{level} is implemented while the current evidenced level is {axis}{current_level}.",
|
||||
recommended_action=f"Produce the canon-required evidence for {axis}{level} or remove the implemented claim.",
|
||||
)
|
||||
)
|
||||
|
||||
for axis, description in sorted(posture.get("gap", {}).items()):
|
||||
results.append(
|
||||
finding(
|
||||
source=source,
|
||||
service=service,
|
||||
finding_class="declared-gap",
|
||||
severity="low",
|
||||
control=axis,
|
||||
owner=posture_owner,
|
||||
evidence_state="gap",
|
||||
reason=description,
|
||||
recommended_action=f"Review the declared {axis}-axis gap and retain, schedule, or resolve it explicitly.",
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def report_digest(report_without_digest: dict[str, Any]) -> str:
|
||||
encoded = json.dumps(
|
||||
report_without_digest,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
|
||||
|
||||
|
||||
def build_report(
|
||||
paths: list[pathlib.Path],
|
||||
*,
|
||||
as_of: dt.datetime,
|
||||
horizon_days: int,
|
||||
fail_on: str,
|
||||
) -> dict[str, Any]:
|
||||
tenancy_schema = json.loads(TENANCY_SCHEMA.read_text(encoding="utf-8"))
|
||||
validation_errors: list[str] = []
|
||||
findings: list[dict[str, Any]] = []
|
||||
|
||||
for path in paths:
|
||||
source = source_identifier(path)
|
||||
try:
|
||||
document = load_document(path)
|
||||
except Exception as exc:
|
||||
validation_errors.append(f"{path}: {exc}")
|
||||
continue
|
||||
errors = TENANCY_VALIDATOR.validate(path, tenancy_schema)
|
||||
if errors:
|
||||
validation_errors.extend(errors)
|
||||
continue
|
||||
for entry in service_entries(document):
|
||||
findings.extend(evaluate_entry(source, entry, as_of, horizon_days))
|
||||
|
||||
findings.sort(
|
||||
key=lambda item: (
|
||||
item["source"],
|
||||
item["service"],
|
||||
item["class"],
|
||||
item["control"],
|
||||
item.get("due", ""),
|
||||
)
|
||||
)
|
||||
counts = Counter(item["severity"] for item in findings)
|
||||
threshold_failed = False
|
||||
if fail_on != "none":
|
||||
threshold = SEVERITY_RANK[fail_on]
|
||||
threshold_failed = any(
|
||||
SEVERITY_RANK[item["severity"]] >= threshold for item in findings
|
||||
)
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"apiVersion": API_VERSION,
|
||||
"kind": KIND,
|
||||
"ok": not validation_errors and not threshold_failed,
|
||||
"as_of": format_timestamp(as_of),
|
||||
"horizon_days": horizon_days,
|
||||
"fail_on": fail_on,
|
||||
"automation": {
|
||||
"mode": "proposal-only",
|
||||
"external_write_permitted": False,
|
||||
"policy_mutation_permitted": False,
|
||||
"declaration_mutation_permitted": False,
|
||||
},
|
||||
"summary": {
|
||||
"total": len(findings),
|
||||
"high": counts["high"],
|
||||
"medium": counts["medium"],
|
||||
"low": counts["low"],
|
||||
"unknown_owner": sum(item["owner"] == "unknown" for item in findings),
|
||||
},
|
||||
"validation_errors": sorted(validation_errors),
|
||||
"findings": findings,
|
||||
}
|
||||
report["report_digest"] = report_digest(report)
|
||||
report_schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8"))
|
||||
jsonschema.Draft202012Validator(
|
||||
report_schema,
|
||||
format_checker=jsonschema.FormatChecker(),
|
||||
).validate(report)
|
||||
return report
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Emit deterministic, proposal-only NetKingdom posture feedback."
|
||||
)
|
||||
parser.add_argument("declarations", nargs="+", type=pathlib.Path)
|
||||
parser.add_argument("--as-of", required=True, help="RFC 3339 timestamp with explicit offset")
|
||||
parser.add_argument("--horizon-days", type=int, default=30)
|
||||
parser.add_argument("--fail-on", choices=("none", "low", "medium", "high"), default="high")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.horizon_days < 0:
|
||||
raise SystemExit("--horizon-days must be non-negative")
|
||||
try:
|
||||
as_of = parse_timestamp(args.as_of)
|
||||
except ValueError as exc:
|
||||
raise SystemExit(f"invalid --as-of: {exc}") from exc
|
||||
|
||||
report = build_report(
|
||||
args.declarations,
|
||||
as_of=as_of,
|
||||
horizon_days=args.horizon_days,
|
||||
fail_on=args.fail_on,
|
||||
)
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
if report["validation_errors"]:
|
||||
return 2
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
261
tools/posture-feedback/tests/test_posture_feedback.py
Normal file
261
tools/posture-feedback/tests/test_posture_feedback.py
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
TOOL_PATH = pathlib.Path(__file__).resolve().parents[1] / "posture_feedback.py"
|
||||
SPEC = importlib.util.spec_from_file_location("posture_feedback", TOOL_PATH)
|
||||
feedback = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC and SPEC.loader
|
||||
sys.modules[SPEC.name] = feedback
|
||||
SPEC.loader.exec_module(feedback)
|
||||
|
||||
|
||||
def declaration(service="example") -> dict:
|
||||
return {
|
||||
"schema_version": "0.1",
|
||||
"framework": "netkingdom-tenancy-posture",
|
||||
"service": service,
|
||||
"role": "test-service",
|
||||
"responsible_repo": "example-owner",
|
||||
"tenancy": {
|
||||
"current": {"I": 1, "A": 1, "E": 0, "P": 0, "R": 1, "V": 0},
|
||||
"target": {"I": 1, "A": 1, "E": 0, "P": 0, "R": 1, "V": 0},
|
||||
"reviewed": "2026-08-01",
|
||||
"review_due": "2026-12-31",
|
||||
"service_class": "interactive",
|
||||
"reason": {
|
||||
"I": "floor explained",
|
||||
"A": "floor explained",
|
||||
"E": "floor explained",
|
||||
"P": "floor explained",
|
||||
"R": "floor explained",
|
||||
"V": "floor explained",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_declaration(tmp_path: pathlib.Path, document: dict, name="tenancy.yaml") -> pathlib.Path:
|
||||
path = tmp_path / name
|
||||
path.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def build(path: pathlib.Path, *, as_of="2026-08-23T12:00:00Z", horizon=14, fail_on="none"):
|
||||
return feedback.build_report(
|
||||
[path],
|
||||
as_of=feedback.parse_timestamp(as_of),
|
||||
horizon_days=horizon,
|
||||
fail_on=fail_on,
|
||||
)
|
||||
|
||||
|
||||
def classes(report: dict) -> list[str]:
|
||||
return [item["class"] for item in report["findings"]]
|
||||
|
||||
|
||||
def test_checked_in_reference_emits_five_proposals_and_no_authority():
|
||||
repo_root = pathlib.Path(__file__).resolve().parents[3]
|
||||
path = repo_root / "examples/posture-feedback/expired-e2.yaml"
|
||||
|
||||
report = build(path)
|
||||
|
||||
assert report["summary"] == {
|
||||
"total": 5,
|
||||
"high": 1,
|
||||
"medium": 3,
|
||||
"low": 1,
|
||||
"unknown_owner": 0,
|
||||
}
|
||||
assert report["automation"] == {
|
||||
"mode": "proposal-only",
|
||||
"external_write_permitted": False,
|
||||
"policy_mutation_permitted": False,
|
||||
"declaration_mutation_permitted": False,
|
||||
}
|
||||
|
||||
|
||||
def test_missing_adversarial_freshness_is_unknown_without_owner_inference(tmp_path):
|
||||
document = declaration()
|
||||
document["tenancy"]["current"]["E"] = 2
|
||||
document["tenancy"]["target"]["E"] = 2
|
||||
document["tenancy"]["reason"].pop("E")
|
||||
document["evidence"] = {"E2": "docs/evidence/e2.md"}
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report = build(path)
|
||||
|
||||
finding = next(item for item in report["findings"] if item["class"] == "evidence-freshness-unknown")
|
||||
assert finding["owner"] == "unknown"
|
||||
assert finding["evidence_state"] == "unknown"
|
||||
|
||||
|
||||
def test_exact_evidence_expiry_is_still_valid_but_due(tmp_path):
|
||||
document = declaration()
|
||||
document["tenancy"]["current"]["E"] = 2
|
||||
document["tenancy"]["target"]["E"] = 2
|
||||
document["tenancy"]["reason"].pop("E")
|
||||
document["evidence"] = {"E2": "docs/evidence/e2.md"}
|
||||
document["evidence_freshness"] = {
|
||||
"E2": {
|
||||
"kind": "adversarial",
|
||||
"observed_at": "2026-08-22T12:00:00Z",
|
||||
"valid_until": "2026-08-23T12:00:00Z",
|
||||
"responsible_repo": "evidence-owner",
|
||||
"scope": "bounded",
|
||||
"remediation": "repeat",
|
||||
}
|
||||
}
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report = build(path, horizon=0)
|
||||
|
||||
assert "evidence-expired" not in classes(report)
|
||||
assert "evidence-due-soon" in classes(report)
|
||||
|
||||
|
||||
def test_review_due_date_expires_after_utc_calendar_day(tmp_path):
|
||||
document = declaration()
|
||||
document["tenancy"]["review_due"] = "2026-08-23"
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
due_today = build(path, as_of="2026-08-23T23:59:59Z", horizon=0)
|
||||
overdue = build(path, as_of="2026-08-24T00:00:00Z", horizon=0)
|
||||
|
||||
assert "posture-review-due-soon" in classes(due_today)
|
||||
assert "posture-review-overdue" not in classes(due_today)
|
||||
assert "posture-review-overdue" in classes(overdue)
|
||||
|
||||
|
||||
def test_posture_owner_is_unknown_when_not_declared(tmp_path):
|
||||
document = declaration()
|
||||
del document["responsible_repo"]
|
||||
document["tenancy"]["review_due"] = "2026-08-01"
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report = build(path)
|
||||
|
||||
finding = next(item for item in report["findings"] if item["class"] == "posture-review-overdue")
|
||||
assert finding["owner"] == "unknown"
|
||||
assert report["summary"]["unknown_owner"] == 1
|
||||
|
||||
|
||||
def test_multi_service_declaration_is_evaluated_per_service(tmp_path):
|
||||
first = declaration("first")
|
||||
second = declaration("second")
|
||||
for entry in (first, second):
|
||||
entry.pop("schema_version")
|
||||
entry.pop("framework")
|
||||
first["tenancy"]["gap"] = {"A": "first gap"}
|
||||
second["tenancy"]["gap"] = {"V": "second gap"}
|
||||
document = {
|
||||
"schema_version": "0.1",
|
||||
"framework": "netkingdom-tenancy-posture",
|
||||
"services": [first, second],
|
||||
}
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report = build(path)
|
||||
|
||||
assert {(item["service"], item["control"]) for item in report["findings"]} == {
|
||||
("first", "A"),
|
||||
("second", "V"),
|
||||
}
|
||||
|
||||
|
||||
def test_mechanical_evidence_without_expiry_creates_no_freshness_finding(tmp_path):
|
||||
document = declaration()
|
||||
document["tenancy"]["current"]["A"] = 2
|
||||
document["tenancy"]["target"]["A"] = 2
|
||||
document["tenancy"]["reason"].pop("A")
|
||||
document["evidence"] = {"A2": "tests/authorization.py"}
|
||||
document["evidence_freshness"] = {
|
||||
"A2": {
|
||||
"kind": "mechanical",
|
||||
"observed_at": "2026-08-22T12:00:00Z",
|
||||
"responsible_repo": "example-owner",
|
||||
"scope": "continuous test",
|
||||
"remediation": "repair the test",
|
||||
}
|
||||
}
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report = build(path)
|
||||
|
||||
assert not any(item["class"].startswith("evidence-") for item in report["findings"])
|
||||
|
||||
|
||||
def test_report_digest_is_stable_for_identical_inputs(tmp_path):
|
||||
path = write_declaration(tmp_path, declaration())
|
||||
|
||||
first = build(path)
|
||||
second = build(path)
|
||||
|
||||
assert first == second
|
||||
assert first["report_digest"].startswith("sha256:")
|
||||
|
||||
|
||||
def test_workspace_source_identifier_is_portable_and_not_an_owner_inference():
|
||||
repo_root = pathlib.Path(__file__).resolve().parents[3]
|
||||
path = repo_root / "examples/posture-feedback/expired-e2.yaml"
|
||||
|
||||
report = build(path)
|
||||
|
||||
assert {item["source"] for item in report["findings"]} == {
|
||||
"net-kingdom/examples/posture-feedback/expired-e2.yaml"
|
||||
}
|
||||
gap = next(item for item in report["findings"] if item["class"] == "declared-gap")
|
||||
zone = next(item for item in report["findings"] if item["class"].startswith("zone-"))
|
||||
assert gap["owner"] == "net-kingdom"
|
||||
assert zone["owner"] == "team:platform-security"
|
||||
assert all(item["owner"] != item["source"] for item in report["findings"])
|
||||
|
||||
|
||||
def test_fail_on_threshold_changes_ok_not_findings(tmp_path):
|
||||
document = declaration()
|
||||
document["tenancy"]["gap"] = {"R": "declared gap"}
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
report_only = build(path, fail_on="none")
|
||||
failing = build(path, fail_on="low")
|
||||
|
||||
assert report_only["ok"] is True
|
||||
assert failing["ok"] is False
|
||||
assert report_only["findings"] == failing["findings"]
|
||||
|
||||
|
||||
def test_invalid_declaration_is_reported_and_cli_exits_two(tmp_path, capsys):
|
||||
document = declaration()
|
||||
del document["role"]
|
||||
path = write_declaration(tmp_path, document)
|
||||
|
||||
exit_code = feedback.main(
|
||||
[
|
||||
"--as-of",
|
||||
"2026-08-23T12:00:00Z",
|
||||
"--fail-on",
|
||||
"none",
|
||||
str(path),
|
||||
]
|
||||
)
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
|
||||
assert exit_code == 2
|
||||
assert payload["ok"] is False
|
||||
assert payload["validation_errors"]
|
||||
|
||||
|
||||
def test_timestamp_without_offset_is_rejected():
|
||||
try:
|
||||
feedback.parse_timestamp("2026-08-23T12:00:00")
|
||||
except ValueError as exc:
|
||||
assert "explicit UTC offset" in str(exc)
|
||||
else:
|
||||
raise AssertionError("timezone-naive timestamp was accepted")
|
||||
|
|
@ -89,6 +89,41 @@ class SemanticValidationTests(unittest.TestCase):
|
|||
document["tenancy"]["review_due"] = "2026-08-16"
|
||||
self.assertIn("review_due precedes reviewed", self.validate(document)[0])
|
||||
|
||||
def test_evidence_freshness_must_reference_evidence_key(self) -> None:
|
||||
document = declaration()
|
||||
document["evidence_freshness"] = {
|
||||
"E2": {
|
||||
"kind": "adversarial",
|
||||
"observed_at": "2026-08-22T22:10:25Z",
|
||||
"valid_until": "2026-08-23T22:10:25Z",
|
||||
"responsible_repo": "example",
|
||||
"scope": "bounded tenant probes",
|
||||
"remediation": "repeat the bounded run",
|
||||
}
|
||||
}
|
||||
self.assertIn(
|
||||
"evidence_freshness E2 has no evidence entry",
|
||||
self.validate(document)[0],
|
||||
)
|
||||
|
||||
def test_evidence_freshness_expiry_must_follow_observation(self) -> None:
|
||||
document = declaration()
|
||||
document["evidence"] = {"E2": "docs/evidence/e2.md"}
|
||||
document["evidence_freshness"] = {
|
||||
"E2": {
|
||||
"kind": "adversarial",
|
||||
"observed_at": "2026-08-22T22:10:25Z",
|
||||
"valid_until": "2026-08-22T22:10:25Z",
|
||||
"responsible_repo": "example",
|
||||
"scope": "bounded tenant probes",
|
||||
"remediation": "repeat the bounded run",
|
||||
}
|
||||
}
|
||||
self.assertIn(
|
||||
"valid_until must be after observed_at",
|
||||
self.validate(document)[0],
|
||||
)
|
||||
|
||||
def test_service_names_are_unique(self) -> None:
|
||||
entry = declaration()
|
||||
document = {
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ def validate_semantics(document: dict[str, Any], path: pathlib.Path) -> list[str
|
|||
reason = posture.get("reason", {})
|
||||
gap = posture.get("gap", {})
|
||||
evidence = entry.get("evidence", {})
|
||||
evidence_freshness = entry.get("evidence_freshness", {})
|
||||
|
||||
reviewed = dt.date.fromisoformat(posture["reviewed"])
|
||||
review_due = dt.date.fromisoformat(posture["review_due"])
|
||||
|
|
@ -92,6 +93,22 @@ def validate_semantics(document: dict[str, Any], path: pathlib.Path) -> list[str
|
|||
f"{name}: implemented {axis}{level} must be above current {axis}{current_level}"
|
||||
)
|
||||
|
||||
for key, freshness in evidence_freshness.items():
|
||||
if key not in evidence:
|
||||
errors.append(f"{name}: evidence_freshness {key} has no evidence entry")
|
||||
observed_at = dt.datetime.fromisoformat(
|
||||
freshness["observed_at"].replace("Z", "+00:00")
|
||||
)
|
||||
valid_until_raw = freshness.get("valid_until")
|
||||
if valid_until_raw:
|
||||
valid_until = dt.datetime.fromisoformat(
|
||||
valid_until_raw.replace("Z", "+00:00")
|
||||
)
|
||||
if valid_until <= observed_at:
|
||||
errors.append(
|
||||
f"{name}: evidence_freshness {key} valid_until must be after observed_at"
|
||||
)
|
||||
|
||||
provider = entry.get("provider", {})
|
||||
for axis, reach in provider.get("axes", {}).items():
|
||||
available = reach["available"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue