Require explicit cadence profile assessment and correct contract pins
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e77d-47a4-7771-8e34-7339c7fac0e4
This commit is contained in:
parent
9383b94019
commit
36303d25a3
7 changed files with 281 additions and 20 deletions
|
|
@ -9,6 +9,56 @@ classes passed with `--load-bearing`, `--rare-load-bearing`, and
|
|||
`--attributive` come from the source's authoritative inventory; the checker
|
||||
does not infer them from names, payloads, or observed traffic.
|
||||
|
||||
## Assessment modes and results
|
||||
|
||||
A security-profile assessment requires at least one source-owned class
|
||||
assertion. For example, to check local-identity's documented rare classes:
|
||||
|
||||
```bash
|
||||
python3 tools/emission-cadence-profile/emission_cadence_profile.py \
|
||||
--contract-schema ../info-tech-canon/infospace/schemas/emission-cadence.schema.yaml \
|
||||
--rare-load-bearing serve/token.token_issued \
|
||||
--rare-load-bearing revoke-token \
|
||||
local-identity/emission-cadence.yaml
|
||||
```
|
||||
|
||||
This declaration currently fails both heartbeat obligations. To intentionally
|
||||
check only its structure against the imported JSON Schema:
|
||||
|
||||
```bash
|
||||
python3 tools/emission-cadence-profile/emission_cadence_profile.py \
|
||||
--contract-schema ../info-tech-canon/infospace/schemas/emission-cadence.schema.yaml \
|
||||
--schema-only local-identity/emission-cadence.yaml
|
||||
```
|
||||
|
||||
| Invocation/result | `contract_valid` | `profile_assessed` | `conformant` | Exit |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Valid schema, no inventory, default mode | true | false | null | 2 |
|
||||
| Valid schema, explicit `--schema-only` | true | false | null | 0 |
|
||||
| Invalid schema/declaration, either mode | false | false | null | 1 |
|
||||
| Supplied inventory, profile passes | true | true | true | 0 |
|
||||
| Supplied inventory, profile fails | true | true | false | 1 |
|
||||
|
||||
`assessment_scope` is `schema-only`, `inventory-missing`, or
|
||||
`supplied-inventory`. The report includes the exact sorted `inventory`
|
||||
assertions. Profile results cover only those assertions: the checker cannot
|
||||
prove that the caller supplied a complete inventory or that events are emitted
|
||||
and observed. `contract_valid` means JSON Schema validity, not every semantic
|
||||
rule in the generic standard; duplicate source IDs are checked during profile
|
||||
assessment.
|
||||
|
||||
`--schema-only` cannot be combined with class assertions or `--fail-on-should`.
|
||||
Blank class arguments are rejected. Invalid CLI options and unreadable input
|
||||
also exit 2. SHOULD findings remain advisory unless `--fail-on-should` is used.
|
||||
|
||||
**Compatibility:** callers that previously omitted class arguments must now
|
||||
choose schema-only validation or supply an inventory. `conformant` can be null;
|
||||
automation claiming profile success must require `profile_assessed == true`
|
||||
and `conformant == true`, rather than merely checking for no findings or an
|
||||
exit code of zero. No evidence classification is inferred from the document.
|
||||
|
||||
## Verification
|
||||
|
||||
Run its tests with:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -348,10 +348,17 @@ def build_report(
|
|||
rare_load_bearing: set[str],
|
||||
attributive: set[str],
|
||||
fail_on_should: bool = False,
|
||||
schema_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
inventory_supplied = bool(load_bearing or rare_load_bearing or attributive)
|
||||
if schema_only and (inventory_supplied or fail_on_should):
|
||||
raise ValueError("schema-only validation cannot include profile options")
|
||||
if any(not name.strip() for name in load_bearing | rare_load_bearing | attributive):
|
||||
raise ValueError("inventory event classes must not be blank")
|
||||
findings = _schema_findings(contract_schema, declaration)
|
||||
contract_valid = not findings
|
||||
if contract_valid:
|
||||
profile_assessed = contract_valid and inventory_supplied and not schema_only
|
||||
if profile_assessed:
|
||||
findings.extend(
|
||||
evaluate_profile(
|
||||
declaration,
|
||||
|
|
@ -362,17 +369,37 @@ def build_report(
|
|||
)
|
||||
must_count = sum(item.level == "MUST" for item in findings)
|
||||
should_count = sum(item.level == "SHOULD" for item in findings)
|
||||
assessment_scope = "supplied-inventory" if inventory_supplied else "inventory-missing"
|
||||
if schema_only:
|
||||
assessment_scope = "schema-only"
|
||||
return {
|
||||
"profile": PROFILE_ID,
|
||||
"contract_schema": contract_schema_path,
|
||||
"declaration": declaration_path,
|
||||
"contract_valid": contract_valid,
|
||||
"conformant": must_count == 0 and (not fail_on_should or should_count == 0),
|
||||
"profile_assessed": profile_assessed,
|
||||
"assessment_scope": assessment_scope,
|
||||
"inventory": {
|
||||
"load_bearing": sorted(load_bearing),
|
||||
"rare_load_bearing": sorted(rare_load_bearing),
|
||||
"attributive": sorted(attributive),
|
||||
},
|
||||
"conformant": (
|
||||
must_count == 0 and (not fail_on_should or should_count == 0)
|
||||
if profile_assessed
|
||||
else None
|
||||
),
|
||||
"summary": {"must": must_count, "should": should_count},
|
||||
"findings": [asdict(item) for item in findings],
|
||||
}
|
||||
|
||||
|
||||
def _event_class(value: str) -> str:
|
||||
if not value.strip():
|
||||
raise argparse.ArgumentTypeError("event class must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate an imported emission-cadence declaration against the NetKingdom profile."
|
||||
|
|
@ -380,16 +407,40 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|||
parser.add_argument("declaration", type=Path)
|
||||
parser.add_argument("--contract-schema", required=True, type=Path)
|
||||
parser.add_argument(
|
||||
"--load-bearing", action="append", default=[], metavar="EVENT_CLASS"
|
||||
"--schema-only",
|
||||
action="store_true",
|
||||
help="Validate only the supplied JSON Schema; do not assess security-profile conformance.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rare-load-bearing", action="append", default=[], metavar="EVENT_CLASS"
|
||||
"--load-bearing",
|
||||
action="append",
|
||||
default=[],
|
||||
type=_event_class,
|
||||
metavar="EVENT_CLASS",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--attributive", action="append", default=[], metavar="EVENT_CLASS"
|
||||
"--rare-load-bearing",
|
||||
action="append",
|
||||
default=[],
|
||||
type=_event_class,
|
||||
metavar="EVENT_CLASS",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--attributive",
|
||||
action="append",
|
||||
default=[],
|
||||
type=_event_class,
|
||||
metavar="EVENT_CLASS",
|
||||
)
|
||||
parser.add_argument("--fail-on-should", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
args = parser.parse_args(argv)
|
||||
if args.schema_only and (
|
||||
args.load_bearing or args.rare_load_bearing or args.attributive or args.fail_on_should
|
||||
):
|
||||
parser.error(
|
||||
"--schema-only cannot be combined with profile inventory or --fail-on-should"
|
||||
)
|
||||
return args
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
|
|
@ -409,8 +460,15 @@ def main(argv: list[str] | None = None) -> int:
|
|||
rare_load_bearing=set(args.rare_load_bearing),
|
||||
attributive=set(args.attributive),
|
||||
fail_on_should=args.fail_on_should,
|
||||
schema_only=args.schema_only,
|
||||
)
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
if not report["contract_valid"]:
|
||||
return 1
|
||||
if args.schema_only:
|
||||
return 0
|
||||
if not report["profile_assessed"]:
|
||||
return 2
|
||||
return 0 if report["conformant"] else 1
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
|
@ -92,14 +93,86 @@ def test_valid_rare_load_bearing_requires_both_positive_controls() -> None:
|
|||
result = report(declaration(rare_entry()), rare={"audit.deny"})
|
||||
|
||||
assert result["contract_valid"] is True
|
||||
assert result["profile_assessed"] is True
|
||||
assert result["inventory"]["rare_load_bearing"] == ["audit.deny"]
|
||||
assert result["conformant"] is True
|
||||
assert result["findings"] == []
|
||||
|
||||
|
||||
def test_omitted_inventory_does_not_claim_profile_conformance() -> None:
|
||||
item = rare_entry()
|
||||
del item["heartbeat"]
|
||||
result = report(declaration(item))
|
||||
|
||||
assert result["contract_valid"] is True
|
||||
assert result["profile_assessed"] is False
|
||||
assert result["conformant"] is None
|
||||
assert result["assessment_scope"] == "inventory-missing"
|
||||
assert result["findings"] == [] # Rarity is never inferred from the entry.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options,valid,exit_code,scope",
|
||||
[
|
||||
([], True, 2, "inventory-missing"),
|
||||
(["--schema-only"], True, 0, "schema-only"),
|
||||
(["--schema-only"], False, 1, "schema-only"),
|
||||
([], False, 1, "inventory-missing"),
|
||||
],
|
||||
)
|
||||
def test_cli_unassessed_results(tmp_path, capsys, options, valid, exit_code, scope):
|
||||
schema = tmp_path / "schema.yaml"
|
||||
document = tmp_path / "declaration.yaml"
|
||||
schema.write_text(yaml.safe_dump(CONTRACT_SCHEMA))
|
||||
item = rare_entry()
|
||||
del item["heartbeat"]
|
||||
document.write_text(yaml.safe_dump(declaration(item) if valid else {}))
|
||||
|
||||
assert (
|
||||
profile.main([str(document), "--contract-schema", str(schema), *options])
|
||||
== exit_code
|
||||
)
|
||||
result = json.loads(capsys.readouterr().out)
|
||||
assert result["contract_valid"] is valid
|
||||
assert result["profile_assessed"] is False
|
||||
assert result["conformant"] is None
|
||||
assert result["assessment_scope"] == scope
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
["--load-bearing", "audit.deny"],
|
||||
["--rare-load-bearing", "audit.deny"],
|
||||
["--attributive", "audit.allow"],
|
||||
["--fail-on-should"],
|
||||
],
|
||||
)
|
||||
def test_schema_only_refuses_profile_options(options):
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
profile.parse_args(
|
||||
["source.yaml", "--contract-schema", "schema.yaml", "--schema-only", *options]
|
||||
)
|
||||
assert exc.value.code == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"option", ["--load-bearing", "--rare-load-bearing", "--attributive"]
|
||||
)
|
||||
def test_blank_class_is_not_an_inventory(option):
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
profile.parse_args(
|
||||
["source.yaml", "--contract-schema", "schema.yaml", option, " "]
|
||||
)
|
||||
assert exc.value.code == 2
|
||||
|
||||
|
||||
def test_contract_validation_runs_before_profile() -> None:
|
||||
result = report({"source": "example"}, rare={"audit.deny"})
|
||||
|
||||
assert result["contract_valid"] is False
|
||||
assert result["profile_assessed"] is False
|
||||
assert result["conformant"] is None
|
||||
assert codes(result) == {"contract-validation-failed"}
|
||||
assert "load-bearing-cadence-missing" not in codes(result)
|
||||
|
||||
|
|
@ -254,8 +327,6 @@ def test_duplicate_source_ids_fail_even_with_distinct_event_classes(upstream_sch
|
|||
def test_should_policy_and_cli(tmp_path, capsys):
|
||||
schema = tmp_path / "schema.json"
|
||||
document = tmp_path / "declaration.yaml"
|
||||
import json
|
||||
|
||||
schema.write_text(json.dumps(CONTRACT_SCHEMA))
|
||||
document.write_text(json.dumps(declaration()))
|
||||
args = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue