Make profile runtime readiness explicit
Some checks failed
ci / validate (push) Has been cancelled

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a0233b-178d-7162-b92f-31a31ea8ca9b
This commit is contained in:
tegwick 2026-08-23 11:32:23 +02:00
parent a224e71555
commit cac605abd7
19 changed files with 339 additions and 41 deletions

View file

@ -12,6 +12,19 @@ def test_profiles_command_lists_committed_constellations(capsys) -> None:
rows = json.loads(capsys.readouterr().out)
assert {row["rein_id"] for row in rows} == {"rein-aharness", "rein-openweights"}
assert {row["operational_readiness"]["status"] for row in rows} == {
"blocked",
"unverified",
}
def test_profiles_text_output_labels_runtime_readiness(capsys) -> None:
assert main(["profiles"]) == 0
output = capsys.readouterr().out
assert "readiness=blocked" in output
assert "readiness=unverified" in output
assert "executable" not in output
def test_execution_request_example_matches_contract() -> None:
@ -19,7 +32,7 @@ def test_execution_request_example_matches_contract() -> None:
request = ExecutionRequest.model_validate_json(fixture.read_text())
assert request.harness_profile_ref == "harness.agent-dev-local@1.0.0"
assert request.harness_profile_ref == "harness.agent-dev@1.0.0"
assert request.assignment_ref == "role-assignment:agent-7:42"

View file

@ -6,12 +6,14 @@ from sandboxer.models import Reachability, SandboxState, SandboxStatus
from glas_harness.contract import (
ExecutionRequest,
ExecutionSummary,
OperationalReadiness,
Rein,
SandboxHandle,
ToolCall,
ToolResult,
)
from glas_harness.gateway import run_execution, run_task_through_rein
from glas_harness.profiles import ProfileCatalog
PROFILE = "harness.agent-dev-local@1.0.0"
@ -83,12 +85,29 @@ def _request(*, profile: str = PROFILE, report_to_hub: bool = False) -> Executio
)
def _catalog_with_readiness(status: str = "ready") -> ProfileCatalog:
catalog = ProfileCatalog()
profile, _ = catalog.resolve(PROFILE)
readiness = OperationalReadiness(
status=status,
reason="test-only readiness state",
owner="tests",
evidence_ref="test:gateway-readiness",
)
catalog.profiles()[(profile.id, profile.version)] = profile.model_copy(
update={"operational_readiness": readiness}
)
return catalog
def test_run_execution_creates_and_destroys_sandbox() -> None:
manager = MagicMock()
manager.create.return_value = _fake_status()
rein = _FakeRein()
result = run_execution(_request(), rein=rein, manager=manager)
result = run_execution(
_request(), catalog=_catalog_with_readiness(), rein=rein, manager=manager
)
assert rein.calls == [
"start_session",
@ -100,6 +119,7 @@ def test_run_execution_creates_and_destroys_sandbox() -> None:
manager.destroy.assert_called_once_with("sbx1")
assert result.ok is True
assert result.evidence.profile_ref == PROFILE
assert result.evidence.profile_readiness.status == "ready"
assert result.evidence.rein_id == "rein-aharness"
assert result.evidence.commit_sha == "deadbeef"
assert result.evidence.tokens_spent == 123
@ -116,7 +136,9 @@ def test_run_execution_normalizes_execution_failure_and_tears_down() -> None:
raise RuntimeError("boom")
rein = _FailingRein()
result = run_execution(_request(), rein=rein, manager=manager)
result = run_execution(
_request(), catalog=_catalog_with_readiness(), rein=rein, manager=manager
)
assert result.ok is False
assert result.evidence.outcome == "failed"
@ -139,6 +161,35 @@ def test_run_execution_refuses_unknown_profile_before_sandbox() -> None:
manager.create.assert_not_called()
def test_run_execution_refuses_blocked_profile_before_sandbox() -> None:
manager = MagicMock()
result = run_execution(_request(), rein=_FakeRein(), manager=manager)
assert result.ok is False
assert result.evidence.outcome == "refused"
assert result.evidence.failure_stage == "resolution"
assert result.evidence.profile_readiness.status == "blocked"
assert "GLAS-IN-0002" in (result.evidence.error or "")
manager.create.assert_not_called()
def test_run_execution_allows_labeled_unverified_proof_attempt() -> None:
manager = MagicMock()
manager.create.return_value = _fake_status()
result = run_execution(
_request(),
catalog=_catalog_with_readiness("unverified"),
rein=_FakeRein(),
manager=manager,
)
assert result.ok is True
assert result.evidence.profile_readiness.status == "unverified"
manager.create.assert_called_once()
def test_run_execution_refuses_worker_identifier_before_sandbox() -> None:
manager = MagicMock()
request = _request().model_copy(update={"actor": "rein-aharness@railiance01"})
@ -160,7 +211,12 @@ def test_hub_receives_normalized_evidence_without_raw_output() -> None:
manager.create.return_value = _fake_status()
with patch("glas_harness.gateway.hub.post_progress_event", return_value=True) as post:
result = run_execution(_request(report_to_hub=True), rein=_FakeRein(), manager=manager)
result = run_execution(
_request(report_to_hub=True),
catalog=_catalog_with_readiness(),
rein=_FakeRein(),
manager=manager,
)
post.assert_called_once()
detail = post.call_args.kwargs["detail"]
@ -181,7 +237,10 @@ def test_hub_failure_detail_excludes_raw_provider_error() -> None:
with patch("glas_harness.gateway.hub.post_progress_event", return_value=True) as post:
result = run_execution(
_request(report_to_hub=True), rein=_FailingRein(), manager=manager
_request(report_to_hub=True),
catalog=_catalog_with_readiness(),
rein=_FailingRein(),
manager=manager,
)
detail = post.call_args.kwargs["detail"]
@ -201,6 +260,7 @@ def test_wrapper_requires_and_reports_harness_profile() -> None:
description="d",
rein=_FakeRein(),
manager=manager,
catalog=_catalog_with_readiness(),
report_to_hub=False,
)
@ -213,7 +273,12 @@ def test_teardown_failure_is_visible_in_evidence() -> None:
manager.create.return_value = _fake_status()
manager.destroy.side_effect = RuntimeError("cannot teardown")
result = run_execution(_request(), rein=_FakeRein(), manager=manager)
result = run_execution(
_request(),
catalog=_catalog_with_readiness(),
rein=_FakeRein(),
manager=manager,
)
assert result.ok is False
assert result.evidence.failure_stage == "teardown"

View file

@ -71,6 +71,14 @@ def test_committed_catalog_resolves_both_constellations() -> None:
"rein-aharness",
"rein-openweights",
}
assert {
(context.profile.id, context.operational_readiness.status)
for context in contexts
} == {
("harness.agent-dev-local", "blocked"),
("harness.agent-dev-openweights-local", "blocked"),
("harness.agent-dev", "unverified"),
}
profile, descriptor = catalog.resolve(
"harness.agent-dev-openweights-local@1.0.0"
)
@ -89,6 +97,41 @@ def test_committed_catalog_resolves_both_constellations() -> None:
assert rein.budget_tokens == 60000
def test_ready_profile_requires_an_evidence_reference(tmp_path) -> None:
profiles = tmp_path / "profiles"
reins = tmp_path / "reins"
profiles.mkdir()
reins.mkdir()
_write(
profiles,
"profile.yaml",
_profile(
extra="""operational_readiness:
status: ready
reason: proof completed
owner: tests
"""
),
)
_write(reins, "rein.yaml", _rein())
with pytest.raises(ProfileError, match="ready operational status requires evidence_ref"):
ProfileCatalog(profiles, reins).profiles()
def test_undeclared_readiness_defaults_to_unverified(tmp_path) -> None:
profiles = tmp_path / "profiles"
reins = tmp_path / "reins"
profiles.mkdir()
reins.mkdir()
_write(profiles, "profile.yaml", _profile())
_write(reins, "rein.yaml", _rein())
context = ProfileCatalog(profiles, reins).resolve_context("harness.test@1.0.0")
assert context.operational_readiness.status == "unverified"
assert context.operational_readiness.reason == "no operational proof declared"
def test_unknown_profile_fails_closed() -> None:
with pytest.raises(UnknownProfileError, match="unknown harness profile"):
ProfileCatalog().resolve("harness.missing@1.0.0")