Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a0233b-178d-7162-b92f-31a31ea8ca9b
This commit is contained in:
parent
a224e71555
commit
cac605abd7
19 changed files with 339 additions and 41 deletions
|
|
@ -12,6 +12,19 @@ def test_profiles_command_lists_committed_constellations(capsys) -> None:
|
|||
|
||||
rows = json.loads(capsys.readouterr().out)
|
||||
assert {row["rein_id"] for row in rows} == {"rein-aharness", "rein-openweights"}
|
||||
assert {row["operational_readiness"]["status"] for row in rows} == {
|
||||
"blocked",
|
||||
"unverified",
|
||||
}
|
||||
|
||||
|
||||
def test_profiles_text_output_labels_runtime_readiness(capsys) -> None:
|
||||
assert main(["profiles"]) == 0
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert "readiness=blocked" in output
|
||||
assert "readiness=unverified" in output
|
||||
assert "executable" not in output
|
||||
|
||||
|
||||
def test_execution_request_example_matches_contract() -> None:
|
||||
|
|
@ -19,7 +32,7 @@ def test_execution_request_example_matches_contract() -> None:
|
|||
|
||||
request = ExecutionRequest.model_validate_json(fixture.read_text())
|
||||
|
||||
assert request.harness_profile_ref == "harness.agent-dev-local@1.0.0"
|
||||
assert request.harness_profile_ref == "harness.agent-dev@1.0.0"
|
||||
assert request.assignment_ref == "role-assignment:agent-7:42"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ from sandboxer.models import Reachability, SandboxState, SandboxStatus
|
|||
from glas_harness.contract import (
|
||||
ExecutionRequest,
|
||||
ExecutionSummary,
|
||||
OperationalReadiness,
|
||||
Rein,
|
||||
SandboxHandle,
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
)
|
||||
from glas_harness.gateway import run_execution, run_task_through_rein
|
||||
from glas_harness.profiles import ProfileCatalog
|
||||
|
||||
|
||||
PROFILE = "harness.agent-dev-local@1.0.0"
|
||||
|
|
@ -83,12 +85,29 @@ def _request(*, profile: str = PROFILE, report_to_hub: bool = False) -> Executio
|
|||
)
|
||||
|
||||
|
||||
def _catalog_with_readiness(status: str = "ready") -> ProfileCatalog:
|
||||
catalog = ProfileCatalog()
|
||||
profile, _ = catalog.resolve(PROFILE)
|
||||
readiness = OperationalReadiness(
|
||||
status=status,
|
||||
reason="test-only readiness state",
|
||||
owner="tests",
|
||||
evidence_ref="test:gateway-readiness",
|
||||
)
|
||||
catalog.profiles()[(profile.id, profile.version)] = profile.model_copy(
|
||||
update={"operational_readiness": readiness}
|
||||
)
|
||||
return catalog
|
||||
|
||||
|
||||
def test_run_execution_creates_and_destroys_sandbox() -> None:
|
||||
manager = MagicMock()
|
||||
manager.create.return_value = _fake_status()
|
||||
rein = _FakeRein()
|
||||
|
||||
result = run_execution(_request(), rein=rein, manager=manager)
|
||||
result = run_execution(
|
||||
_request(), catalog=_catalog_with_readiness(), rein=rein, manager=manager
|
||||
)
|
||||
|
||||
assert rein.calls == [
|
||||
"start_session",
|
||||
|
|
@ -100,6 +119,7 @@ def test_run_execution_creates_and_destroys_sandbox() -> None:
|
|||
manager.destroy.assert_called_once_with("sbx1")
|
||||
assert result.ok is True
|
||||
assert result.evidence.profile_ref == PROFILE
|
||||
assert result.evidence.profile_readiness.status == "ready"
|
||||
assert result.evidence.rein_id == "rein-aharness"
|
||||
assert result.evidence.commit_sha == "deadbeef"
|
||||
assert result.evidence.tokens_spent == 123
|
||||
|
|
@ -116,7 +136,9 @@ def test_run_execution_normalizes_execution_failure_and_tears_down() -> None:
|
|||
raise RuntimeError("boom")
|
||||
|
||||
rein = _FailingRein()
|
||||
result = run_execution(_request(), rein=rein, manager=manager)
|
||||
result = run_execution(
|
||||
_request(), catalog=_catalog_with_readiness(), rein=rein, manager=manager
|
||||
)
|
||||
|
||||
assert result.ok is False
|
||||
assert result.evidence.outcome == "failed"
|
||||
|
|
@ -139,6 +161,35 @@ def test_run_execution_refuses_unknown_profile_before_sandbox() -> None:
|
|||
manager.create.assert_not_called()
|
||||
|
||||
|
||||
def test_run_execution_refuses_blocked_profile_before_sandbox() -> None:
|
||||
manager = MagicMock()
|
||||
|
||||
result = run_execution(_request(), rein=_FakeRein(), manager=manager)
|
||||
|
||||
assert result.ok is False
|
||||
assert result.evidence.outcome == "refused"
|
||||
assert result.evidence.failure_stage == "resolution"
|
||||
assert result.evidence.profile_readiness.status == "blocked"
|
||||
assert "GLAS-IN-0002" in (result.evidence.error or "")
|
||||
manager.create.assert_not_called()
|
||||
|
||||
|
||||
def test_run_execution_allows_labeled_unverified_proof_attempt() -> None:
|
||||
manager = MagicMock()
|
||||
manager.create.return_value = _fake_status()
|
||||
|
||||
result = run_execution(
|
||||
_request(),
|
||||
catalog=_catalog_with_readiness("unverified"),
|
||||
rein=_FakeRein(),
|
||||
manager=manager,
|
||||
)
|
||||
|
||||
assert result.ok is True
|
||||
assert result.evidence.profile_readiness.status == "unverified"
|
||||
manager.create.assert_called_once()
|
||||
|
||||
|
||||
def test_run_execution_refuses_worker_identifier_before_sandbox() -> None:
|
||||
manager = MagicMock()
|
||||
request = _request().model_copy(update={"actor": "rein-aharness@railiance01"})
|
||||
|
|
@ -160,7 +211,12 @@ def test_hub_receives_normalized_evidence_without_raw_output() -> None:
|
|||
manager.create.return_value = _fake_status()
|
||||
|
||||
with patch("glas_harness.gateway.hub.post_progress_event", return_value=True) as post:
|
||||
result = run_execution(_request(report_to_hub=True), rein=_FakeRein(), manager=manager)
|
||||
result = run_execution(
|
||||
_request(report_to_hub=True),
|
||||
catalog=_catalog_with_readiness(),
|
||||
rein=_FakeRein(),
|
||||
manager=manager,
|
||||
)
|
||||
|
||||
post.assert_called_once()
|
||||
detail = post.call_args.kwargs["detail"]
|
||||
|
|
@ -181,7 +237,10 @@ def test_hub_failure_detail_excludes_raw_provider_error() -> None:
|
|||
|
||||
with patch("glas_harness.gateway.hub.post_progress_event", return_value=True) as post:
|
||||
result = run_execution(
|
||||
_request(report_to_hub=True), rein=_FailingRein(), manager=manager
|
||||
_request(report_to_hub=True),
|
||||
catalog=_catalog_with_readiness(),
|
||||
rein=_FailingRein(),
|
||||
manager=manager,
|
||||
)
|
||||
|
||||
detail = post.call_args.kwargs["detail"]
|
||||
|
|
@ -201,6 +260,7 @@ def test_wrapper_requires_and_reports_harness_profile() -> None:
|
|||
description="d",
|
||||
rein=_FakeRein(),
|
||||
manager=manager,
|
||||
catalog=_catalog_with_readiness(),
|
||||
report_to_hub=False,
|
||||
)
|
||||
|
||||
|
|
@ -213,7 +273,12 @@ def test_teardown_failure_is_visible_in_evidence() -> None:
|
|||
manager.create.return_value = _fake_status()
|
||||
manager.destroy.side_effect = RuntimeError("cannot teardown")
|
||||
|
||||
result = run_execution(_request(), rein=_FakeRein(), manager=manager)
|
||||
result = run_execution(
|
||||
_request(),
|
||||
catalog=_catalog_with_readiness(),
|
||||
rein=_FakeRein(),
|
||||
manager=manager,
|
||||
)
|
||||
|
||||
assert result.ok is False
|
||||
assert result.evidence.failure_stage == "teardown"
|
||||
|
|
|
|||
|
|
@ -71,6 +71,14 @@ def test_committed_catalog_resolves_both_constellations() -> None:
|
|||
"rein-aharness",
|
||||
"rein-openweights",
|
||||
}
|
||||
assert {
|
||||
(context.profile.id, context.operational_readiness.status)
|
||||
for context in contexts
|
||||
} == {
|
||||
("harness.agent-dev-local", "blocked"),
|
||||
("harness.agent-dev-openweights-local", "blocked"),
|
||||
("harness.agent-dev", "unverified"),
|
||||
}
|
||||
profile, descriptor = catalog.resolve(
|
||||
"harness.agent-dev-openweights-local@1.0.0"
|
||||
)
|
||||
|
|
@ -89,6 +97,41 @@ def test_committed_catalog_resolves_both_constellations() -> None:
|
|||
assert rein.budget_tokens == 60000
|
||||
|
||||
|
||||
def test_ready_profile_requires_an_evidence_reference(tmp_path) -> None:
|
||||
profiles = tmp_path / "profiles"
|
||||
reins = tmp_path / "reins"
|
||||
profiles.mkdir()
|
||||
reins.mkdir()
|
||||
_write(
|
||||
profiles,
|
||||
"profile.yaml",
|
||||
_profile(
|
||||
extra="""operational_readiness:
|
||||
status: ready
|
||||
reason: proof completed
|
||||
owner: tests
|
||||
"""
|
||||
),
|
||||
)
|
||||
_write(reins, "rein.yaml", _rein())
|
||||
|
||||
with pytest.raises(ProfileError, match="ready operational status requires evidence_ref"):
|
||||
ProfileCatalog(profiles, reins).profiles()
|
||||
|
||||
|
||||
def test_undeclared_readiness_defaults_to_unverified(tmp_path) -> None:
|
||||
profiles = tmp_path / "profiles"
|
||||
reins = tmp_path / "reins"
|
||||
profiles.mkdir()
|
||||
reins.mkdir()
|
||||
_write(profiles, "profile.yaml", _profile())
|
||||
_write(reins, "rein.yaml", _rein())
|
||||
|
||||
context = ProfileCatalog(profiles, reins).resolve_context("harness.test@1.0.0")
|
||||
|
||||
assert context.operational_readiness.status == "unverified"
|
||||
assert context.operational_readiness.reason == "no operational proof declared"
|
||||
|
||||
def test_unknown_profile_fails_closed() -> None:
|
||||
with pytest.raises(UnknownProfileError, match="unknown harness profile"):
|
||||
ProfileCatalog().resolve("harness.missing@1.0.0")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue