Reserve worker spend durably before governed dispatch
Some checks failed
Governed runtime contract / contract (push) Has been cancelled
Some checks failed
Governed runtime contract / contract (push) Has been cancelled
Assistant: codex Assistant-Model: gpt-5.6-luna Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc
This commit is contained in:
parent
4ffb8acb19
commit
38b25fbc26
14 changed files with 1378 additions and 9 deletions
|
|
@ -9,6 +9,7 @@ import pytest
|
|||
|
||||
from glas_harness.contract import (
|
||||
ExecutionSummary,
|
||||
ExecutionLimits,
|
||||
OperationalReadiness,
|
||||
Rein,
|
||||
ToolResult,
|
||||
|
|
@ -29,6 +30,7 @@ from rein_aharness.ops_run_client import (
|
|||
)
|
||||
from test_repository_artifact import commit, git
|
||||
from rein_aharness.repository_grant import RepositoryGrant
|
||||
from rein_aharness.spend_admission import SpendLedger, SpendPolicy, digest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
os.environ.get("REIN_REAL_BWRAP") != "1", reason="opt-in kernel namespace proof"
|
||||
|
|
@ -69,7 +71,7 @@ print(git('rev-parse','HEAD'))
|
|||
|
||||
def end_session(self, session):
|
||||
return ExecutionSummary(
|
||||
committed=True, commit_sha=self.head, outcome="succeeded", tokens_spent=0
|
||||
committed=True, commit_sha=self.head, outcome="succeeded", tokens_spent=0, cost_usd=0.0
|
||||
)
|
||||
|
||||
def cleanup_session(self, session):
|
||||
|
|
@ -77,7 +79,8 @@ print(git('rev-parse','HEAD'))
|
|||
assert git(self.source, "rev-parse", "HEAD") == self.baseline
|
||||
|
||||
|
||||
def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize("with_spend", [False, True])
|
||||
def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch, with_spend):
|
||||
monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data"))
|
||||
monkeypatch.setenv("REIN_AHARNESS_STATE_DIR", str(tmp_path / "state"))
|
||||
monkeypatch.setenv("SANDBOXER_NO_STATE_HUB", "1")
|
||||
|
|
@ -129,6 +132,7 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch):
|
|||
profile, _ = catalog.resolve(run.harness_profile_ref)
|
||||
catalog.profiles()[(profile.id, profile.version)] = profile.model_copy(
|
||||
update={
|
||||
"limits": ExecutionLimits(max_budget_usd=4.0, max_turns=8),
|
||||
"operational_readiness": OperationalReadiness(
|
||||
status="ready",
|
||||
reason="isolated test fixture",
|
||||
|
|
@ -137,12 +141,37 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch):
|
|||
)
|
||||
}
|
||||
)
|
||||
spend = None
|
||||
if with_spend:
|
||||
private = tmp_path / "spend-state"
|
||||
private.mkdir(mode=0o700)
|
||||
admitted_profile, descriptor = catalog.resolve(run.harness_profile_ref)
|
||||
policy = SpendPolicy(
|
||||
version="1", envelope_id="bwrap-fixture", authority_ref="test:no-live-authority",
|
||||
valid_from="2026-09-01T00:00:00Z", expires_at="2099-01-01T00:00:00Z",
|
||||
timezone="Europe/Berlin", worker_id="fixture-worker",
|
||||
activity_definition_id="fixture-definition", target_repo=str(source),
|
||||
project="fixture-factory", profile_ref=run.harness_profile_ref,
|
||||
profile_sha256=digest(admitted_profile.model_dump(mode="json")),
|
||||
descriptor_sha256=digest(descriptor.model_dump(mode="json")),
|
||||
repository_grant_id=grant.grant_id, max_budget_usd="4", max_liability_usd="5",
|
||||
max_turns=8, eur_per_usd="1", per_run_eur="5", daily_eur="10", total_eur="15",
|
||||
)
|
||||
policy_path = private / "policy.json"
|
||||
policy_path.write_text(json.dumps(policy.__dict__))
|
||||
policy_path.chmod(0o600)
|
||||
spend = SpendLedger(private / "spend.sqlite3", policy)
|
||||
spend.initialize()
|
||||
client.config.execution_project = "fixture-factory"
|
||||
client.config.spend_policy_path = str(policy_path)
|
||||
client.config.spend_ledger_path = str(spend.path)
|
||||
monkeypatch.setattr("glas_harness.profiles.ProfileCatalog", lambda: catalog)
|
||||
manager = SandboxManager(store=SandboxStore(tmp_path / "sandboxes.json"))
|
||||
rein = DeterministicRein(source, baseline)
|
||||
monkeypatch.setattr(
|
||||
"glas_harness.gateway.run_execution",
|
||||
lambda request, **kwargs: run_execution(
|
||||
request, catalog=catalog, rein=rein, manager=manager, **kwargs
|
||||
request, catalog=kwargs.pop("catalog", catalog), rein=rein, manager=manager, **kwargs
|
||||
),
|
||||
)
|
||||
outbox = CloseOutbox(state_dir=tmp_path / "state")
|
||||
|
|
@ -165,7 +194,14 @@ def test_real_bwrap_worker_import_and_close_replay(tmp_path, monkeypatch):
|
|||
client.claim.return_value = []
|
||||
second = process_one(client, outbox=outbox, report_to_hub=False)
|
||||
assert second.empty and rein.calls == 1
|
||||
assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}
|
||||
if spend is not None:
|
||||
rows = spend.status()["reservations"]
|
||||
assert len(rows) == 1 and rows[0]["state"] == "charged"
|
||||
assert rows[0]["liability"] == 5_000_000
|
||||
client.claim.return_value = [run]
|
||||
replay = process_one(client, outbox=outbox, report_to_hub=False)
|
||||
assert not replay.ok and rein.calls == 1
|
||||
assert client.complete.call_count == 2
|
||||
assert client.heartbeat.call_count >= 1
|
||||
assert git(source, "rev-list", "--count", "HEAD") == "2"
|
||||
assert outbox.status() == {"pending": 0, "delivered": 1, "quarantined": 0}
|
||||
|
|
|
|||
492
tests/test_spend_admission.py
Normal file
492
tests/test_spend_admission.py
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
"""No-provider proofs for admission, persistence, accounting and recovery."""
|
||||
|
||||
import json
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from rein_aharness.ops_run_client import ActivityCoreOpsClient, OpsRun, OpsRunConfig
|
||||
from rein_aharness.repository_grant import RepositoryGrant
|
||||
from rein_aharness.spend_admission import (
|
||||
SpendAdmissionError,
|
||||
SpendLedger,
|
||||
SpendPolicy,
|
||||
digest,
|
||||
worker_spend,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 9, 9, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def run(key="one"):
|
||||
return OpsRun(
|
||||
id=f"run-{key}",
|
||||
activity_definition_id="def-1",
|
||||
idempotency_key=key,
|
||||
target_repo="target",
|
||||
title="private intent",
|
||||
description="private prompt",
|
||||
attempt=1,
|
||||
claim_owner="worker-1",
|
||||
harness_profile_ref="harness.test@1.0.0",
|
||||
repository_grant=RepositoryGrant("1", ("docs/",), 1, 1, False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ledger(tmp_path):
|
||||
private = tmp_path / "private"
|
||||
private.mkdir(mode=0o700)
|
||||
policy = SpendPolicy(
|
||||
version="1",
|
||||
envelope_id="test-only",
|
||||
authority_ref="fixture:no-live-authority",
|
||||
valid_from="2026-09-01T00:00:00Z",
|
||||
expires_at="2099-01-01T00:00:00Z",
|
||||
timezone="Europe/Berlin",
|
||||
worker_id="worker-1",
|
||||
activity_definition_id="def-1",
|
||||
target_repo=str(tmp_path / "target"),
|
||||
project="fixture-factory",
|
||||
profile_ref="harness.test@1.0.0",
|
||||
profile_sha256="a" * 64,
|
||||
descriptor_sha256="b" * 64,
|
||||
repository_grant_id=run().repository_grant.grant_id,
|
||||
max_budget_usd="4",
|
||||
max_liability_usd="5",
|
||||
max_turns=8,
|
||||
eur_per_usd="1",
|
||||
per_run_eur="5",
|
||||
daily_eur="10",
|
||||
total_eur="15",
|
||||
)
|
||||
store = SpendLedger(private / "spend.sqlite3", policy)
|
||||
store.initialize()
|
||||
return store
|
||||
|
||||
|
||||
def success(store, item, cost=1):
|
||||
return {
|
||||
"ok": True,
|
||||
"evidence": {
|
||||
"request_id": item.id,
|
||||
"profile_ref": store.policy.profile_ref,
|
||||
"cost_usd": cost,
|
||||
"outcome": "succeeded",
|
||||
"session_cleanup": "succeeded",
|
||||
"sandbox_destroy": "succeeded",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def configured(store):
|
||||
path = store.path.parent / "policy.json"
|
||||
path.write_text(json.dumps(store.policy.__dict__))
|
||||
path.chmod(0o600)
|
||||
return OpsRunConfig(
|
||||
worker_id="worker-1",
|
||||
execution_project="fixture-factory",
|
||||
spend_policy_path=str(path),
|
||||
spend_ledger_path=str(store.path),
|
||||
repo_roots=(str(Path(store.policy.target_repo).parent),),
|
||||
)
|
||||
|
||||
|
||||
def test_daily_total_and_no_self_reported_refund(ledger):
|
||||
for key in ("one", "two"):
|
||||
item = run(key)
|
||||
ledger.reserve(item, now=NOW)
|
||||
ledger.observe(item.id, success(ledger, item, 0), now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="daily"):
|
||||
ledger.reserve(run("three"), now=NOW)
|
||||
tomorrow = NOW + timedelta(days=1)
|
||||
ledger.reserve(run("three"), now=tomorrow)
|
||||
ledger.observe(run("three").id, success(ledger, run("three")), now=tomorrow)
|
||||
with pytest.raises(SpendAdmissionError, match="total"):
|
||||
ledger.reserve(run("four"), now=tomorrow + timedelta(days=1))
|
||||
assert sum(r["liability"] for r in ledger.status()["reservations"]) == 15_000_000
|
||||
|
||||
|
||||
def test_crash_survives_reopen_and_blocks_across_days(ledger):
|
||||
ledger.reserve(run(), now=NOW)
|
||||
reopened = SpendLedger(ledger.path, ledger.policy)
|
||||
for day in (NOW, NOW + timedelta(days=5)):
|
||||
with pytest.raises(SpendAdmissionError, match="unresolved"):
|
||||
reopened.preflight(now=day)
|
||||
reopened.reconcile(
|
||||
run().id,
|
||||
cost_usd="2",
|
||||
receipt="audit:stopped-and-final",
|
||||
now=NOW + timedelta(days=5),
|
||||
)
|
||||
reopened.reconcile(
|
||||
run().id,
|
||||
cost_usd="2",
|
||||
receipt="audit:stopped-and-final",
|
||||
now=NOW + timedelta(days=5),
|
||||
)
|
||||
with pytest.raises(SpendAdmissionError, match="replay"):
|
||||
reopened.reserve(replace(run(), attempt=2), now=NOW + timedelta(days=5))
|
||||
with pytest.raises(SpendAdmissionError, match="replay"):
|
||||
reopened.reserve(
|
||||
replace(run(), id="different-row"), now=NOW + timedelta(days=5)
|
||||
)
|
||||
assert reopened.status()["reservations"][0]["liability"] == 5_000_000
|
||||
|
||||
|
||||
def test_full_charge_on_both_sides_of_midnight(ledger):
|
||||
before = datetime(2026, 9, 9, 21, 59, tzinfo=UTC)
|
||||
after = before + timedelta(minutes=2)
|
||||
ledger.reserve(run(), now=before)
|
||||
ledger.observe(run().id, success(ledger, run()), now=after)
|
||||
row = ledger.status()["reservations"][0]
|
||||
assert (row["start_day"], row["end_day"]) == ("2026-09-09", "2026-09-10")
|
||||
ledger.reserve(run("two"), now=after)
|
||||
ledger.observe(run("two").id, success(ledger, run("two")), now=after)
|
||||
with pytest.raises(SpendAdmissionError, match="daily"):
|
||||
ledger.preflight(now=after)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cost", [None, True, -1, "NaN", "Infinity", {}, 1000001])
|
||||
def test_unknown_accounting_never_releases_hold(ledger, cost):
|
||||
ledger.reserve(run(), now=NOW)
|
||||
ledger.observe(run().id, success(ledger, run(), cost), now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="unresolved"):
|
||||
ledger.preflight(now=NOW)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("request_id", "wrong"),
|
||||
("profile_ref", "wrong"),
|
||||
("outcome", "failed"),
|
||||
("sandbox_destroy", "failed"),
|
||||
("session_cleanup", "failed"),
|
||||
],
|
||||
)
|
||||
def test_incomplete_or_mismatched_result_retains_hold(ledger, field, value):
|
||||
ledger.reserve(run(), now=NOW)
|
||||
raw = success(ledger, run())
|
||||
raw["evidence"][field] = value
|
||||
ledger.observe(run().id, raw, now=NOW)
|
||||
assert ledger.status()["reservations"][0]["state"] == "held"
|
||||
|
||||
|
||||
def test_overrun_is_recorded_and_freezes_even_after_reconcile(ledger):
|
||||
ledger.reserve(run(), now=NOW)
|
||||
ledger.observe(run().id, success(ledger, run(), 7), now=NOW)
|
||||
assert ledger.status()["breached"]
|
||||
assert ledger.status()["reservations"][0]["liability"] == 7_000_000
|
||||
with pytest.raises(SpendAdmissionError, match="discard"):
|
||||
ledger.reconcile(run().id, cost_usd="0", receipt="audit:final", now=NOW)
|
||||
ledger.reconcile(run().id, cost_usd="7", receipt="audit:final", now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="breached"):
|
||||
ledger.preflight(now=NOW + timedelta(days=1))
|
||||
|
||||
|
||||
def test_missing_corrupt_changed_policy_and_reinit_refuse(ledger):
|
||||
with pytest.raises(SpendAdmissionError):
|
||||
ledger.initialize()
|
||||
with pytest.raises(SpendAdmissionError, match="policy mismatch"):
|
||||
SpendLedger(ledger.path, replace(ledger.policy, total_eur="20")).preflight(
|
||||
now=NOW
|
||||
)
|
||||
ledger.path.unlink()
|
||||
with pytest.raises(SpendAdmissionError, match="unavailable"):
|
||||
ledger.preflight(now=NOW)
|
||||
assert not ledger.path.exists()
|
||||
ledger.path.write_bytes(b"corrupt")
|
||||
ledger.path.chmod(0o600)
|
||||
with pytest.raises(SpendAdmissionError, match="inconsistent"):
|
||||
ledger.preflight(now=NOW)
|
||||
|
||||
|
||||
def test_private_paths_and_policy(ledger):
|
||||
config = configured(ledger)
|
||||
assert worker_spend(config).policy.sha256 == ledger.policy.sha256
|
||||
Path(config.spend_policy_path).chmod(0o644)
|
||||
with pytest.raises(SpendAdmissionError, match="private"):
|
||||
worker_spend(config)
|
||||
with pytest.raises(SpendAdmissionError, match="both"):
|
||||
worker_spend(replace(config, spend_policy_path=None))
|
||||
with pytest.raises(SpendAdmissionError, match="outside"):
|
||||
SpendLedger(Path(ledger.policy.target_repo) / "spend.db", ledger.policy)
|
||||
|
||||
|
||||
def test_expiry_clock_rollback_and_upward_rounding(ledger):
|
||||
ledger.preflight(now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="backwards"):
|
||||
ledger.preflight(now=NOW - timedelta(seconds=1))
|
||||
with pytest.raises(SpendAdmissionError, match="valid"):
|
||||
ledger.preflight(now=datetime(2099, 1, 1, tzinfo=UTC))
|
||||
policy = replace(ledger.policy, eur_per_usd="0.99999999")
|
||||
assert policy.reservation == 5_000_000
|
||||
with pytest.raises(SpendAdmissionError, match="envelope"):
|
||||
replace(ledger.policy, eur_per_usd="1.00000001")
|
||||
|
||||
|
||||
def _reserve_process(path, policy, key):
|
||||
try:
|
||||
SpendLedger(Path(path), SpendPolicy(**policy)).reserve(run(key), now=NOW)
|
||||
return "reserved"
|
||||
except SpendAdmissionError:
|
||||
return "refused"
|
||||
|
||||
|
||||
def test_concurrent_processes_cannot_double_admit(ledger):
|
||||
with ProcessPoolExecutor(max_workers=4) as pool:
|
||||
results = list(
|
||||
pool.map(
|
||||
_reserve_process,
|
||||
[str(ledger.path)] * 4,
|
||||
[ledger.policy.__dict__] * 4,
|
||||
["a", "b", "c", "d"],
|
||||
)
|
||||
)
|
||||
assert sorted(results) == ["refused", "refused", "refused", "reserved"]
|
||||
assert len(ledger.status()["reservations"]) == 1
|
||||
|
||||
|
||||
def test_pending_spend_blocks_claim_but_close_replay_still_runs(ledger, monkeypatch):
|
||||
from rein_aharness.claim_loop import process_one
|
||||
from rein_aharness.close_outbox import CloseOutbox, CloseRequest
|
||||
|
||||
ledger.reserve(run(), now=NOW)
|
||||
client = MagicMock(spec=ActivityCoreOpsClient)
|
||||
client.config = configured(ledger)
|
||||
client.complete.return_value = SimpleNamespace(state="succeeded")
|
||||
outbox = CloseOutbox()
|
||||
outbox.enqueue(
|
||||
CloseRequest(
|
||||
run_id="run-older",
|
||||
transaction_id="tx-1",
|
||||
worker_id="worker-1",
|
||||
action="complete",
|
||||
result={"ok": True},
|
||||
)
|
||||
)
|
||||
result = process_one(client, outbox=outbox)
|
||||
assert not result.claimed and "unresolved" in result.reason
|
||||
client.claim.assert_not_called()
|
||||
client.complete.assert_called_once()
|
||||
|
||||
|
||||
def test_nonprofile_row_cannot_bypass_spend_gate(ledger, monkeypatch):
|
||||
from rein_aharness.claim_loop import process_one
|
||||
|
||||
client = MagicMock(spec=ActivityCoreOpsClient)
|
||||
client.config = configured(ledger)
|
||||
client.claim.return_value = [replace(run(), harness_profile_ref=None)]
|
||||
client.fail.return_value = SimpleNamespace(state="failed")
|
||||
dispatched = []
|
||||
monkeypatch.setattr(
|
||||
"rein_aharness.claim_loop.execute_approach",
|
||||
lambda *a, **k: dispatched.append(True),
|
||||
)
|
||||
result = process_one(client)
|
||||
assert not result.ok and "profile" in result.reason and not dispatched
|
||||
assert ledger.status()["reservations"] == []
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dispatch_case(ledger, monkeypatch):
|
||||
import subprocess
|
||||
|
||||
from glas_harness.contract import ExecutionLimits, OperationalReadiness
|
||||
from glas_harness.profiles import ProfileCatalog
|
||||
|
||||
target = Path(ledger.policy.target_repo)
|
||||
target.mkdir()
|
||||
subprocess.run(["git", "init", "-q", str(target)], check=True)
|
||||
catalog = ProfileCatalog()
|
||||
profile, descriptor = catalog.resolve("harness.agent-dev-local@1.0.0")
|
||||
profile = profile.model_copy(
|
||||
update={
|
||||
"id": "harness.test",
|
||||
"operational_readiness": OperationalReadiness(
|
||||
status="ready", evidence_ref="test:only"
|
||||
),
|
||||
"limits": ExecutionLimits(max_budget_usd=4.0, max_turns=8),
|
||||
}
|
||||
)
|
||||
catalog.profiles()[(profile.id, profile.version)] = profile
|
||||
monkeypatch.setattr("glas_harness.profiles.ProfileCatalog", lambda: catalog)
|
||||
policy = replace(
|
||||
ledger.policy,
|
||||
profile_sha256=digest(profile.model_dump(mode="json")),
|
||||
descriptor_sha256=digest(descriptor.model_dump(mode="json")),
|
||||
)
|
||||
store = SpendLedger(ledger.path.parent / "dispatch.sqlite3", policy)
|
||||
store.initialize()
|
||||
# Artifact transfer is exercised unmodified by the separate actual bwrap test.
|
||||
transfer = MagicMock()
|
||||
transfer.import_after_teardown.return_value = {"fixture": True}
|
||||
monkeypatch.setattr(
|
||||
"rein_aharness.repository_artifact.RepositoryArtifactTransfer",
|
||||
lambda *a, **k: transfer,
|
||||
)
|
||||
return store, configured(store), catalog, transfer
|
||||
|
||||
|
||||
def test_reservation_precedes_gateway_and_reclaim_never_repeats(dispatch_case):
|
||||
from rein_aharness.glas_execution import GlasExecutionError, execute_profiled_run
|
||||
|
||||
store, config, catalog, transfer = dispatch_case
|
||||
calls = []
|
||||
|
||||
def gateway(request, **kwargs):
|
||||
assert kwargs["catalog"] is catalog
|
||||
assert store.status()["reservations"][0]["state"] == "held"
|
||||
calls.append(request)
|
||||
return success(store, run())
|
||||
|
||||
execute_profiled_run(run(), config, gateway=gateway, report_to_hub=False)
|
||||
assert store.status()["reservations"][0]["state"] == "charged"
|
||||
assert calls[0].project == "fixture-factory"
|
||||
transfer.import_after_teardown.assert_called_once()
|
||||
with pytest.raises(GlasExecutionError, match="replay"):
|
||||
execute_profiled_run(
|
||||
replace(run(), attempt=2), config, gateway=gateway, report_to_hub=False
|
||||
)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
["worker", "definition", "profile", "grant", "project", "limits", "descriptor"],
|
||||
)
|
||||
def test_scope_mismatch_denied_before_gateway_or_reservation(dispatch_case, change):
|
||||
from rein_aharness.glas_execution import GlasExecutionError, execute_profiled_run
|
||||
|
||||
store, config, catalog, _transfer = dispatch_case
|
||||
item = run()
|
||||
if change == "worker":
|
||||
item.claim_owner = "other"
|
||||
if change == "definition":
|
||||
item.activity_definition_id = "other"
|
||||
if change == "profile":
|
||||
item.harness_profile_ref = "harness.agent-dev-local@1.0.0"
|
||||
if change == "grant":
|
||||
item.repository_grant = RepositoryGrant("1", ("elsewhere/",), 1, 1, False)
|
||||
if change == "project":
|
||||
config.execution_project = "other"
|
||||
if change == "limits":
|
||||
catalog.profiles()[("harness.test", "1.0.0")].limits.max_budget_usd = 5.0
|
||||
if change == "descriptor":
|
||||
catalog.reins()["rein-aharness"].version = "99.0.0"
|
||||
gateway = MagicMock()
|
||||
with pytest.raises(GlasExecutionError):
|
||||
execute_profiled_run(item, config, gateway=gateway, report_to_hub=False)
|
||||
gateway.assert_not_called()
|
||||
assert store.status()["reservations"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure", ["exception", "missing-cost", "overrun", "lease-loss"]
|
||||
)
|
||||
def test_uncertain_gateway_never_imports_and_blocks_next_claim(dispatch_case, failure):
|
||||
from rein_aharness.execution_cancel import ExecutionCancel, ExecutionCancelled
|
||||
from rein_aharness.glas_execution import GlasExecutionError, execute_profiled_run
|
||||
|
||||
store, config, _catalog, transfer = dispatch_case
|
||||
cancel = ExecutionCancel()
|
||||
|
||||
def gateway(request, **kwargs):
|
||||
if failure == "exception":
|
||||
raise RuntimeError("private provider response")
|
||||
if failure == "lease-loss":
|
||||
cancel.cancel("lease-loss")
|
||||
return success(
|
||||
store,
|
||||
run(),
|
||||
None if failure == "missing-cost" else 7 if failure == "overrun" else 1,
|
||||
)
|
||||
|
||||
with pytest.raises((GlasExecutionError, ExecutionCancelled)):
|
||||
execute_profiled_run(
|
||||
run(), config, gateway=gateway, cancel=cancel, report_to_hub=False
|
||||
)
|
||||
transfer.import_after_teardown.assert_not_called()
|
||||
assert store.status()["reservations"][0]["state"] == "held"
|
||||
assert "private" not in json.dumps(store.status())
|
||||
with pytest.raises(SpendAdmissionError):
|
||||
store.preflight()
|
||||
|
||||
|
||||
def test_cli_reconciliation_requires_termination_attestation(ledger, capsys):
|
||||
from rein_aharness.cli import main
|
||||
|
||||
config = configured(ledger)
|
||||
ledger.reserve(run(), now=NOW)
|
||||
args = [
|
||||
"spend",
|
||||
"reconcile",
|
||||
"--policy",
|
||||
config.spend_policy_path,
|
||||
"--ledger",
|
||||
config.spend_ledger_path,
|
||||
"--run-id",
|
||||
run().id,
|
||||
"--cost-usd",
|
||||
"1",
|
||||
"--receipt",
|
||||
"audit:final",
|
||||
]
|
||||
assert main(args) == 2
|
||||
assert ledger.status()["reservations"][0]["state"] == "held"
|
||||
assert main(args + ["--provider-stopped"]) == 0
|
||||
assert ledger.status()["reservations"][0]["state"] == "charged"
|
||||
capsys.readouterr()
|
||||
|
||||
|
||||
def test_required_admission_missing_never_claims():
|
||||
from rein_aharness.claim_loop import process_one
|
||||
|
||||
client = MagicMock(spec=ActivityCoreOpsClient)
|
||||
client.config = OpsRunConfig(require_spend_admission=True)
|
||||
result = process_one(client)
|
||||
assert not result.claimed and "not configured" in result.reason
|
||||
client.claim.assert_not_called()
|
||||
|
||||
|
||||
def test_policy_file_rejects_duplicates_and_symlinks(ledger):
|
||||
config = configured(ledger)
|
||||
path = Path(config.spend_policy_path)
|
||||
text = path.read_text()
|
||||
path.write_text(text[:-1] + ', "max_budget_usd": "4"}')
|
||||
with pytest.raises(SpendAdmissionError, match="invalid"):
|
||||
SpendPolicy.load(path)
|
||||
path.write_text(text)
|
||||
link = path.with_name("symlink.json")
|
||||
link.symlink_to(path)
|
||||
with pytest.raises(SpendAdmissionError, match="private"):
|
||||
SpendPolicy.load(link)
|
||||
|
||||
|
||||
def test_reconcile_unknown_identity_or_conflicting_receipt(ledger):
|
||||
with pytest.raises(SpendAdmissionError, match="unknown"):
|
||||
ledger.reconcile("absent", cost_usd="1", receipt="audit:final", now=NOW)
|
||||
ledger.reserve(run(), now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="receipt"):
|
||||
ledger.reconcile(
|
||||
run().id, cost_usd="1", receipt="private text with spaces", now=NOW
|
||||
)
|
||||
ledger.reconcile(run().id, cost_usd="1", receipt="audit:final", now=NOW)
|
||||
with pytest.raises(SpendAdmissionError, match="conflict"):
|
||||
ledger.reconcile(run().id, cost_usd="0", receipt="audit:other", now=NOW)
|
||||
|
||||
|
||||
def test_decimal_conversion_never_rounds_liability_down(ledger):
|
||||
from rein_aharness.spend_admission import cap_micros, converted_micros
|
||||
|
||||
assert converted_micros("1.00000000000000000000000000000001", "1") == 1_000_001
|
||||
assert cap_micros("0.99999999999999999999999999999999") == 999_999
|
||||
assert (
|
||||
ledger.observe(run().id, success(ledger, run(), "1e-99999"), now=NOW) is False
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue