coulomb/rein-aharness
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: reject broken runtime launchers and preserve failed proof evidence
Some checks failed
Governed runtime contract / contract (push) Failing after 16s
Details

Browse source 
Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e387-534d-70e3-ad53-4ea05676db8c
This commit is contained in:
tegwick 2026-09-27 23:19:50 +02:00
parent 4e666d6fa3
commit 43e621439a
26 changed files with 1367 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

40
docs/evidence/2026-09-27-grant-aware-worker-rollout.json Normal file
View file

@ -0,0 +1,40 @@
{
"observed_at": "2026-09-27T20:01:04.485643+00:00",
"activity_revision": "428f2d71b0f5ade11457e16d7b6341f571f6735f",
"platform_revision": "c3607fffeade70acd361a4757e72a062725cb83a",
"deployment_diff": "worker image only; reuse already deployed API image; API/router/config/schedules unchanged",
"tests": {
"focused_grant_tests_passed": 21,
"queue_repair_guard_tests_passed": 9,
"native_binding_tests_passed": 5,
"actual_image_rule_emission_queue_support": true
},
"argocd": {
"sync": "Synced",
"health": "Healthy",
"phase": "Succeeded"
},
"deployments": [
{
"name": "actcore-api",
"generation": 50,
"observed_generation": 50,
"ready": 1,
"image": "forgejo.coulomb.social/coulomb/activity-core@sha256:713bddad10a41950f446100a8b370fca8ccdd0b8969cccae751e3870c9c63ccd"
},
{
"name": "actcore-worker",
"generation": 63,
"observed_generation": 63,
"ready": 1,
"image": "forgejo.coulomb.social/coulomb/activity-core@sha256:713bddad10a41950f446100a8b370fca8ccdd0b8969cccae751e3870c9c63ccd"
},
{
"name": "actcore-event-router",
"generation": 33,
"observed_generation": 33,
"ready": 1,
"image": "forgejo.coulomb.social/coulomb/activity-core@sha256:cd4e924c2809f0d1d319e53a9ddd20db2e88a1b543d62e6442702c184b7842f4"
}
]
}

44
docs/evidence/2026-09-27-metered-acceptance-confirmations.json Normal file
View file

@ -0,0 +1,44 @@
[
{
"memo_id": "metered-20260927-provider-apply",
"disposition": "accept",
"approval_id": "0c05bd0a-f81f-451d-840c-5565628e2edc",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:23:59+00:00"
},
{
"memo_id": "metered-20260927-provider-exec",
"disposition": "accept",
"approval_id": "7b32443a-a817-400c-a130-01b9ef04c8ee",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:24:15+00:00"
},
{
"memo_id": "metered-20260927-provider-verify",
"disposition": "accept",
"approval_id": "47f118a3-a86c-43ad-969d-42e09a0f45bb",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:24:32+00:00"
},
{
"memo_id": "metered-20260927-worker-apply",
"disposition": "accept",
"approval_id": "2ce76d7f-01c3-4b63-8476-8d1230679769",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:24:43+00:00"
},
{
"memo_id": "metered-20260927-worker-exec",
"disposition": "accept",
"approval_id": "dc22666a-d5d7-46bc-9490-ebe9fe09dc38",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:24:55+00:00"
},
{
"memo_id": "metered-20260927-worker-verify",
"disposition": "accept",
"approval_id": "c2af4bcf-15b4-4305-aac1-acc7e4dcb099",
"native_submission_state": "confirmed",
"approved_at": "2026-09-27T19:25:10+00:00"
}
]

12
docs/evidence/2026-09-27-metered-attended-execution.json Normal file
View file

@ -0,0 +1,12 @@
{
"phase": "failed",
"status": "failed",
"credential_values_emitted": false,
"reader_scope_verified": true,
"kv_version": 1,
"named_owner_images_verified": true,
"remote_exit_code": 1,
"owner_forwards_closed": true,
"failure_type": "ValueError",
"failure_code": "attended_execution_failed"
}

12
docs/evidence/2026-09-27-metered-attended-resume.json Normal file
View file

@ -0,0 +1,12 @@
{
"phase": "failed",
"status": "failed",
"credential_values_emitted": false,
"reader_scope_verified": true,
"kv_version": 1,
"named_owner_images_verified": true,
"remote_exit_code": 1,
"owner_forwards_closed": true,
"failure_type": "ValueError",
"failure_code": "attended_execution_failed"
}

32
docs/evidence/2026-09-27-metered-native-consumes.json Normal file
View file

@ -0,0 +1,32 @@
{
"0c05bd0a-f81f-451d-840c-5565628e2edc": {
"approval_id": "0c05bd0a-f81f-451d-840c-5565628e2edc",
"valid_now": false,
"consumed": true
},
"47f118a3-a86c-43ad-969d-42e09a0f45bb": {
"approval_id": "47f118a3-a86c-43ad-969d-42e09a0f45bb",
"valid_now": false,
"consumed": true
},
"7b32443a-a817-400c-a130-01b9ef04c8ee": {
"approval_id": "7b32443a-a817-400c-a130-01b9ef04c8ee",
"valid_now": false,
"consumed": true
},
"2ce76d7f-01c3-4b63-8476-8d1230679769": {
"approval_id": "2ce76d7f-01c3-4b63-8476-8d1230679769",
"valid_now": false,
"consumed": true
},
"c2af4bcf-15b4-4305-aac1-acc7e4dcb099": {
"approval_id": "c2af4bcf-15b4-4305-aac1-acc7e4dcb099",
"valid_now": false,
"consumed": true
},
"dc22666a-d5d7-46bc-9490-ebe9fe09dc38": {
"approval_id": "dc22666a-d5d7-46bc-9490-ebe9fe09dc38",
"valid_now": false,
"consumed": true
}
}

60
docs/evidence/2026-09-27-metered-native-execution.json Normal file
View file

@ -0,0 +1,60 @@
{
"status": "failed",
"phase": "one_trigger_started",
"observed_at": "2026-09-27T19:48:11.627308+00:00",
"actions": [
{
"catalog": "glas-claude-agent-dev-anthropic",
"action": "apply",
"approval_id": "0c05bd0a-f81f-451d-840c-5565628e2edc",
"exit_code": 0,
"limits": {
"token_ttl": 300,
"token_max_ttl": 900,
"secret_id_ttl": 300,
"secret_id_num_uses": 1,
"token_num_uses": 8
}
},
{
"catalog": "activity-core-metered-worker-token",
"action": "apply",
"approval_id": "2ce76d7f-01c3-4b63-8476-8d1230679769",
"exit_code": 0,
"limits": {
"token_ttl": 300,
"token_max_ttl": 900,
"secret_id_ttl": 300,
"secret_id_num_uses": 1,
"token_num_uses": 8
}
},
{
"catalog": "glas-claude-agent-dev-anthropic",
"action": "verify",
"approval_id": "47f118a3-a86c-43ad-969d-42e09a0f45bb",
"exit_code": 0,
"sibling_denied": true,
"metadata_denied": true,
"session_revoked": true,
"revoked_lookup_status": 403
},
{
"catalog": "activity-core-metered-worker-token",
"action": "verify",
"approval_id": "c2af4bcf-15b4-4305-aac1-acc7e4dcb099",
"exit_code": 0,
"sibling_denied": true,
"metadata_denied": true,
"session_revoked": true,
"revoked_lookup_status": 403
}
],
"automatic_retry": false,
"trigger": {
"workflow_id": "activity-5bae5505-77f1-5ba3-8dfa-2e10ed15531e:manual-dab6c4c7-db03-4887-a17b-9d99b752482e",
"trigger_key": "manual-dab6c4c7-db03-4887-a17b-9d99b752482e"
},
"failure_type": "ValueError",
"failure_code": "natural_queue_binding_refused"
}

63
docs/evidence/2026-09-27-metered-native-resume.json Normal file
View file

@ -0,0 +1,63 @@
{
"status": "attempt_failed",
"phase": "one_exec_returned",
"observed_at": "2026-09-27T20:01:30.849242+00:00",
"actions": [
{
"catalog": "glas-claude-agent-dev-anthropic",
"action": "exec",
"approval_id": "7b32443a-a817-400c-a130-01b9ef04c8ee",
"exit_code": 1
},
{
"catalog": "activity-core-metered-worker-token",
"action": "exec",
"approval_id": "dc22666a-d5d7-46bc-9490-ebe9fe09dc38",
"exit_code": 1
}
],
"automatic_retry": false,
"prior_receipt": "execution.json",
"completed_actions_not_replayed": [
"provider/apply",
"worker/apply",
"provider/verify",
"worker/verify",
"queue/trigger"
],
"trigger": {
"workflow_id": "activity-5bae5505-77f1-5ba3-8dfa-2e10ed15531e:manual-dab6c4c7-db03-4887-a17b-9d99b752482e",
"trigger_key": "manual-dab6c4c7-db03-4887-a17b-9d99b752482e"
},
"queued_run": {
"id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"state": "open",
"attempt": 0,
"claim_owner": null,
"target_repo": "hfact-glas-proof",
"harness_profile_ref": "harness.agent-dev-local@1.1.1",
"repository_grant": {
"publish": false,
"version": "1",
"commit_count": {
"max": 1,
"min": 1
},
"allowed_paths": [
"PROOF.md"
]
},
"triggering_event_id": "manual-dab6c4c7-db03-4887-a17b-9d99b752482e"
},
"exec_exit_code": 1,
"owner_results": [
{
"ok": false,
"claimed": true,
"empty": false,
"run_id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"ops_state": "failed"
}
],
"private_runtime_removed": true
}

39
docs/evidence/2026-09-27-metered-postflight.json Normal file
View file

@ -0,0 +1,39 @@
{
"runtime_sha256": "b6e4e8a429393d68831c996a65c0489664205df969ac9882e581983ec2da4969",
"repository_head": "679d23e0517707ba63e25399b5262f0d2f37315d",
"repository_clean": true,
"repository_lock_available": true,
"sandbox": {
"sandbox_id": "b67907f1",
"state": "destroyed",
"created_at": "2026-09-27T20:01:43.835992Z",
"destroyed_at": "2026-09-27T20:01:45.650937Z"
},
"removed_paths": {
"workspace_dir": true
},
"private_credential_directories_remaining": 0,
"close_outbox_pending": 0,
"close_outbox_quarantined": 0,
"spend_reservations": [
{
"run_id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"definition_id": "5bae5505-77f1-5ba3-8dfa-2e10ed15531e",
"idempotency_key": "5bae5505-77f1-5ba3-8dfa-2e10ed15531e:execute-hfact-glas-metered-proof:manual-dab6c4c7-db03-4887-a17b-9d99b752482e",
"attempt": 1,
"state": "held",
"liability": 10000000,
"start_day": "2026-09-27",
"end_day": null,
"observed_usd": null,
"receipt": null
}
],
"provider_request_reservations": [],
"request_routes": [
{
"run_id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"revoked": 1
}
]
}

27
docs/evidence/2026-09-27-metered-queue-grant-reconciliation.json Normal file
View file

@ -0,0 +1,27 @@
{
"status": "applied",
"observed_at": "2026-09-27T19:59:50.642053+00:00",
"worker_pod": "actcore-worker-659497dcf9-rvf4h",
"worker_image": "sha256:713bddad10a41950f446100a8b370fca8ccdd0b8969cccae751e3870c9c63ccd",
"run_id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"definition_id": "5bae5505-77f1-5ba3-8dfa-2e10ed15531e",
"definition_version": 1,
"triggering_event_id": "manual-dab6c4c7-db03-4887-a17b-9d99b752482e",
"source": "typed_existing_definition_rule",
"grant_id": "656911f7dff83e871434b415f0cee685",
"grant": {
"version": "1",
"allowed_paths": [
"PROOF.md"
],
"commit_count": {
"min": 1,
"max": 1
},
"publish": false
},
"attempt": 0,
"claim_owner": null,
"new_trigger": false,
"new_task": false
}

20
docs/evidence/2026-09-27-metered-renewal-installation.json Normal file
View file

@ -0,0 +1,20 @@
{
"status": "installed",
"observed_at": "2026-09-27T19:39:35.079399+00:00",
"old_counts": {
"reservations": 0,
"request_routes": 0,
"request_reservations": 0
},
"old_ledger_preserved": true,
"new_ledger": "/home/tegwick/hfact/owner-metered/spend-tool-proof-renewal-20260927.sqlite3",
"old_dispatch_pins_retired": true,
"owner_check": {
"ok": true,
"check_only": true,
"dispatch_enabled": false,
"policy_sha256": "de3d01c9f4753994e8f73c111cde328a649e05dbb2563c37b5c12d86a488b2fa",
"runtime_sha256": "b6e4e8a429393d68831c996a65c0489664205df969ac9882e581983ec2da4969"
},
"dispatches": 0
}

109
docs/evidence/2026-09-27-metered-terminal-delivery.json Normal file
View file

@ -0,0 +1,109 @@
{
"source": "railiance01/native-metered-20260927/native-evidence; allowlisted fields only",
"native_records": [
{
"action": "apply",
"catalog_id": "glas-claude-agent-dev-anthropic",
"result": "applied",
"record_id": "53136c39-5fb6-4f61-8d73-231c5d0fe57d",
"detail": {
"approval_consumed": true,
"approval_id": "0c05bd0a-f81f-451d-840c-5565628e2edc",
"approval_consume_idempotent": false
}
},
{
"action": "apply",
"catalog_id": "activity-core-metered-worker-token",
"result": "applied",
"record_id": "f218efea-a990-4393-a99c-e405050a349e",
"detail": {
"approval_consumed": true,
"approval_id": "2ce76d7f-01c3-4b63-8476-8d1230679769",
"approval_consume_idempotent": false
}
},
{
"action": "verify",
"catalog_id": "glas-claude-agent-dev-anthropic",
"result": "pass",
"record_id": "9a3347b1-9b1c-42a1-a504-ce791fdf1215",
"detail": {
"approval_consumed": true,
"approval_id": "47f118a3-a86c-43ad-969d-42e09a0f45bb",
"approval_consume_idempotent": false
}
},
{
"action": "verify",
"catalog_id": "activity-core-metered-worker-token",
"result": "pass",
"record_id": "43517ef4-ab0e-47c4-ae96-4284a6f67b2e",
"detail": {
"approval_consumed": true,
"approval_id": "c2af4bcf-15b4-4305-aac1-acc7e4dcb099",
"approval_consume_idempotent": false
}
},
{
"action": "exec",
"catalog_id": "activity-core-metered-worker-token",
"result": "exit-1",
"record_id": "7b3f492c-cf80-4e4d-8158-7e4504b136ca",
"detail": {
"approval_consumed": true,
"approval_id": "dc22666a-d5d7-46bc-9490-ebe9fe09dc38",
"approval_consume_idempotent": false,
"session": {
"established": true,
"revocation_attempted": true,
"revocation_succeeded": true,
"session_handle": "a0f9a121b064"
},
"companion_of": "glas-claude-agent-dev-anthropic",
"exec_owner_sha256": "310600eab467ed79fc3851178a065de12d57d3ada5bcf68d9c364ed11b3ee50f"
}
},
{
"action": "exec",
"catalog_id": "glas-claude-agent-dev-anthropic",
"result": "exit-1",
"record_id": "1b6fabda-dbe5-4fe3-a74e-b9abf62050c1",
"detail": {
"approval_consumed": true,
"approval_id": "7b32443a-a817-400c-a130-01b9ef04c8ee",
"approval_consume_idempotent": false,
"session": {
"established": true,
"revocation_attempted": true,
"revocation_succeeded": true,
"session_handle": "d7e5f59e383d"
},
"companions": [
"activity-core-metered-worker-token"
],
"exec_owner_sha256": "310600eab467ed79fc3851178a065de12d57d3ada5bcf68d9c364ed11b3ee50f"
}
}
],
"terminal_queue": {
"id": "6efa9436-6a91-47c0-94e5-b31b5a0e7e3a",
"state": "failed",
"attempt": 1,
"claim_owner": "rein-aharness-metered@railiance01",
"lease_until": null
},
"verification": {
"full_suite": "406 passed, 9 skipped",
"focused_owner_claim_spend_glas": "99 passed (includes new close-evidence test)",
"native_procedure_and_queue_guards": "14 passed",
"sandbox_builder": "12 passed",
"activity_core_grant": "21 passed"
},
"remaining": {
"workplan": "blocked",
"parent_eur_reservation": "10.00 held; observed USD unknown",
"retry_authorized": false,
"corrected_artifact_admitted": false
}
}

97
docs/native-metered-proof.md Normal file
View file

@ -0,0 +1,97 @@
# Attended disposable proof — 2026-09-27
Existing work: REINAH-WP-0003-T05/T06 and SECRETS-WP-0009-T03.
No new workplan, publication, automatic retry, or factory operating admission.
The operator accepted replacement spend memo `infd-20260927-b02`, confirmed
USD 10 including tax/conversion fees fits EUR 10, and accepted all six
`metered-20260927-{provider,worker}-{apply,verify,exec}` decisions. Native
submission confirmations and replacement host installation are separate receipts
in `docs/evidence/2026-09-27-metered-*.json`. The earlier expired spend grant is
preserved and never used.
`scripts/metered-native-owner.py` checks the frozen six-request packet and exact
recipient files before native claim/PDP checks. The provider's human-control
flag and the worker companion's ordinary flag remain unchanged. Apply and
verify each consume their own native approval before OpenBao effects. Exec
uses the existing Secrets Engine primary/companion consume and delivery code;
the procedure does not manufacture decisions, claims, or delivery state.
The attended workstation wrapper `scripts/attended-metered-proof.py` follows
the existing scoped approval-client reader and nested platform-admin OIDC
procedure. The operator-controlled SSH session runs the controller on Railiance
because the approved command and owner-file bindings name that host. Client
secret, short-lived operator/negative-test tokens and PDP caller token cross
encrypted SSH stdin only. The host uses a fresh owner-only temporary directory
on `/run/user/1000` tmpfs. Approval bearer tokens are minted in memory. No
cluster reader, persistent service, credential rotation or new custody path is
created. Warden self-revokes both attended sessions; private files and the
named-pod forwarding processes are removed on normal exit, including failure.
The delivered model child receives only its catalog-bound environment plus
its separately approved provider and worker credentials.
Before activation, the installer locks and checks all three old ledger tables,
refusing any prior reservation or liability. It preserves the old ledger and
backs up both old configuration files. A new private ledger is initialized
without resetting old evidence. Replacement file hashes retire dispatch using
the old catalog pins. The immutable runtime's backend-free owner check passes.
The single trigger occurs after both lanes verify. The controller waits for
exactly one open, unclaimed, attempt-zero task with the admitted profile and
one-file/one-commit/no-publication grant. It records trigger and exec intent
before either action. An existing execution receipt refuses all replays,
including uncertain/failed attempts. The scheduled definition stays disabled.
The approved maximum request hold remains USD 4.64; at most two such holds fit
the USD 10 liability cap. The [provider tariff](https://platform.claude.com/docs/en/about-claude/pricing)
was rechecked immediately before activation: Sonnet 5 global standard output is
USD 10/million tokens; the conservative input bound of USD 4/million covers
one-hour cache writes. The proof has no authority to enlarge these limits.
After interruption, inspect the durable local/remote receipts and native
consume state before any further action. Explicitly reconcile remaining
`metered-native-*` / `metered-attended-*` private runtime directories and any
open queue item; never rerun the command as a cleanup strategy. Hard-kill
cleanup is not claimed by this wrapper. Provider-request reservations remain
conservative until reconciled. T04's tenant migrations and broader T06
acceptance requirements remain in the existing workplan.
## Actual execution and corrections
All six decisions were accepted and consumed once. Both lane apply/verify
operations passed; exec delivered through two AppRole sessions whose revocation
succeeded. The single definition trigger initially produced a row with no grant:
the deployed Activity Core worker predated the API's grant-carriage support.
Activity Core `428f2d7` and Platform `c3607ff` changed only the worker image to
the already-deployed grant-aware API image through the existing GitOps parent
and child applications. Argo reports Synced/Healthy/Succeeded.
The explicit repair script validates and locks the original disabled definition
and original open, unclaimed, attempt-zero job. It copies only the exact typed
grant from that definition; it creates no row or trigger. Nine negative/positive
guard tests pass. This repair is recorded separately and is not represented as
successful natural grant carriage by the old producer.
The `resume` mode is limited to that recorded pre-execution queue-binding failure.
It verifies the four completed native actions, installed policies/role limits,
zero prior request/reservation counts, and the same repaired job. It repeats
neither apply/verify nor queue creation. Both remaining exec approvals were then
consumed. There is no further resume path for the attempted job.
Job `6efa9436-6a91-47c0-94e5-b31b5a0e7e3a` closed `failed`, attempt 1, with its
lease cleared. Its installed Python launchers still referenced a deleted build
directory: uv's long-path shell trampoline had escaped the builder's direct
shebang rewrite. A no-credential synthetic probe reproduced exit 127 for both
launchers inside the actual pinned bwrap artifact, with zero provider forwards.
The runtime builder correction is Sand-boxer `81b5fcf`; relocation tests execute
after the old build directory disappears. Rein now also rejects these launchers
before claim and retains bounded gateway stage/cleanup facts on accounting refusal.
The approved artifact itself remains unchanged. Sandbox `b67907f1` and its
workspace are destroyed; the repository is clean at its original commit, the
lock is available, and no close-outbox item or private credential directory is
left over. The request route is revoked and there are no provider-request
reservations. The parent EUR 10 reservation remains held; observed billing was
not invented and the ledger was not reset. A corrected artifact and its changed
pins need admission, held-liability reconciliation remains an owner action, and
another attempt requires separate authority. REINAH-WP-0003 stays blocked.

3
docs/owner-bootstrap.md
View file

@ -47,6 +47,9 @@ profile/descriptor digests, operational readiness, model, empty-egress bwrap pro
and runtime digest are checked before claim. Runtime, owner state and target checkout
must not overlap. Provision parent and request ledgers as separate reviewed actions.
The normal ACTIVITY_CORE/AGENT_HARNESS worker and state configuration still applies.
The governed Python console launchers must be executable regular files naming
`/opt/sandboxer/runtime/bin/python3`, rather than a build-host interpreter.
The installed bwrap proof also runs their `--help` commands before any model request.
This config is not an authorization decision or a custody provenance proof. The
invoking credential engine must already have passed its exact action approval,