feat(runtime): consume governed Activity Core closes
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
This commit is contained in:
parent
0e6d795aa4
commit
d00ffcb402
22 changed files with 1218 additions and 148 deletions
|
|
@ -17,12 +17,19 @@ import time
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from rein_aharness import metrics
|
||||
from rein_aharness.approaches import (
|
||||
APPROACH_UNMATCHED,
|
||||
ApproachResult,
|
||||
execute_approach,
|
||||
select_approach,
|
||||
)
|
||||
from rein_aharness.close_outbox import (
|
||||
CloseOutbox,
|
||||
CloseRequest,
|
||||
PermanentCloseDeliveryError,
|
||||
ReplayReport,
|
||||
)
|
||||
from rein_aharness.execution_cancel import (
|
||||
ExecutionCancel,
|
||||
ExecutionCancelled,
|
||||
|
|
@ -32,11 +39,13 @@ from rein_aharness.glas_execution import (
|
|||
GLAS_APPROACH,
|
||||
GlasExecutionError,
|
||||
execute_profiled_run,
|
||||
normalise_execution_evidence_for_close,
|
||||
)
|
||||
from rein_aharness.lease_monitor import LeaseLoss, LeaseMonitor
|
||||
from rein_aharness.repository_transaction import (
|
||||
DirtyRepositoryError,
|
||||
GitRepositoryError,
|
||||
RepositoryAcceptanceError,
|
||||
RepositoryBusyError,
|
||||
RepositoryTransaction,
|
||||
RepositoryTransactionError,
|
||||
|
|
@ -52,6 +61,17 @@ from rein_aharness.ops_run_client import (
|
|||
|
||||
logger = logging.getLogger("rein_aharness.claim_loop")
|
||||
|
||||
_PERMANENT_CLOSE_CODES = frozenset(
|
||||
{
|
||||
"not_found",
|
||||
"wrong_owner",
|
||||
"expired_lease",
|
||||
"state_conflict",
|
||||
"terminal_conflict",
|
||||
"evidence_conflict",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessResult:
|
||||
|
|
@ -180,16 +200,97 @@ def _cancelled_result(
|
|||
return None
|
||||
|
||||
|
||||
def _deliver_close(client: ActivityCoreOpsClient, request: CloseRequest) -> OpsRun:
|
||||
if request.worker_id != client.config.worker_id:
|
||||
raise PermanentCloseDeliveryError("outbox worker identity no longer matches")
|
||||
try:
|
||||
if request.action == "complete":
|
||||
return client.complete(request.run_id, result=request.result)
|
||||
return client.fail(
|
||||
request.run_id,
|
||||
error=request.error,
|
||||
reopen=request.reopen,
|
||||
result=request.result,
|
||||
)
|
||||
except OpsRunError as exc:
|
||||
if exc.code in _PERMANENT_CLOSE_CODES:
|
||||
raise PermanentCloseDeliveryError(
|
||||
f"Activity Core refused close with {exc.code}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
|
||||
def replay_close_outbox(
|
||||
client: ActivityCoreOpsClient,
|
||||
outbox: CloseOutbox | None = None,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> ReplayReport:
|
||||
"""Replay close-only work; this function never dispatches a workload."""
|
||||
store = outbox or CloseOutbox()
|
||||
return store.replay(lambda request: _deliver_close(client, request), limit=limit)
|
||||
|
||||
|
||||
def _durable_close(
|
||||
client: ActivityCoreOpsClient,
|
||||
outbox: CloseOutbox,
|
||||
request: CloseRequest,
|
||||
) -> tuple[bool, str | None, str]:
|
||||
"""Enqueue before delivery and report the durable outcome for this entry."""
|
||||
receipt = outbox.enqueue(request)
|
||||
delivered: dict[str, OpsRun] = {}
|
||||
|
||||
def deliver(item: CloseRequest) -> None:
|
||||
delivered[item.entry_id] = _deliver_close(client, item)
|
||||
|
||||
report = outbox.replay(deliver)
|
||||
state = outbox.entry_state(receipt.entry_id)
|
||||
if state != "delivered":
|
||||
reason = (
|
||||
"close evidence quarantined"
|
||||
if state == "quarantined"
|
||||
else "close evidence remains pending"
|
||||
)
|
||||
return False, None, reason
|
||||
row = delivered.get(receipt.entry_id)
|
||||
if row is not None:
|
||||
return True, row.state, ""
|
||||
expected_state = "succeeded" if request.action == "complete" else "failed"
|
||||
if report.delivered:
|
||||
logger.info("replayed %s close evidence entries", report.delivered)
|
||||
return True, expected_state, ""
|
||||
|
||||
|
||||
def _bounded_reason(value: str, *, default: str) -> str:
|
||||
cleaned = " ".join(str(value).split())
|
||||
return cleaned[:500] or default
|
||||
|
||||
|
||||
def process_one(
|
||||
client: ActivityCoreOpsClient | None = None,
|
||||
*,
|
||||
report_to_hub: bool = True,
|
||||
commit: bool = True,
|
||||
dry_run: bool = False,
|
||||
outbox: CloseOutbox | None = None,
|
||||
) -> ProcessResult:
|
||||
"""Claim at most one ops_run, execute approach, complete or fail."""
|
||||
client = client or ActivityCoreOpsClient()
|
||||
cfg = client.config
|
||||
outbox = outbox or CloseOutbox()
|
||||
|
||||
replay = replay_close_outbox(client, outbox)
|
||||
if replay.remaining:
|
||||
return ProcessResult(
|
||||
claimed=False,
|
||||
retry_full_interval=True,
|
||||
reason=(
|
||||
"required Activity Core close evidence remains pending "
|
||||
f"(attempted={replay.attempted} remaining={replay.remaining})"
|
||||
),
|
||||
)
|
||||
if replay.quarantined:
|
||||
logger.error("quarantined %s close evidence entries", replay.quarantined)
|
||||
|
||||
try:
|
||||
claimed = client.claim(limit=1)
|
||||
|
|
@ -244,6 +345,36 @@ def process_one(
|
|||
client,
|
||||
run,
|
||||
report_to_hub=report_to_hub,
|
||||
outbox=outbox,
|
||||
)
|
||||
|
||||
if run.repository_grant is not None:
|
||||
reason = (
|
||||
"refused: repository_grant requires an authoritative "
|
||||
"harness_profile_ref"
|
||||
)
|
||||
try:
|
||||
out = client.fail(
|
||||
run.id,
|
||||
error=reason,
|
||||
reopen=False,
|
||||
result={"ok": False, "approach": approach, "reason": reason},
|
||||
)
|
||||
except OpsRunError as exc:
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=approach,
|
||||
ok=False,
|
||||
reason=f"grant-route refusal close failed: {exc}",
|
||||
)
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=approach,
|
||||
ok=False,
|
||||
reason=reason,
|
||||
ops_state=out.state,
|
||||
)
|
||||
|
||||
cancel = ExecutionCancel()
|
||||
|
|
@ -340,61 +471,213 @@ def _process_profiled_run(
|
|||
run: OpsRun,
|
||||
*,
|
||||
report_to_hub: bool,
|
||||
outbox: CloseOutbox,
|
||||
) -> ProcessResult:
|
||||
"""Execute a profiled row without consulting or falling back to legacy routing."""
|
||||
"""Keep the claim heartbeat alive through evidence persistence and close."""
|
||||
cancel = ExecutionCancel()
|
||||
hb = _Heartbeat(client, run.id, client.config.lease_seconds, cancel=cancel)
|
||||
hb.start()
|
||||
execution_error: str | None = None
|
||||
execution_reason = ""
|
||||
execution_reopen = False
|
||||
gateway_result: dict[str, Any] | None = None
|
||||
tx_evidence: dict[str, Any] | None = None
|
||||
_set_active_run_cancel(cancel)
|
||||
try:
|
||||
try:
|
||||
target = resolve_ops_target(run, client.config)
|
||||
with using_cancel(cancel):
|
||||
with RepositoryTransaction(target, correlation_id=run.id) as tx:
|
||||
gateway_result = execute_profiled_run(
|
||||
run,
|
||||
client.config,
|
||||
report_to_hub=report_to_hub,
|
||||
cancel=cancel,
|
||||
)
|
||||
tx_evidence = tx.evidence()
|
||||
except ExecutionCancelled as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = str(exc)
|
||||
except DirtyRepositoryError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
except RepositoryBusyError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
execution_reopen = True
|
||||
except (GitRepositoryError, RepositoryTransactionError, TaskSpecError) as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
except GlasExecutionError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = str(exc)
|
||||
return _process_profiled_run_active(
|
||||
client,
|
||||
run,
|
||||
report_to_hub=report_to_hub,
|
||||
outbox=outbox,
|
||||
cancel=cancel,
|
||||
hb=hb,
|
||||
)
|
||||
finally:
|
||||
hb.stop()
|
||||
_set_active_run_cancel(None)
|
||||
|
||||
|
||||
def _process_profiled_run_active(
|
||||
client: ActivityCoreOpsClient,
|
||||
run: OpsRun,
|
||||
*,
|
||||
report_to_hub: bool,
|
||||
outbox: CloseOutbox,
|
||||
cancel: ExecutionCancel,
|
||||
hb: _Heartbeat,
|
||||
) -> ProcessResult:
|
||||
"""Execute a profiled row without consulting or falling back to legacy routing."""
|
||||
execution_error: str | None = None
|
||||
execution_reason = ""
|
||||
execution_reopen = False
|
||||
gateway_result: dict[str, Any] | None = None
|
||||
safe_evidence: dict[str, Any] = {}
|
||||
execution_ok = False
|
||||
tx: RepositoryTransaction | None = None
|
||||
tx_evidence: dict[str, Any] | None = None
|
||||
try:
|
||||
target = resolve_ops_target(run, client.config)
|
||||
with using_cancel(cancel):
|
||||
with RepositoryTransaction(target, correlation_id=run.id) as active_tx:
|
||||
tx = active_tx
|
||||
gateway_result = execute_profiled_run(
|
||||
run,
|
||||
client.config,
|
||||
report_to_hub=report_to_hub,
|
||||
cancel=cancel,
|
||||
)
|
||||
safe_evidence = normalise_execution_evidence_for_close(
|
||||
gateway_result["evidence"]
|
||||
)
|
||||
execution_ok = gateway_result["ok"]
|
||||
execution_reason = _bounded_reason(
|
||||
str(
|
||||
safe_evidence.get("error")
|
||||
or safe_evidence.get("outcome")
|
||||
or ""
|
||||
),
|
||||
default="Glas execution failed",
|
||||
)
|
||||
if run.repository_grant is not None:
|
||||
try:
|
||||
active_tx.validate_acceptance(
|
||||
run.repository_grant.acceptance_policy()
|
||||
)
|
||||
except RepositoryAcceptanceError as exc:
|
||||
if execution_ok or exc.code != "head-unchanged":
|
||||
execution_ok = False
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = _bounded_reason(
|
||||
str(exc),
|
||||
default="repository acceptance failed",
|
||||
)
|
||||
except ExecutionCancelled as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = _bounded_reason(str(exc), default="execution cancelled")
|
||||
except DirtyRepositoryError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = _bounded_reason(
|
||||
f"refused: {exc}", default="dirty repository refused"
|
||||
)
|
||||
except RepositoryBusyError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = _bounded_reason(
|
||||
f"refused: {exc}", default="busy repository refused"
|
||||
)
|
||||
execution_reopen = True
|
||||
except (GitRepositoryError, RepositoryTransactionError, TaskSpecError) as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = _bounded_reason(
|
||||
f"refused: {exc}", default="repository transaction refused"
|
||||
)
|
||||
except GlasExecutionError:
|
||||
execution_error = "GlasExecutionError"
|
||||
execution_reason = "profiled execution failed (GlasExecutionError)"
|
||||
|
||||
if tx is not None and tx.baseline is not None:
|
||||
tx_evidence = tx.evidence()
|
||||
if run.repository_grant is not None:
|
||||
tx_evidence["repository_grant"] = run.repository_grant.evidence()
|
||||
|
||||
if run.repository_grant is not None and tx is not None:
|
||||
try:
|
||||
accepted = tx.acceptance
|
||||
commit_sha = safe_evidence.get("commit_sha")
|
||||
metrics.record_external_execution(
|
||||
tx.repo,
|
||||
"rein-aharness",
|
||||
success=(
|
||||
execution_error is None
|
||||
and execution_ok
|
||||
and not cancel.cancelled
|
||||
and not hb.monitor.lost
|
||||
),
|
||||
committed=bool(accepted or commit_sha),
|
||||
head_after=(
|
||||
accepted.head
|
||||
if accepted is not None
|
||||
else commit_sha if isinstance(commit_sha, str) else None
|
||||
),
|
||||
reason=execution_reason or None,
|
||||
metadata={
|
||||
"ops_run_id": run.id,
|
||||
"harness_profile_ref": run.harness_profile_ref,
|
||||
"repository_grant_id": run.repository_grant.grant_id,
|
||||
},
|
||||
session_id=tx.transaction_id,
|
||||
)
|
||||
assert tx_evidence is not None
|
||||
tx_evidence["metrics"] = {
|
||||
"storage": "external",
|
||||
"session_id": tx.transaction_id,
|
||||
"projection_ready": True,
|
||||
}
|
||||
except OSError:
|
||||
execution_ok = False
|
||||
execution_error = "ExternalMetricsError"
|
||||
execution_reason = "required external metrics persistence failed"
|
||||
|
||||
# Re-check cancellation after metrics persistence. The heartbeat remains
|
||||
# active through this point and through close delivery, so a late signal or
|
||||
# lease loss cannot turn a cancelled execution into a successful close.
|
||||
refused = _cancelled_result(run.id, GLAS_APPROACH, hb)
|
||||
if refused is not None:
|
||||
if tx_evidence is not None:
|
||||
refused.detail["repository_transaction"] = tx_evidence
|
||||
if execution_error == "ExternalMetricsError":
|
||||
refused.reason = (
|
||||
f"{refused.reason}; required external metrics persistence failed"
|
||||
)
|
||||
if hb.monitor.lost or tx is None or tx_evidence is None:
|
||||
return refused
|
||||
payload = {
|
||||
"ok": False,
|
||||
"approach": GLAS_APPROACH,
|
||||
"reason": refused.reason,
|
||||
"repository_transaction": tx_evidence,
|
||||
}
|
||||
try:
|
||||
delivered, state, close_reason = _durable_close(
|
||||
client,
|
||||
outbox,
|
||||
CloseRequest(
|
||||
run_id=run.id,
|
||||
transaction_id=tx.transaction_id,
|
||||
worker_id=client.config.worker_id,
|
||||
action="fail",
|
||||
result=payload,
|
||||
error=refused.reason,
|
||||
reopen=False,
|
||||
),
|
||||
)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
refused.reason = f"close evidence failure ({type(exc).__name__})"
|
||||
return refused
|
||||
if delivered:
|
||||
refused.ops_state = state
|
||||
else:
|
||||
refused.reason = close_reason
|
||||
return refused
|
||||
|
||||
if execution_error is not None:
|
||||
reason = execution_reason or f"profiled execution failed ({execution_error})"
|
||||
ok = execution_error is None and execution_ok
|
||||
reason = execution_reason or (
|
||||
"profiled execution failed"
|
||||
if not ok
|
||||
else ""
|
||||
)
|
||||
payload: dict[str, Any] = {
|
||||
"ok": ok,
|
||||
"approach": GLAS_APPROACH,
|
||||
}
|
||||
if safe_evidence:
|
||||
payload["execution_evidence"] = safe_evidence
|
||||
if reason:
|
||||
payload["reason"] = reason
|
||||
if tx_evidence is not None:
|
||||
payload["repository_transaction"] = tx_evidence
|
||||
|
||||
if tx is None or tx_evidence is None:
|
||||
try:
|
||||
out = client.fail(
|
||||
run.id,
|
||||
error=reason,
|
||||
reopen=execution_reopen,
|
||||
result={"ok": False, "approach": GLAS_APPROACH, "reason": reason},
|
||||
result=payload,
|
||||
)
|
||||
except OpsRunError as close_exc:
|
||||
return ProcessResult(
|
||||
|
|
@ -403,6 +686,7 @@ def _process_profiled_run(
|
|||
approach=GLAS_APPROACH,
|
||||
ok=False,
|
||||
reason=f"close ops_run failed: {close_exc}; {reason}",
|
||||
detail=payload,
|
||||
)
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
|
|
@ -411,34 +695,43 @@ def _process_profiled_run(
|
|||
ok=False,
|
||||
reason=reason,
|
||||
ops_state=out.state,
|
||||
detail=payload,
|
||||
)
|
||||
|
||||
assert gateway_result is not None
|
||||
evidence = gateway_result["evidence"]
|
||||
ok = gateway_result["ok"]
|
||||
reason = str(evidence.get("error") or evidence.get("outcome") or "Glas execution failed")
|
||||
detail = {"execution_evidence": evidence}
|
||||
if tx_evidence is not None:
|
||||
detail["repository_transaction"] = tx_evidence
|
||||
gateway_result = {**gateway_result, "repository_transaction": tx_evidence}
|
||||
action = "complete" if ok else "fail"
|
||||
try:
|
||||
if ok:
|
||||
out = client.complete(run.id, result=gateway_result)
|
||||
else:
|
||||
out = client.fail(
|
||||
run.id,
|
||||
error=reason,
|
||||
reopen=False,
|
||||
result=gateway_result,
|
||||
)
|
||||
except OpsRunError as exc:
|
||||
close_request = CloseRequest(
|
||||
run_id=run.id,
|
||||
transaction_id=tx.transaction_id,
|
||||
worker_id=client.config.worker_id,
|
||||
action=action,
|
||||
result=payload,
|
||||
error=reason if action == "fail" else "",
|
||||
reopen=False,
|
||||
)
|
||||
delivered, state, close_reason = _durable_close(
|
||||
client,
|
||||
outbox,
|
||||
close_request,
|
||||
)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=GLAS_APPROACH,
|
||||
ok=False,
|
||||
reason=f"close ops_run failed: {exc}; gateway_ok={ok} {reason}",
|
||||
detail=detail,
|
||||
reason=f"close evidence failure ({type(exc).__name__})",
|
||||
detail=payload,
|
||||
)
|
||||
|
||||
if not delivered:
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=GLAS_APPROACH,
|
||||
ok=False,
|
||||
reason=close_reason,
|
||||
detail=payload,
|
||||
)
|
||||
|
||||
return ProcessResult(
|
||||
|
|
@ -447,8 +740,8 @@ def _process_profiled_run(
|
|||
approach=GLAS_APPROACH,
|
||||
ok=ok,
|
||||
reason="" if ok else reason,
|
||||
ops_state=out.state,
|
||||
detail=detail,
|
||||
ops_state=state,
|
||||
detail=payload,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,35 @@ def _cmd_claim_loop(args: argparse.Namespace) -> int:
|
|||
)
|
||||
|
||||
|
||||
def _cmd_close_outbox(args: argparse.Namespace) -> int:
|
||||
from rein_aharness.claim_loop import replay_close_outbox
|
||||
from rein_aharness.close_outbox import CloseOutbox
|
||||
from rein_aharness.ops_run_client import ActivityCoreOpsClient
|
||||
|
||||
outbox = CloseOutbox()
|
||||
if args.action == "replay":
|
||||
if not 1 <= args.limit <= 1000:
|
||||
print("error: --limit must be between 1 and 1000", file=sys.stderr)
|
||||
return 2
|
||||
report = replay_close_outbox(
|
||||
ActivityCoreOpsClient(),
|
||||
outbox,
|
||||
limit=args.limit,
|
||||
)
|
||||
payload = {
|
||||
"attempted": report.attempted,
|
||||
"delivered": report.delivered,
|
||||
"failed": report.failed,
|
||||
"quarantined_this_run": report.quarantined,
|
||||
"remaining": report.remaining,
|
||||
**outbox.status(),
|
||||
}
|
||||
else:
|
||||
payload = outbox.status()
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
return 1 if payload["pending"] or payload["quarantined"] else 0
|
||||
|
||||
|
||||
def _cmd_run(args: argparse.Namespace) -> int:
|
||||
from rein_aharness.intake import IntakeError, IssueCoreClient, poll_next
|
||||
|
||||
|
|
@ -524,6 +553,18 @@ def main(argv: list[str] | None = None) -> int:
|
|||
claim_loop.add_argument("--no-commit", action="store_true")
|
||||
claim_loop.add_argument("-v", "--verbose", action="store_true")
|
||||
|
||||
close_outbox = sub.add_parser(
|
||||
"close-outbox",
|
||||
help="Inspect or replay durable Activity Core terminal-close evidence",
|
||||
)
|
||||
close_outbox.add_argument("action", choices=("status", "replay"))
|
||||
close_outbox.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Maximum pending entries to replay (default: 100)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.command == "validate":
|
||||
|
|
@ -538,6 +579,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||
if args.command == "claim-loop":
|
||||
return _cmd_claim_loop(args)
|
||||
|
||||
if args.command == "close-outbox":
|
||||
return _cmd_close_outbox(args)
|
||||
|
||||
if args.command == "run":
|
||||
return _cmd_run(args)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
"""Durable, idempotent close-evidence outbox core.
|
||||
"""Durable, idempotent Activity Core close-evidence outbox.
|
||||
|
||||
The outbox is intentionally not wired into the live claim loop yet. It stores
|
||||
bounded Activity Core completion/failure intents outside target checkouts so a
|
||||
future reconciler can retry close delivery without re-running repository work.
|
||||
It stores bounded completion/failure intents outside target checkouts so the
|
||||
claim worker can retry close delivery without re-running repository work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -32,6 +31,7 @@ _RECORD_KEYS = frozenset(
|
|||
"entry_id",
|
||||
"run_id",
|
||||
"transaction_id",
|
||||
"worker_id",
|
||||
"action",
|
||||
"result",
|
||||
"error",
|
||||
|
|
@ -61,12 +61,17 @@ class OutboxCorruptError(CloseOutboxError):
|
|||
"""A durable record cannot be safely decoded or validated."""
|
||||
|
||||
|
||||
class PermanentCloseDeliveryError(CloseOutboxError):
|
||||
"""Activity Core conclusively refused a close intent; do not retry it."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CloseRequest:
|
||||
"""One bounded Activity Core completion or failure intent."""
|
||||
|
||||
run_id: str
|
||||
transaction_id: str
|
||||
worker_id: str
|
||||
action: str
|
||||
result: dict[str, Any]
|
||||
error: str = ""
|
||||
|
|
@ -80,6 +85,7 @@ class CloseRequest:
|
|||
)
|
||||
_validate_identifier(self.run_id, "run_id")
|
||||
_validate_identifier(self.transaction_id, "transaction_id")
|
||||
_validate_identifier(self.worker_id, "worker_id")
|
||||
if self.action not in {"complete", "fail"}:
|
||||
raise InvalidCloseRequestError("action must be 'complete' or 'fail'")
|
||||
if not isinstance(self.result, dict):
|
||||
|
|
@ -118,6 +124,7 @@ class CloseRequest:
|
|||
"entry_id": self.entry_id,
|
||||
"run_id": self.run_id,
|
||||
"transaction_id": self.transaction_id,
|
||||
"worker_id": self.worker_id,
|
||||
"action": self.action,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
|
|
@ -128,6 +135,7 @@ class CloseRequest:
|
|||
return CloseRequest(
|
||||
run_id=self.run_id,
|
||||
transaction_id=self.transaction_id,
|
||||
worker_id=self.worker_id,
|
||||
action=self.action,
|
||||
result=json.loads(json.dumps(self.result)),
|
||||
error=self.error,
|
||||
|
|
@ -203,6 +211,7 @@ class _Record:
|
|||
request = CloseRequest(
|
||||
run_id=value["run_id"],
|
||||
transaction_id=value["transaction_id"],
|
||||
worker_id=value["worker_id"],
|
||||
action=value["action"],
|
||||
result=value["result"],
|
||||
error=value["error"],
|
||||
|
|
@ -328,6 +337,10 @@ class CloseOutbox:
|
|||
_atomic_write_json(path, trying.payload())
|
||||
try:
|
||||
deliver(record.request.detached_copy())
|
||||
except PermanentCloseDeliveryError as exc:
|
||||
self._quarantine(path, str(exc))
|
||||
quarantined += 1
|
||||
continue
|
||||
except Exception as exc:
|
||||
retained = _Record(
|
||||
request=record.request,
|
||||
|
|
@ -370,6 +383,32 @@ class CloseOutbox:
|
|||
with self._locked():
|
||||
return len(tuple(self.pending_dir.glob("*.json")))
|
||||
|
||||
def status(self) -> dict[str, int]:
|
||||
"""Return bounded operator counts without decoding record contents."""
|
||||
with self._locked():
|
||||
quarantined = tuple(
|
||||
path
|
||||
for path in self.quarantine_dir.glob("*.json")
|
||||
if not path.name.endswith(".error.json")
|
||||
)
|
||||
return {
|
||||
"pending": len(tuple(self.pending_dir.glob("*.json"))),
|
||||
"delivered": len(tuple(self.delivered_dir.glob("*.json"))),
|
||||
"quarantined": len(quarantined),
|
||||
}
|
||||
|
||||
def entry_state(self, entry_id: str) -> str | None:
|
||||
"""Return the durable state for one already-validated entry id."""
|
||||
_validate_identifier(entry_id, "entry_id")
|
||||
with self._locked():
|
||||
if (self.delivered_dir / f"{entry_id}.json").exists():
|
||||
return "delivered"
|
||||
if (self.pending_dir / f"{entry_id}.json").exists():
|
||||
return "pending"
|
||||
if tuple(self.quarantine_dir.glob(f"{entry_id}.*.json")):
|
||||
return "quarantined"
|
||||
return None
|
||||
|
||||
def _load(self, path: Path) -> _Record:
|
||||
try:
|
||||
if path.stat().st_size > _MAX_PAYLOAD_BYTES + 8192:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ with an actionable error if the governed runtime is not installed.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -24,12 +25,86 @@ _SCALAR_REFS = (
|
|||
"duty_ref",
|
||||
)
|
||||
_LIST_REFS = ("goal_refs", "resource_envelope_refs")
|
||||
_EVIDENCE_STRING_FIELDS = (
|
||||
"request_id",
|
||||
"correlation_id",
|
||||
"actor",
|
||||
"project",
|
||||
"target_repo",
|
||||
"contract_version",
|
||||
"profile_ref",
|
||||
"rein_id",
|
||||
"rein_version",
|
||||
"model_route",
|
||||
"resolved_model",
|
||||
"sandbox_profile",
|
||||
"sandbox_id",
|
||||
"tool_profile",
|
||||
"outcome",
|
||||
"failure_stage",
|
||||
"error",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"commit_sha",
|
||||
"tool_events_completeness",
|
||||
)
|
||||
_EVIDENCE_NUMBER_FIELDS = (
|
||||
"duration_s",
|
||||
"tokens_spent",
|
||||
"token_budget",
|
||||
"tool_events_count",
|
||||
)
|
||||
|
||||
|
||||
class GlasExecutionError(RuntimeError):
|
||||
"""The authoritative Glas invocation could not produce a GatewayResult."""
|
||||
|
||||
|
||||
def normalise_execution_evidence_for_close(raw: Any) -> dict[str, Any]:
|
||||
"""Retain only bounded Glas evidence fields safe for durable close state."""
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
result: dict[str, Any] = {}
|
||||
for key in _EVIDENCE_STRING_FIELDS:
|
||||
value = raw.get(key)
|
||||
if isinstance(value, str):
|
||||
result[key] = value[:2000] if key == "error" else value[:500]
|
||||
for key in _EVIDENCE_NUMBER_FIELDS:
|
||||
value = raw.get(key)
|
||||
if (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(value)
|
||||
and value >= 0
|
||||
):
|
||||
result[key] = value
|
||||
artifacts = raw.get("artifacts")
|
||||
if isinstance(artifacts, list):
|
||||
result["artifacts"] = [
|
||||
item[:1000]
|
||||
for item in artifacts[:20]
|
||||
if isinstance(item, str) and item
|
||||
]
|
||||
refs: dict[str, Any] = {}
|
||||
raw_refs = raw.get("refs")
|
||||
if isinstance(raw_refs, dict):
|
||||
for key in _SCALAR_REFS:
|
||||
value = raw_refs.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
refs[key] = value[:500]
|
||||
for key in _LIST_REFS:
|
||||
value = raw_refs.get(key)
|
||||
if isinstance(value, list):
|
||||
refs[key] = [
|
||||
item[:500]
|
||||
for item in value[:50]
|
||||
if isinstance(item, str) and item
|
||||
]
|
||||
if refs:
|
||||
result["refs"] = refs
|
||||
return result
|
||||
|
||||
|
||||
def _request_kwargs(run: OpsRun, config: OpsRunConfig, report_to_hub: bool) -> dict[str, Any]:
|
||||
if not run.harness_profile_ref:
|
||||
raise GlasExecutionError(f"ops_run {run.id} has no harness_profile_ref")
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from typing import Any
|
|||
import httpx
|
||||
|
||||
from rein_aharness.intake import resolve_target_repo
|
||||
from rein_aharness.repository_grant import RepositoryGrant, RepositoryGrantError
|
||||
from rein_aharness.taskspec import TaskSpec, TaskSpecError
|
||||
|
||||
DEFAULT_ACTIVITY_CORE_URL = "http://127.0.0.1:8010"
|
||||
|
|
@ -41,10 +42,12 @@ class OpsRunError(RuntimeError):
|
|||
*,
|
||||
action: str | None = None,
|
||||
status_code: int | None = None,
|
||||
code: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.action = action
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
|
||||
@property
|
||||
def lease_rejected(self) -> bool:
|
||||
|
|
@ -73,11 +76,25 @@ class OpsRun:
|
|||
approach_hint: str | None = None
|
||||
harness_profile_ref: str | None = None
|
||||
execution_refs: dict[str, Any] = field(default_factory=dict)
|
||||
repository_grant: RepositoryGrant | None = None
|
||||
result: dict[str, Any] = field(default_factory=dict)
|
||||
close_disposition: str | None = None
|
||||
close_intent_digest: str | None = None
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, data: dict[str, Any]) -> "OpsRun":
|
||||
grant = None
|
||||
if data.get("repository_grant") is not None:
|
||||
try:
|
||||
grant = RepositoryGrant.from_mapping(data["repository_grant"])
|
||||
except RepositoryGrantError as exc:
|
||||
run_id = str(data.get("id") or "unknown")[:200]
|
||||
raise OpsRunError(
|
||||
f"ops_run {run_id} has an invalid repository_grant",
|
||||
action="decode",
|
||||
code="invalid_repository_grant",
|
||||
) from exc
|
||||
return cls(
|
||||
id=str(data.get("id") or ""),
|
||||
activity_definition_id=str(data.get("activity_definition_id") or ""),
|
||||
|
|
@ -97,7 +114,14 @@ class OpsRun:
|
|||
approach_hint=data.get("approach_hint"),
|
||||
harness_profile_ref=data.get("harness_profile_ref"),
|
||||
execution_refs=dict(data.get("execution_refs") or {}),
|
||||
repository_grant=grant,
|
||||
result=dict(data.get("result") or {}),
|
||||
close_disposition=_bounded_optional_string(
|
||||
data.get("close_disposition"), 80
|
||||
),
|
||||
close_intent_digest=_bounded_optional_string(
|
||||
data.get("close_intent_digest"), 128
|
||||
),
|
||||
raw=data,
|
||||
)
|
||||
|
||||
|
|
@ -279,6 +303,7 @@ class ActivityCoreOpsClient:
|
|||
f"{action} ops_run {run_id} failed: HTTP {exc.response.status_code}",
|
||||
action=action,
|
||||
status_code=exc.response.status_code,
|
||||
code=_response_error_code(exc.response),
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise OpsRunError(
|
||||
|
|
@ -313,6 +338,7 @@ def ops_run_to_taskspec(
|
|||
hub_task_id=run.id,
|
||||
completion_event_type=completion_event_type,
|
||||
timeout_seconds=timeout_seconds,
|
||||
repository_grant=run.repository_grant,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -325,3 +351,19 @@ def resolve_ops_target(run: OpsRun, config: OpsRunConfig | None = None) -> Path:
|
|||
repo_map=cfg.repo_map,
|
||||
repo_roots=cfg.repo_roots,
|
||||
)
|
||||
|
||||
|
||||
def _response_error_code(response: httpx.Response) -> str | None:
|
||||
try:
|
||||
payload = response.json()
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
detail = payload.get("detail") if isinstance(payload, dict) else None
|
||||
code = detail.get("code") if isinstance(detail, dict) else None
|
||||
return _bounded_optional_string(code, 80)
|
||||
|
||||
|
||||
def _bounded_optional_string(value: Any, limit: int) -> str | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
return value[:limit]
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@
|
|||
|
||||
Wired into `run_task`, profile-absent `execute_approach` mutators, and the
|
||||
profiled claim path under REINAH-WP-0003-T02 / ADR-002. Explicit local
|
||||
TaskSpec grants activate repository acceptance; queued/profiled grant carriage
|
||||
remains an upstream contract dependency.
|
||||
TaskSpec and profiled Activity Core grants activate repository acceptance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -318,6 +317,24 @@ class RepositoryTransaction:
|
|||
"branch-changed",
|
||||
f"expected={baseline.branch or 'detached'} actual={post.branch or 'detached'}",
|
||||
)
|
||||
if not post.clean:
|
||||
raise RepositoryAcceptanceError(
|
||||
"dirty-post-state",
|
||||
f"entries={post.dirty_entries} status_digest={post.status_digest}",
|
||||
)
|
||||
if post.remote_refs != baseline.remote_refs:
|
||||
raise RepositoryAcceptanceError(
|
||||
"remote-refs-changed",
|
||||
"local remote-tracking refs moved during the transaction",
|
||||
)
|
||||
if (
|
||||
post.protected_git_metadata_digest
|
||||
!= baseline.protected_git_metadata_digest
|
||||
):
|
||||
raise RepositoryAcceptanceError(
|
||||
"git-metadata-changed",
|
||||
"protected Git config, hooks, or info metadata changed",
|
||||
)
|
||||
if post.head == baseline.head:
|
||||
raise RepositoryAcceptanceError("head-unchanged", "no new commit to accept")
|
||||
|
||||
|
|
@ -378,25 +395,6 @@ class RepositoryTransaction:
|
|||
f"count={len(invalid_paths)} paths={visible[:220]}",
|
||||
)
|
||||
|
||||
if not post.clean:
|
||||
raise RepositoryAcceptanceError(
|
||||
"dirty-post-state",
|
||||
f"entries={post.dirty_entries} status_digest={post.status_digest}",
|
||||
)
|
||||
if post.remote_refs != baseline.remote_refs:
|
||||
raise RepositoryAcceptanceError(
|
||||
"remote-refs-changed",
|
||||
"local remote-tracking refs moved during the transaction",
|
||||
)
|
||||
if (
|
||||
post.protected_git_metadata_digest
|
||||
!= baseline.protected_git_metadata_digest
|
||||
):
|
||||
raise RepositoryAcceptanceError(
|
||||
"git-metadata-changed",
|
||||
"protected Git config, hooks, or info metadata changed",
|
||||
)
|
||||
|
||||
acceptance = RepositoryAcceptance(
|
||||
policy_id=policy.policy_id,
|
||||
head=post.head,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue