coulomb/rein-aharness
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(consistency): sync task status from DB [auto]

Browse source 
Updated by fix-consistency on 2026-09-04:
  - update .custodian-brief.md for rein-aharness

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
This commit is contained in:
custodian-sync 2026-09-04 11:00:20 +02:00
parent 7641fcde40
commit f01b765668
20 changed files with 1027 additions and 165 deletions
Download patch file Download diff file Expand all files Collapse all files

179
rein_aharness/claim_loop.py
View file

@ -23,16 +23,30 @@ from rein_aharness.approaches import (
execute_approach,
select_approach,
)
from rein_aharness.execution_cancel import (
ExecutionCancel,
ExecutionCancelled,
using_cancel,
)
from rein_aharness.glas_execution import (
GLAS_APPROACH,
GlasExecutionError,
execute_profiled_run,
)
from rein_aharness.lease_monitor import LeaseMonitor
from rein_aharness.lease_monitor import LeaseLoss, LeaseMonitor
from rein_aharness.repository_transaction import (
DirtyRepositoryError,
GitRepositoryError,
RepositoryBusyError,
RepositoryTransaction,
RepositoryTransactionError,
)
from rein_aharness.taskspec import TaskSpecError
from rein_aharness.ops_run_client import (
ActivityCoreOpsClient,
OpsRun,
OpsRunConfig,
resolve_ops_target,
OpsRunError,
)
@ -57,19 +71,41 @@ def _heartbeat_interval(lease_seconds: int) -> float:
return max(30.0, min(300.0, lease_seconds / 3.0))
_active_run_cancel: ExecutionCancel | None = None
_active_run_cancel_lock = threading.Lock()
def _set_active_run_cancel(cancel: ExecutionCancel | None) -> None:
global _active_run_cancel
with _active_run_cancel_lock:
_active_run_cancel = cancel
def _cancel_active_run(reason: str) -> None:
with _active_run_cancel_lock:
cancel = _active_run_cancel
if cancel is not None:
cancel.cancel(reason)
class _Heartbeat:
def __init__(
self,
client: ActivityCoreOpsClient,
run_id: str,
lease_seconds: int,
cancel: ExecutionCancel | None = None,
):
self._client = client
self._run_id = run_id
self._lease = lease_seconds
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self.monitor = LeaseMonitor()
self.cancel = cancel if cancel is not None else ExecutionCancel()
self.monitor = LeaseMonitor(self._on_lost)
def _on_lost(self, _loss: LeaseLoss) -> None:
self.cancel.cancel("lease-loss")
def start(self) -> None:
interval = _heartbeat_interval(self._lease)
@ -109,6 +145,41 @@ class _Heartbeat:
self.monitor.stop()
def _cancelled_result(
run_id: str,
approach: str | None,
heartbeat: _Heartbeat,
) -> ProcessResult | None:
"""Skip Activity Core close after lease loss or execution cancellation."""
loss = heartbeat.monitor.evidence
if heartbeat.monitor.lost:
assert loss is not None
return ProcessResult(
claimed=True,
run_id=run_id,
approach=approach,
ok=False,
reason=f"lease lost ({loss.error_type})",
detail={
"lease_loss": {
"error_type": loss.error_type,
"observed_at": loss.observed_at,
}
},
)
if heartbeat.cancel.cancelled:
reason = heartbeat.cancel.reason or "cancelled"
return ProcessResult(
claimed=True,
run_id=run_id,
approach=approach,
ok=False,
reason=f"execution cancelled ({reason})",
detail={"cancellation": heartbeat.cancel.evidence()},
)
return None
def process_one(
client: ActivityCoreOpsClient | None = None,
*,
@ -175,39 +246,30 @@ def process_one(
report_to_hub=report_to_hub,
)
hb = _Heartbeat(client, run.id, cfg.lease_seconds)
cancel = ExecutionCancel()
hb = _Heartbeat(client, run.id, cfg.lease_seconds, cancel=cancel)
hb.start()
execution_error: str | None = None
_set_active_run_cancel(cancel)
try:
try:
ar: ApproachResult = execute_approach(
run,
approach=approach,
config=cfg,
report_to_hub=report_to_hub,
commit=commit,
)
with using_cancel(cancel):
ar: ApproachResult = execute_approach(
run,
approach=approach,
config=cfg,
report_to_hub=report_to_hub,
commit=commit,
)
except Exception as exc:
execution_error = type(exc).__name__
finally:
hb.stop()
_set_active_run_cancel(None)
if hb.monitor.lost:
loss = hb.monitor.evidence
assert loss is not None
return ProcessResult(
claimed=True,
run_id=run.id,
approach=approach,
ok=False,
reason=f"lease lost ({loss.error_type})",
detail={
"lease_loss": {
"error_type": loss.error_type,
"observed_at": loss.observed_at,
}
},
)
refused = _cancelled_result(run.id, approach, hb)
if refused is not None:
return refused
if execution_error is not None:
return ProcessResult(
@ -280,40 +342,50 @@ def _process_profiled_run(
report_to_hub: bool,
) -> ProcessResult:
"""Execute a profiled row without consulting or falling back to legacy routing."""
hb = _Heartbeat(client, run.id, client.config.lease_seconds)
cancel = ExecutionCancel()
hb = _Heartbeat(client, run.id, client.config.lease_seconds, cancel=cancel)
hb.start()
execution_error: str | None = None
execution_reason = ""
execution_reopen = False
gateway_result: dict[str, Any] | None = None
tx_evidence: dict[str, Any] | None = None
_set_active_run_cancel(cancel)
try:
try:
gateway_result = execute_profiled_run(
run,
client.config,
report_to_hub=report_to_hub,
)
target = resolve_ops_target(run, client.config)
with using_cancel(cancel):
with RepositoryTransaction(target, correlation_id=run.id) as tx:
gateway_result = execute_profiled_run(
run,
client.config,
report_to_hub=report_to_hub,
cancel=cancel,
)
tx_evidence = tx.evidence()
except ExecutionCancelled as exc:
execution_error = type(exc).__name__
execution_reason = str(exc)
except DirtyRepositoryError as exc:
execution_error = type(exc).__name__
execution_reason = f"refused: {exc}"
except RepositoryBusyError as exc:
execution_error = type(exc).__name__
execution_reason = f"refused: {exc}"
execution_reopen = True
except (GitRepositoryError, RepositoryTransactionError, TaskSpecError) as exc:
execution_error = type(exc).__name__
execution_reason = f"refused: {exc}"
except GlasExecutionError as exc:
execution_error = type(exc).__name__
execution_reason = str(exc)
finally:
hb.stop()
_set_active_run_cancel(None)
if hb.monitor.lost:
loss = hb.monitor.evidence
assert loss is not None
return ProcessResult(
claimed=True,
run_id=run.id,
approach=GLAS_APPROACH,
ok=False,
reason=f"lease lost ({loss.error_type})",
detail={
"lease_loss": {
"error_type": loss.error_type,
"observed_at": loss.observed_at,
}
},
)
refused = _cancelled_result(run.id, GLAS_APPROACH, hb)
if refused is not None:
return refused
if execution_error is not None:
reason = execution_reason or f"profiled execution failed ({execution_error})"
@ -321,7 +393,7 @@ def _process_profiled_run(
out = client.fail(
run.id,
error=reason,
reopen=False,
reopen=execution_reopen,
result={"ok": False, "approach": GLAS_APPROACH, "reason": reason},
)
except OpsRunError as close_exc:
@ -345,6 +417,10 @@ def _process_profiled_run(
evidence = gateway_result["evidence"]
ok = gateway_result["ok"]
reason = str(evidence.get("error") or evidence.get("outcome") or "Glas execution failed")
detail = {"execution_evidence": evidence}
if tx_evidence is not None:
detail["repository_transaction"] = tx_evidence
gateway_result = {**gateway_result, "repository_transaction": tx_evidence}
try:
if ok:
out = client.complete(run.id, result=gateway_result)
@ -362,7 +438,7 @@ def _process_profiled_run(
approach=GLAS_APPROACH,
ok=False,
reason=f"close ops_run failed: {exc}; gateway_ok={ok} {reason}",
detail={"execution_evidence": evidence},
detail=detail,
)
return ProcessResult(
@ -372,7 +448,7 @@ def _process_profiled_run(
ok=ok,
reason="" if ok else reason,
ops_state=out.state,
detail={"execution_evidence": evidence},
detail=detail,
)
@ -423,6 +499,7 @@ def run_claim_loop(
def _handle_sig(*_args: Any) -> None:
logger.info("shutdown signal received")
stop.set()
_cancel_active_run("signal")
signal.signal(signal.SIGINT, _handle_sig)
signal.signal(signal.SIGTERM, _handle_sig)