chore(consistency): sync task status from DB [auto]
Updated by fix-consistency on 2026-09-04: - update .custodian-brief.md for rein-aharness Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a06ba0-10aa-7ea0-b20a-4f3fac39efe9
This commit is contained in:
parent
7641fcde40
commit
f01b765668
20 changed files with 1027 additions and 165 deletions
|
|
@ -33,6 +33,11 @@ from llm_connect.claude_code import ClaudeCodeAdapter
|
|||
from llm_connect.exceptions import LLMSubprocessError, LLMTimeoutError
|
||||
from llm_connect.models import LLMResponse, RunConfig
|
||||
|
||||
from rein_aharness.execution_cancel import (
|
||||
ExecutionCancel,
|
||||
ExecutionCancelled,
|
||||
resolve_cancel,
|
||||
)
|
||||
from rein_aharness.profiles import ToolProfile, get_profile
|
||||
|
||||
# Backward-compatible alias for the seed profile allow-list string.
|
||||
|
|
@ -41,6 +46,29 @@ ALLOWED_TOOLS = get_profile("green-commit-only").allowed_tools
|
|||
ToolEventCallback = Callable[[dict[str, Any]], None]
|
||||
|
||||
|
||||
def _kill_process(proc: subprocess.Popen[str] | Any) -> None:
|
||||
poll = getattr(proc, "poll", None)
|
||||
if callable(poll):
|
||||
try:
|
||||
status = poll()
|
||||
except Exception:
|
||||
status = None
|
||||
if isinstance(status, int):
|
||||
return
|
||||
kill = getattr(proc, "kill", None)
|
||||
if callable(kill):
|
||||
try:
|
||||
kill()
|
||||
except Exception:
|
||||
return
|
||||
wait = getattr(proc, "wait", None)
|
||||
if callable(wait):
|
||||
try:
|
||||
wait(timeout=2)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _is_tool_event(event: dict[str, Any]) -> bool:
|
||||
"""True for tool_use/tool_result content blocks and hook lifecycle events.
|
||||
|
||||
|
|
@ -64,11 +92,13 @@ class AgenticClaudeCodeAdapter(ClaudeCodeAdapter):
|
|||
*,
|
||||
tool_profile: str | ToolProfile = "green-commit-only",
|
||||
on_tool_event: ToolEventCallback | None = None,
|
||||
cancel: ExecutionCancel | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self._workdir = workdir
|
||||
self._on_tool_event = on_tool_event
|
||||
self._cancel = cancel
|
||||
if isinstance(tool_profile, ToolProfile):
|
||||
self._profile = tool_profile
|
||||
else:
|
||||
|
|
@ -105,27 +135,23 @@ class AgenticClaudeCodeAdapter(ClaudeCodeAdapter):
|
|||
return response
|
||||
|
||||
def _execute_blocking(self, cmd: list[str], prompt: str, timeout: int) -> LLMResponse:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=prompt,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
cwd=self._workdir,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise LLMTimeoutError(
|
||||
f"claude CLI timed out after {timeout}s", cause=exc
|
||||
) from exc
|
||||
if result.returncode != 0:
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
cwd=self._workdir,
|
||||
)
|
||||
stdout, stderr = self._wait_for_process(proc, prompt, timeout)
|
||||
if proc.returncode != 0:
|
||||
raise LLMSubprocessError(
|
||||
f"claude CLI exited with code {result.returncode}",
|
||||
return_code=result.returncode,
|
||||
stderr=result.stderr,
|
||||
f"claude CLI exited with code {proc.returncode}",
|
||||
return_code=proc.returncode,
|
||||
stderr=stderr,
|
||||
)
|
||||
return LLMResponse(
|
||||
content=result.stdout,
|
||||
content=stdout,
|
||||
model=self._model or "claude-code-cli",
|
||||
usage={},
|
||||
finish_reason="stop",
|
||||
|
|
@ -170,12 +196,7 @@ class AgenticClaudeCodeAdapter(ClaudeCodeAdapter):
|
|||
proc.stdin.write(prompt)
|
||||
proc.stdin.close()
|
||||
reader_thread.start()
|
||||
try:
|
||||
returncode = proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise LLMTimeoutError(f"claude CLI timed out after {timeout}s", cause=exc) from exc
|
||||
returncode = self._wait_for_process(proc, None, timeout, communicate=False)
|
||||
reader_thread.join(timeout=5)
|
||||
stderr = proc.stderr.read() if proc.stderr else ""
|
||||
|
||||
|
|
@ -200,6 +221,38 @@ class AgenticClaudeCodeAdapter(ClaudeCodeAdapter):
|
|||
},
|
||||
)
|
||||
|
||||
def _wait_for_process(
|
||||
self,
|
||||
proc: subprocess.Popen[str],
|
||||
prompt: str | None,
|
||||
timeout: int,
|
||||
*,
|
||||
communicate: bool = True,
|
||||
) -> Any:
|
||||
cancel = resolve_cancel(self._cancel)
|
||||
if cancel is not None:
|
||||
cancel.check()
|
||||
cancel.register_process(proc)
|
||||
try:
|
||||
if communicate:
|
||||
result: Any = proc.communicate(input=prompt, timeout=timeout)
|
||||
else:
|
||||
result = proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
_kill_process(proc)
|
||||
raise LLMTimeoutError(
|
||||
f"claude CLI timed out after {timeout}s", cause=exc
|
||||
) from exc
|
||||
except ExecutionCancelled:
|
||||
raise
|
||||
except Exception:
|
||||
if cancel is not None and cancel.cancelled:
|
||||
raise ExecutionCancelled(cancel.reason or "cancelled") from None
|
||||
raise
|
||||
if cancel is not None:
|
||||
cancel.check()
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _handle_stream_event(event: dict[str, Any], text_parts: list[str]) -> None:
|
||||
if event.get("type") != "assistant":
|
||||
|
|
|
|||
|
|
@ -17,6 +17,13 @@ from pathlib import Path
|
|||
from typing import Any
|
||||
|
||||
from rein_aharness.ops_run_client import OpsRun, OpsRunConfig, resolve_ops_target
|
||||
from rein_aharness.repository_transaction import (
|
||||
DirtyRepositoryError,
|
||||
GitRepositoryError,
|
||||
RepositoryBusyError,
|
||||
RepositoryTransaction,
|
||||
RepositoryTransactionError,
|
||||
)
|
||||
from rein_aharness.taskspec import TaskSpecError
|
||||
|
||||
# Approach command names (stable; used in metrics + ops_run.result)
|
||||
|
|
@ -188,7 +195,7 @@ def execute_approach(
|
|||
reopen=False,
|
||||
)
|
||||
|
||||
try:
|
||||
def _dispatch(tx: RepositoryTransaction | None) -> ApproachResult:
|
||||
if name == APPROACH_FI_RESEARCH_BRIEF:
|
||||
return _run_fi(target, report_to_hub=report_to_hub, commit=commit)
|
||||
if name == APPROACH_BRIEF_DAILY:
|
||||
|
|
@ -204,7 +211,53 @@ def execute_approach(
|
|||
target, report_to_hub=report_to_hub, commit=commit
|
||||
)
|
||||
if name == APPROACH_AGENT_SESSION:
|
||||
return _run_agent_session(run, target, report_to_hub=report_to_hub)
|
||||
return _run_agent_session(
|
||||
run, target, report_to_hub=report_to_hub, transaction=tx
|
||||
)
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"approach not implemented: {name}",
|
||||
reopen=False,
|
||||
)
|
||||
|
||||
if name == APPROACH_AGENT_SESSION:
|
||||
try:
|
||||
return _dispatch(None)
|
||||
except Exception as exc: # noqa: BLE001 — surface as fail ops_run
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"{type(exc).__name__}: {exc}",
|
||||
reopen=True,
|
||||
)
|
||||
|
||||
try:
|
||||
with RepositoryTransaction(target, correlation_id=run.id) as tx:
|
||||
result = _dispatch(tx)
|
||||
result.result["repository_transaction"] = tx.evidence()
|
||||
return result
|
||||
except DirtyRepositoryError as exc:
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"refused: {exc}",
|
||||
reopen=False,
|
||||
)
|
||||
except RepositoryBusyError as exc:
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"refused: {exc}",
|
||||
reopen=True,
|
||||
)
|
||||
except (GitRepositoryError, RepositoryTransactionError) as exc:
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"refused: {exc}",
|
||||
reopen=False,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface as fail ops_run
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
|
|
@ -213,14 +266,6 @@ def execute_approach(
|
|||
reopen=True,
|
||||
)
|
||||
|
||||
return ApproachResult(
|
||||
ok=False,
|
||||
approach=name,
|
||||
reason=f"approach not implemented: {name}",
|
||||
reopen=False,
|
||||
)
|
||||
|
||||
|
||||
def _run_fi(target: Path, *, report_to_hub: bool, commit: bool) -> ApproachResult:
|
||||
from rein_aharness.fi_research_brief import run_fi_research_brief
|
||||
|
||||
|
|
@ -360,7 +405,11 @@ def _run_mail_pipeline(
|
|||
|
||||
|
||||
def _run_agent_session(
|
||||
run: OpsRun, target: Path, *, report_to_hub: bool
|
||||
run: OpsRun,
|
||||
target: Path,
|
||||
*,
|
||||
report_to_hub: bool,
|
||||
transaction: RepositoryTransaction | None = None,
|
||||
) -> ApproachResult:
|
||||
from rein_aharness.ops_run_client import ops_run_to_taskspec
|
||||
from rein_aharness.runner import run_task
|
||||
|
|
@ -384,16 +433,19 @@ def _run_agent_session(
|
|||
)
|
||||
# target already resolved into TaskSpec
|
||||
assert spec.target_repo == target or True
|
||||
r = run_task(spec, report_to_hub=report_to_hub)
|
||||
r = run_task(spec, report_to_hub=report_to_hub, transaction=transaction)
|
||||
result = {
|
||||
"committed": r.committed,
|
||||
"head_after": r.head_after,
|
||||
"tool_profile": r.tool_profile,
|
||||
"tokens_spent": r.tokens_spent,
|
||||
}
|
||||
if r.transaction:
|
||||
result["repository_transaction"] = r.transaction
|
||||
return ApproachResult(
|
||||
ok=r.ok,
|
||||
approach=APPROACH_AGENT_SESSION,
|
||||
result={
|
||||
"committed": r.committed,
|
||||
"head_after": r.head_after,
|
||||
"tool_profile": r.tool_profile,
|
||||
"tokens_spent": r.tokens_spent,
|
||||
},
|
||||
result=result,
|
||||
reason=r.reason,
|
||||
reopen=not r.ok,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -23,16 +23,30 @@ from rein_aharness.approaches import (
|
|||
execute_approach,
|
||||
select_approach,
|
||||
)
|
||||
from rein_aharness.execution_cancel import (
|
||||
ExecutionCancel,
|
||||
ExecutionCancelled,
|
||||
using_cancel,
|
||||
)
|
||||
from rein_aharness.glas_execution import (
|
||||
GLAS_APPROACH,
|
||||
GlasExecutionError,
|
||||
execute_profiled_run,
|
||||
)
|
||||
from rein_aharness.lease_monitor import LeaseMonitor
|
||||
from rein_aharness.lease_monitor import LeaseLoss, LeaseMonitor
|
||||
from rein_aharness.repository_transaction import (
|
||||
DirtyRepositoryError,
|
||||
GitRepositoryError,
|
||||
RepositoryBusyError,
|
||||
RepositoryTransaction,
|
||||
RepositoryTransactionError,
|
||||
)
|
||||
from rein_aharness.taskspec import TaskSpecError
|
||||
from rein_aharness.ops_run_client import (
|
||||
ActivityCoreOpsClient,
|
||||
OpsRun,
|
||||
OpsRunConfig,
|
||||
resolve_ops_target,
|
||||
OpsRunError,
|
||||
)
|
||||
|
||||
|
|
@ -57,19 +71,41 @@ def _heartbeat_interval(lease_seconds: int) -> float:
|
|||
return max(30.0, min(300.0, lease_seconds / 3.0))
|
||||
|
||||
|
||||
_active_run_cancel: ExecutionCancel | None = None
|
||||
_active_run_cancel_lock = threading.Lock()
|
||||
|
||||
|
||||
def _set_active_run_cancel(cancel: ExecutionCancel | None) -> None:
|
||||
global _active_run_cancel
|
||||
with _active_run_cancel_lock:
|
||||
_active_run_cancel = cancel
|
||||
|
||||
|
||||
def _cancel_active_run(reason: str) -> None:
|
||||
with _active_run_cancel_lock:
|
||||
cancel = _active_run_cancel
|
||||
if cancel is not None:
|
||||
cancel.cancel(reason)
|
||||
|
||||
|
||||
class _Heartbeat:
|
||||
def __init__(
|
||||
self,
|
||||
client: ActivityCoreOpsClient,
|
||||
run_id: str,
|
||||
lease_seconds: int,
|
||||
cancel: ExecutionCancel | None = None,
|
||||
):
|
||||
self._client = client
|
||||
self._run_id = run_id
|
||||
self._lease = lease_seconds
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self.monitor = LeaseMonitor()
|
||||
self.cancel = cancel if cancel is not None else ExecutionCancel()
|
||||
self.monitor = LeaseMonitor(self._on_lost)
|
||||
|
||||
def _on_lost(self, _loss: LeaseLoss) -> None:
|
||||
self.cancel.cancel("lease-loss")
|
||||
|
||||
def start(self) -> None:
|
||||
interval = _heartbeat_interval(self._lease)
|
||||
|
|
@ -109,6 +145,41 @@ class _Heartbeat:
|
|||
self.monitor.stop()
|
||||
|
||||
|
||||
def _cancelled_result(
|
||||
run_id: str,
|
||||
approach: str | None,
|
||||
heartbeat: _Heartbeat,
|
||||
) -> ProcessResult | None:
|
||||
"""Skip Activity Core close after lease loss or execution cancellation."""
|
||||
loss = heartbeat.monitor.evidence
|
||||
if heartbeat.monitor.lost:
|
||||
assert loss is not None
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run_id,
|
||||
approach=approach,
|
||||
ok=False,
|
||||
reason=f"lease lost ({loss.error_type})",
|
||||
detail={
|
||||
"lease_loss": {
|
||||
"error_type": loss.error_type,
|
||||
"observed_at": loss.observed_at,
|
||||
}
|
||||
},
|
||||
)
|
||||
if heartbeat.cancel.cancelled:
|
||||
reason = heartbeat.cancel.reason or "cancelled"
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run_id,
|
||||
approach=approach,
|
||||
ok=False,
|
||||
reason=f"execution cancelled ({reason})",
|
||||
detail={"cancellation": heartbeat.cancel.evidence()},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def process_one(
|
||||
client: ActivityCoreOpsClient | None = None,
|
||||
*,
|
||||
|
|
@ -175,39 +246,30 @@ def process_one(
|
|||
report_to_hub=report_to_hub,
|
||||
)
|
||||
|
||||
hb = _Heartbeat(client, run.id, cfg.lease_seconds)
|
||||
cancel = ExecutionCancel()
|
||||
hb = _Heartbeat(client, run.id, cfg.lease_seconds, cancel=cancel)
|
||||
hb.start()
|
||||
execution_error: str | None = None
|
||||
_set_active_run_cancel(cancel)
|
||||
try:
|
||||
try:
|
||||
ar: ApproachResult = execute_approach(
|
||||
run,
|
||||
approach=approach,
|
||||
config=cfg,
|
||||
report_to_hub=report_to_hub,
|
||||
commit=commit,
|
||||
)
|
||||
with using_cancel(cancel):
|
||||
ar: ApproachResult = execute_approach(
|
||||
run,
|
||||
approach=approach,
|
||||
config=cfg,
|
||||
report_to_hub=report_to_hub,
|
||||
commit=commit,
|
||||
)
|
||||
except Exception as exc:
|
||||
execution_error = type(exc).__name__
|
||||
finally:
|
||||
hb.stop()
|
||||
_set_active_run_cancel(None)
|
||||
|
||||
if hb.monitor.lost:
|
||||
loss = hb.monitor.evidence
|
||||
assert loss is not None
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=approach,
|
||||
ok=False,
|
||||
reason=f"lease lost ({loss.error_type})",
|
||||
detail={
|
||||
"lease_loss": {
|
||||
"error_type": loss.error_type,
|
||||
"observed_at": loss.observed_at,
|
||||
}
|
||||
},
|
||||
)
|
||||
refused = _cancelled_result(run.id, approach, hb)
|
||||
if refused is not None:
|
||||
return refused
|
||||
|
||||
if execution_error is not None:
|
||||
return ProcessResult(
|
||||
|
|
@ -280,40 +342,50 @@ def _process_profiled_run(
|
|||
report_to_hub: bool,
|
||||
) -> ProcessResult:
|
||||
"""Execute a profiled row without consulting or falling back to legacy routing."""
|
||||
hb = _Heartbeat(client, run.id, client.config.lease_seconds)
|
||||
cancel = ExecutionCancel()
|
||||
hb = _Heartbeat(client, run.id, client.config.lease_seconds, cancel=cancel)
|
||||
hb.start()
|
||||
execution_error: str | None = None
|
||||
execution_reason = ""
|
||||
execution_reopen = False
|
||||
gateway_result: dict[str, Any] | None = None
|
||||
tx_evidence: dict[str, Any] | None = None
|
||||
_set_active_run_cancel(cancel)
|
||||
try:
|
||||
try:
|
||||
gateway_result = execute_profiled_run(
|
||||
run,
|
||||
client.config,
|
||||
report_to_hub=report_to_hub,
|
||||
)
|
||||
target = resolve_ops_target(run, client.config)
|
||||
with using_cancel(cancel):
|
||||
with RepositoryTransaction(target, correlation_id=run.id) as tx:
|
||||
gateway_result = execute_profiled_run(
|
||||
run,
|
||||
client.config,
|
||||
report_to_hub=report_to_hub,
|
||||
cancel=cancel,
|
||||
)
|
||||
tx_evidence = tx.evidence()
|
||||
except ExecutionCancelled as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = str(exc)
|
||||
except DirtyRepositoryError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
except RepositoryBusyError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
execution_reopen = True
|
||||
except (GitRepositoryError, RepositoryTransactionError, TaskSpecError) as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = f"refused: {exc}"
|
||||
except GlasExecutionError as exc:
|
||||
execution_error = type(exc).__name__
|
||||
execution_reason = str(exc)
|
||||
finally:
|
||||
hb.stop()
|
||||
_set_active_run_cancel(None)
|
||||
|
||||
if hb.monitor.lost:
|
||||
loss = hb.monitor.evidence
|
||||
assert loss is not None
|
||||
return ProcessResult(
|
||||
claimed=True,
|
||||
run_id=run.id,
|
||||
approach=GLAS_APPROACH,
|
||||
ok=False,
|
||||
reason=f"lease lost ({loss.error_type})",
|
||||
detail={
|
||||
"lease_loss": {
|
||||
"error_type": loss.error_type,
|
||||
"observed_at": loss.observed_at,
|
||||
}
|
||||
},
|
||||
)
|
||||
refused = _cancelled_result(run.id, GLAS_APPROACH, hb)
|
||||
if refused is not None:
|
||||
return refused
|
||||
|
||||
if execution_error is not None:
|
||||
reason = execution_reason or f"profiled execution failed ({execution_error})"
|
||||
|
|
@ -321,7 +393,7 @@ def _process_profiled_run(
|
|||
out = client.fail(
|
||||
run.id,
|
||||
error=reason,
|
||||
reopen=False,
|
||||
reopen=execution_reopen,
|
||||
result={"ok": False, "approach": GLAS_APPROACH, "reason": reason},
|
||||
)
|
||||
except OpsRunError as close_exc:
|
||||
|
|
@ -345,6 +417,10 @@ def _process_profiled_run(
|
|||
evidence = gateway_result["evidence"]
|
||||
ok = gateway_result["ok"]
|
||||
reason = str(evidence.get("error") or evidence.get("outcome") or "Glas execution failed")
|
||||
detail = {"execution_evidence": evidence}
|
||||
if tx_evidence is not None:
|
||||
detail["repository_transaction"] = tx_evidence
|
||||
gateway_result = {**gateway_result, "repository_transaction": tx_evidence}
|
||||
try:
|
||||
if ok:
|
||||
out = client.complete(run.id, result=gateway_result)
|
||||
|
|
@ -362,7 +438,7 @@ def _process_profiled_run(
|
|||
approach=GLAS_APPROACH,
|
||||
ok=False,
|
||||
reason=f"close ops_run failed: {exc}; gateway_ok={ok} {reason}",
|
||||
detail={"execution_evidence": evidence},
|
||||
detail=detail,
|
||||
)
|
||||
|
||||
return ProcessResult(
|
||||
|
|
@ -372,7 +448,7 @@ def _process_profiled_run(
|
|||
ok=ok,
|
||||
reason="" if ok else reason,
|
||||
ops_state=out.state,
|
||||
detail={"execution_evidence": evidence},
|
||||
detail=detail,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -423,6 +499,7 @@ def run_claim_loop(
|
|||
def _handle_sig(*_args: Any) -> None:
|
||||
logger.info("shutdown signal received")
|
||||
stop.set()
|
||||
_cancel_active_run("signal")
|
||||
|
||||
signal.signal(signal.SIGINT, _handle_sig)
|
||||
signal.signal(signal.SIGTERM, _handle_sig)
|
||||
|
|
|
|||
141
rein_aharness/execution_cancel.py
Normal file
141
rein_aharness/execution_cancel.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
"""One-shot cancellation for an in-flight adapter call.
|
||||
|
||||
Lease loss, run timeout, and process signals all need the same adapter-side
|
||||
effect: stop the current subprocess or HTTP client without retaining provider
|
||||
output. The claim loop owns the reason; adapters only observe and stop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
_ALLOWED_REASONS = frozenset({"lease-loss", "timeout", "signal"})
|
||||
_REASON_LIMIT = 32
|
||||
|
||||
_active: contextvars.ContextVar["ExecutionCancel | None"] = contextvars.ContextVar(
|
||||
"rein_aharness_execution_cancel",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
class ExecutionCancelled(RuntimeError):
|
||||
"""Bounded proof that an adapter stopped because the run was cancelled."""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
self.reason = _normalize_reason(reason)
|
||||
super().__init__(f"execution cancelled ({self.reason})")
|
||||
|
||||
|
||||
class ExecutionCancel:
|
||||
"""Thread-safe one-shot cancel with optional process/HTTP stop callbacks."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._event = threading.Event()
|
||||
self._reason: str | None = None
|
||||
self._stops: list[Callable[[], None]] = []
|
||||
|
||||
@property
|
||||
def cancelled(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
@property
|
||||
def reason(self) -> str | None:
|
||||
with self._lock:
|
||||
return self._reason
|
||||
|
||||
def wait(self, timeout: float | None = None) -> bool:
|
||||
return self._event.wait(timeout)
|
||||
|
||||
def check(self) -> None:
|
||||
if self.cancelled:
|
||||
raise ExecutionCancelled(self.reason or "cancelled")
|
||||
|
||||
def cancel(self, reason: str) -> str:
|
||||
"""Record the first reason and invoke registered stop callbacks once."""
|
||||
callbacks: list[Callable[[], None]] = []
|
||||
with self._lock:
|
||||
if self._reason is None:
|
||||
self._reason = _normalize_reason(reason)
|
||||
self._event.set()
|
||||
callbacks = list(self._stops)
|
||||
recorded = self._reason
|
||||
for callback in callbacks:
|
||||
_invoke_stop(callback)
|
||||
return recorded
|
||||
|
||||
def register_stop(self, stop: Callable[[], None]) -> None:
|
||||
"""Register a best-effort stopper; invoke immediately if already cancelled."""
|
||||
invoke_now = False
|
||||
with self._lock:
|
||||
self._stops.append(stop)
|
||||
invoke_now = self._reason is not None
|
||||
if invoke_now:
|
||||
_invoke_stop(stop)
|
||||
|
||||
def register_process(self, proc: Any) -> None:
|
||||
"""Kill an in-flight subprocess. Safe against mock and exited processes."""
|
||||
self.register_stop(lambda: _kill_process(proc))
|
||||
|
||||
def evidence(self) -> dict[str, str]:
|
||||
if not self.cancelled:
|
||||
return {}
|
||||
return {"cancelled": "true", "reason": self.reason or "cancelled"}
|
||||
|
||||
|
||||
def active_cancel() -> ExecutionCancel | None:
|
||||
return _active.get()
|
||||
|
||||
|
||||
def resolve_cancel(explicit: ExecutionCancel | None = None) -> ExecutionCancel | None:
|
||||
return explicit if explicit is not None else active_cancel()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def using_cancel(cancel: ExecutionCancel) -> Iterator[ExecutionCancel]:
|
||||
token = _active.set(cancel)
|
||||
try:
|
||||
yield cancel
|
||||
finally:
|
||||
_active.reset(token)
|
||||
|
||||
|
||||
def _normalize_reason(reason: str) -> str:
|
||||
text = (reason or "").strip()[:_REASON_LIMIT]
|
||||
if text in _ALLOWED_REASONS:
|
||||
return text
|
||||
return "cancelled"
|
||||
|
||||
|
||||
def _invoke_stop(stop: Callable[[], None]) -> None:
|
||||
try:
|
||||
stop()
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _kill_process(proc: Any) -> None:
|
||||
poll = getattr(proc, "poll", None)
|
||||
if callable(poll):
|
||||
try:
|
||||
status = poll()
|
||||
except Exception:
|
||||
status = None
|
||||
if isinstance(status, int):
|
||||
return
|
||||
kill = getattr(proc, "kill", None)
|
||||
if callable(kill):
|
||||
try:
|
||||
kill()
|
||||
except Exception:
|
||||
return
|
||||
wait = getattr(proc, "wait", None)
|
||||
if callable(wait):
|
||||
try:
|
||||
wait(timeout=2)
|
||||
except Exception:
|
||||
return
|
||||
|
|
@ -11,6 +11,7 @@ from __future__ import annotations
|
|||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from rein_aharness.execution_cancel import ExecutionCancel, ExecutionCancelled, resolve_cancel
|
||||
from rein_aharness.ops_run_client import OpsRun, OpsRunConfig, resolve_ops_target
|
||||
|
||||
GLAS_APPROACH = "glas-profile"
|
||||
|
|
@ -65,8 +66,12 @@ def execute_profiled_run(
|
|||
report_to_hub: bool = True,
|
||||
request_factory: Callable[..., Any] | None = None,
|
||||
gateway: Callable[[Any], Any] | None = None,
|
||||
cancel: ExecutionCancel | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Invoke Glas and return its complete JSON-compatible GatewayResult."""
|
||||
guard = resolve_cancel(cancel)
|
||||
if guard is not None:
|
||||
guard.check()
|
||||
if request_factory is None or gateway is None:
|
||||
try:
|
||||
from glas_harness.contract import ExecutionRequest
|
||||
|
|
@ -83,10 +88,16 @@ def execute_profiled_run(
|
|||
request = request_factory(**_request_kwargs(run, config, report_to_hub))
|
||||
result = gateway(request)
|
||||
raw = result.model_dump(mode="json") if hasattr(result, "model_dump") else result
|
||||
except ExecutionCancelled:
|
||||
raise
|
||||
except GlasExecutionError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if guard is not None and guard.cancelled:
|
||||
raise ExecutionCancelled(guard.reason or "cancelled") from None
|
||||
raise GlasExecutionError(f"Glas gateway invocation failed: {exc}") from exc
|
||||
if guard is not None:
|
||||
guard.check()
|
||||
|
||||
if not isinstance(raw, dict) or not isinstance(raw.get("ok"), bool):
|
||||
raise GlasExecutionError("Glas gateway returned an invalid GatewayResult")
|
||||
|
|
|
|||
|
|
@ -11,6 +11,12 @@ from typing import Any
|
|||
|
||||
import httpx
|
||||
|
||||
from rein_aharness.execution_cancel import (
|
||||
ExecutionCancel,
|
||||
ExecutionCancelled,
|
||||
resolve_cancel,
|
||||
)
|
||||
|
||||
_SAFE_RESPONSE_METADATA_KEYS = frozenset(
|
||||
{
|
||||
"finish_reason",
|
||||
|
|
@ -50,20 +56,27 @@ class LLMConnectClient:
|
|||
*,
|
||||
model: str = "",
|
||||
config: dict[str, Any] | None = None,
|
||||
cancel: ExecutionCancel | None = None,
|
||||
) -> str:
|
||||
run_config = dict(config or {})
|
||||
if model and "model_name" not in run_config:
|
||||
run_config["model_name"] = model
|
||||
run_config.setdefault("timeout_seconds", int(self.timeout_seconds))
|
||||
payload: dict[str, Any] = {"prompt": prompt, "config": run_config}
|
||||
url = f"{self.base_url}/execute"
|
||||
guard = resolve_cancel(cancel)
|
||||
if guard is not None:
|
||||
guard.check()
|
||||
try:
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/execute",
|
||||
json=payload,
|
||||
timeout=self.timeout_seconds,
|
||||
)
|
||||
resp = self._post(url, payload, guard)
|
||||
except ExecutionCancelled:
|
||||
raise
|
||||
except httpx.HTTPError as exc:
|
||||
if guard is not None and guard.cancelled:
|
||||
raise ExecutionCancelled(guard.reason or "cancelled") from None
|
||||
raise LLMConnectError(f"llm-connect request failed: {exc}") from exc
|
||||
if guard is not None:
|
||||
guard.check()
|
||||
if resp.status_code >= 400:
|
||||
raise LLMConnectError(_llm_connect_error_text(resp))
|
||||
try:
|
||||
|
|
@ -76,6 +89,18 @@ class LLMConnectClient:
|
|||
raise LLMConnectError("llm-connect response missing string content")
|
||||
return content
|
||||
|
||||
def _post(
|
||||
self,
|
||||
url: str,
|
||||
payload: dict[str, Any],
|
||||
guard: ExecutionCancel | None,
|
||||
) -> httpx.Response:
|
||||
if guard is None:
|
||||
return httpx.post(url, json=payload, timeout=self.timeout_seconds)
|
||||
with httpx.Client(timeout=self.timeout_seconds) as client:
|
||||
guard.register_stop(client.close)
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
def _llm_connect_error_text(resp: httpx.Response) -> str:
|
||||
"""Keep llm-connect's safe cause without copying an upstream response blob."""
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
"""Process-safe transaction boundary for a local Git checkout.
|
||||
|
||||
This module is intentionally not wired into the claim loop yet. It provides
|
||||
the repository-local half of HARNESS-WP-0003-T02 while the Activity Core lease
|
||||
boundary is still under owner review.
|
||||
Wired into `run_task`, profile-absent `execute_approach` mutators, and the
|
||||
profiled claim path under REINAH-WP-0003-T02 / ADR-002. Repository
|
||||
acceptance (T03) remains a separate read-only validator and is not applied
|
||||
to live results yet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
|
|
@ -17,9 +17,17 @@ from pathlib import Path
|
|||
from typing import Any, Callable
|
||||
|
||||
from rein_aharness import hub, metrics
|
||||
from rein_aharness.execution_cancel import ExecutionCancelled
|
||||
from rein_aharness.manifest import resolve_run_policy
|
||||
from rein_aharness.persona import load_persona_bundle
|
||||
from rein_aharness.profiles import UnknownToolProfileError, get_profile
|
||||
from rein_aharness.repository_transaction import (
|
||||
DirtyRepositoryError,
|
||||
GitRepositoryError,
|
||||
RepositoryBusyError,
|
||||
RepositoryTransaction,
|
||||
RepositoryTransactionError,
|
||||
)
|
||||
from rein_aharness.taskspec import TaskSpec
|
||||
|
||||
PROMPT_TEMPLATE = """\
|
||||
|
|
@ -59,6 +67,7 @@ class RunResult:
|
|||
# called with emit_tool_events=True. See adapter.py's module docstring
|
||||
# for why this is observation, not external tool dispatch.
|
||||
tool_events: list[dict[str, Any]] = field(default_factory=list)
|
||||
transaction: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def _git(repo: Path, *args: str) -> str:
|
||||
|
|
@ -73,6 +82,28 @@ def _git(repo: Path, *args: str) -> str:
|
|||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _refused_run(
|
||||
*,
|
||||
reason: str,
|
||||
model: str | None = None,
|
||||
head_before: str = "",
|
||||
transaction: dict[str, Any] | None = None,
|
||||
) -> RunResult:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before=head_before,
|
||||
head_after=head_before,
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
reason=reason,
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
model=model,
|
||||
transaction=transaction,
|
||||
)
|
||||
|
||||
|
||||
def run_task(
|
||||
spec: TaskSpec,
|
||||
adapter=None,
|
||||
|
|
@ -83,21 +114,14 @@ def run_task(
|
|||
model: str | None = None,
|
||||
tool_profile_override: str | None = None,
|
||||
budget_tokens_override: int | None = None,
|
||||
transaction: RepositoryTransaction | None = None,
|
||||
) -> RunResult:
|
||||
if spec.repository_grant is not None:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before="",
|
||||
head_after="",
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
return _refused_run(
|
||||
reason=(
|
||||
"refused: repository_grant enforcement is not enabled; "
|
||||
"no adapter was dispatched"
|
||||
),
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
model=model,
|
||||
)
|
||||
try:
|
||||
|
|
@ -110,31 +134,9 @@ def run_task(
|
|||
budget_tokens = budget_tokens_override
|
||||
profile = get_profile(profile_name)
|
||||
except UnknownToolProfileError as exc:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before="",
|
||||
head_after="",
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
reason=f"refused: {exc}",
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
model=model,
|
||||
)
|
||||
return _refused_run(reason=f"refused: {exc}", model=model)
|
||||
except Exception as exc:
|
||||
return RunResult(
|
||||
ok=False,
|
||||
committed=False,
|
||||
head_before="",
|
||||
head_after="",
|
||||
persona_source="none",
|
||||
session_output="",
|
||||
reason=f"manifest resolution failed: {exc}",
|
||||
tool_profile="",
|
||||
budget_tokens=None,
|
||||
model=model,
|
||||
)
|
||||
return _refused_run(reason=f"manifest resolution failed: {exc}", model=model)
|
||||
|
||||
collected_events: list[dict[str, Any]] = []
|
||||
|
||||
|
|
@ -162,7 +164,58 @@ def run_task(
|
|||
adapter_kwargs["model"] = model
|
||||
adapter = AgenticClaudeCodeAdapter(**adapter_kwargs)
|
||||
|
||||
head_before = _git(spec.target_repo, "rev-parse", "HEAD")
|
||||
def _run_locked(tx: RepositoryTransaction) -> RunResult:
|
||||
return _execute_locked_task(
|
||||
spec,
|
||||
tx,
|
||||
adapter=adapter,
|
||||
profile=profile,
|
||||
blueprint=blueprint,
|
||||
lane=lane,
|
||||
budget_tokens=budget_tokens,
|
||||
collected_events=collected_events,
|
||||
report_to_hub=report_to_hub,
|
||||
write_metrics=write_metrics,
|
||||
model=model,
|
||||
)
|
||||
|
||||
if transaction is not None:
|
||||
return _run_locked(transaction)
|
||||
try:
|
||||
with RepositoryTransaction(
|
||||
spec.target_repo,
|
||||
correlation_id=spec.hub_task_id or spec.title,
|
||||
) as tx:
|
||||
return _run_locked(tx)
|
||||
except DirtyRepositoryError as exc:
|
||||
return _refused_run(
|
||||
reason=f"refused: {exc}",
|
||||
model=model,
|
||||
head_before=exc.baseline.head,
|
||||
transaction={"baseline": exc.baseline.evidence()},
|
||||
)
|
||||
except RepositoryBusyError as exc:
|
||||
return _refused_run(reason=f"refused: {exc}", model=model)
|
||||
except (GitRepositoryError, RepositoryTransactionError) as exc:
|
||||
return _refused_run(reason=f"refused: {exc}", model=model)
|
||||
|
||||
|
||||
def _execute_locked_task(
|
||||
spec: TaskSpec,
|
||||
tx: RepositoryTransaction,
|
||||
*,
|
||||
adapter: Any,
|
||||
profile: Any,
|
||||
blueprint: str,
|
||||
lane: str | None,
|
||||
budget_tokens: int | None,
|
||||
collected_events: list[dict[str, Any]],
|
||||
report_to_hub: bool,
|
||||
write_metrics: bool,
|
||||
model: str | None,
|
||||
) -> RunResult:
|
||||
assert tx.baseline is not None
|
||||
head_before = tx.baseline.head
|
||||
persona, persona_source = load_persona_bundle(blueprint, spec.target_repo)
|
||||
prompt = PROMPT_TEMPLATE.format(
|
||||
persona=persona or "(no persona bundle available for this run)",
|
||||
|
|
@ -188,6 +241,11 @@ def run_task(
|
|||
resolved_model = response.model
|
||||
session_ok = True
|
||||
reason = ""
|
||||
except ExecutionCancelled as exc:
|
||||
session_output = ""
|
||||
session_ok = False
|
||||
reason = f"execution cancelled ({exc.reason})"
|
||||
resolved_model = model
|
||||
except Exception as exc: # adapter / budget failures must still be reported
|
||||
session_output = ""
|
||||
session_ok = False
|
||||
|
|
@ -217,6 +275,7 @@ def run_task(
|
|||
tokens_spent=tokens_spent,
|
||||
execution_time_s=execution_time_s,
|
||||
tool_events=collected_events,
|
||||
transaction=tx.evidence(),
|
||||
)
|
||||
|
||||
if write_metrics:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue